Transitioned the in-memory representation of fpga definitions from json dicts to proto buffer objects.

Author: zapta

apio/commands/apio_api.py

@@ -397,7 +397,7 @@ def _get_project_cli(
         "id": pr.fpga_id,
         "is-custom": apio_ctx.definitions.is_custom_fpga(pr.fpga_id),
     }
-    fpga_dict["definition"] = pr.fpga_info
+    fpga_dict["definition"] = proto_util.proto_to_json_dict(pr.fpga_definition)
     section_dict["fpga"] = fpga_dict
 
     programmer_dict = {
@@ -478,10 +478,9 @@ def _get_boards_cli(
 
         # -- Add board's fpga information.
         fpga_id = board_definition.fpga_id
-        fpga_info = apio_ctx.definitions.fpgas.get(fpga_id, {})
-        assert "id" not in fpga_info
+        fpga_definition = apio_ctx.definitions.fpgas[fpga_id]
         fpga_dict = {"id": fpga_id}
-        fpga_dict.update(fpga_info)
+        fpga_dict.update(proto_util.proto_to_json_dict(fpga_definition))
         board_dict["fpga"] = fpga_dict
 
         # -- Add board's programmer information.
@@ -552,8 +551,8 @@ def _get_fpgas_cli(
 
     # -- Generate the fpgas section
     section = {}
-    for fpga_id, fpga_info in apio_ctx.definitions.fpgas.items():
-        section[fpga_id] = fpga_info
+    for fpga_id, fpga_definition in apio_ctx.definitions.fpgas.items():
+        section[fpga_id] = proto_util.proto_to_json_dict(fpga_definition)
 
     top_dict["fpgas"] = section
 

apio/commands/apio_boards.py

@@ -22,6 +22,7 @@
 from apio.utils import util, cmd_util
 from apio.commands import options
 from apio.managers.examples import Examples
+from apio.common.proto.apio_common_pb2 import ApioArch
 from apio.apio_context import (
     ApioContext,
     PackagesPolicy,
@@ -36,7 +37,7 @@ class Entry:
 
     # pylint: disable=too-many-instance-attributes
 
-    board: str
+    board_id: str
     examples_count: str
     board_description: str
     fpga_arch: str
@@ -47,31 +48,30 @@ class Entry:
 
     def sort_key(self):
         """A key for sorting the fpga entries in our preferred order."""
-        return (util.fpga_arch_sort_key(self.fpga_arch), self.board.lower())
+        return (util.fpga_arch_sort_key(self.fpga_arch), self.board_id.lower())
 
 
-def _collect_board_entries(apio_ctx) -> List[Entry]:
+def _collect_board_entries(apio_ctx: ApioContext) -> List[Entry]:
 
     # -- Get examples counts by board. This is a sparse dictionary.
     examples = Examples(apio_ctx)
     examples_counts: Dict[str, int] = examples.count_examples_by_board()
 
     # -- Collect the boards info into a list of entires, one per board.
     result: List[Entry] = []
-    for board, board_definition in apio_ctx.definitions.boards.items():
+    for board_id, board_definition in apio_ctx.definitions.boards.items():
         fpga_id = board_definition.fpga_id
-        fpga_info = apio_ctx.definitions.fpgas.get(fpga_id, {})
-
-        examples_count = "   " + str(examples_counts.get(board, ""))
+        fpga_definition = apio_ctx.definitions.fpgas[fpga_id]
+        examples_count = "   " + str(examples_counts.get(board_id, ""))
         board_description = board_definition.description
-        fpga_arch = fpga_info.get("arch", "")
-        fpga_size = fpga_info.get("size", "")
-        fpga_part_num = fpga_info.get("part-num", "")
+        fpga_arch = ApioArch.Name(fpga_definition.arch)
+        fpga_size = fpga_definition.size
+        fpga_part_num = fpga_definition.part_num
         programmer_id = board_definition.programmer.id
 
         result.append(
             Entry(
-                board=board,
+                board_id=board_id,
                 examples_count=examples_count,
                 board_description=board_description,
                 fpga_arch=fpga_arch,
@@ -127,12 +127,12 @@ def _list_boards(apio_ctx: ApioContext, verbose: bool):
 
         # -- Collect row values.
         values = []
-        values.append(entry.board)
+        values.append(entry.board_id)
         values.append(str(entry.examples_count))
         if verbose:
             values.append(entry.board_description)
         values.append(entry.fpga_arch)
-        values.append(str(entry.fpga_size))
+        values.append(entry.fpga_size)
         if verbose:
             values.append(entry.fpga_id)
         values.append(entry.fpga_part_num)
@@ -169,7 +169,7 @@ def _list_boards_docs_format(apio_ctx: ApioContext):
     entries: List[Entry] = _collect_board_entries(apio_ctx)
 
     # -- Determine column sizes
-    w1 = max(len("BOARD"), *(len(entry.board) for entry in entries))
+    w1 = max(len("BOARD"), *(len(entry.board_id) for entry in entries))
     w2 = max(len("SIZE"), *(len(entry.fpga_size) for entry in entries))
     w3 = max(
         len("DESCRIPTION"),
@@ -222,7 +222,7 @@ def _list_boards_docs_format(apio_ctx: ApioContext):
 
         cwrite(
             "| {0} | {1} | {2} | {3} |\n".format(
-                entry.board.ljust(w1),
+                entry.board_id.ljust(w1),
                 entry.fpga_size.ljust(w2),
                 entry.board_description.ljust(w3),
                 entry.fpga_part_num.ljust(w4),

apio/commands/apio_fpgas.py

@@ -11,20 +11,23 @@
 from datetime import date
 from pathlib import Path
 from dataclasses import dataclass
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Tuple
 import click
 from rich.table import Table
 from rich import box
 from apio.common import apio_console
 from apio.common.apio_console import cout, ctable, cwrite
 from apio.common.apio_styles import INFO, BORDER, EMPH1
+from apio.common.proto.apio_common_pb2 import ApioArch
+from apio.common.proto.apio_definitions_pb2 import FpgaDefinition
+from apio.utils import proto_util
 from apio.apio_context import (
     ApioContext,
     PackagesPolicy,
     ProjectPolicy,
     RemoteConfigPolicy,
 )
-from apio.utils import util, cmd_util, resource_util
+from apio.utils import util, cmd_util
 from apio.commands import options
 
 
@@ -44,6 +47,16 @@ def sort_key(self):
         return (util.fpga_arch_sort_key(self.fpga_arch), self.fpga.lower())
 
 
+def _get_fpga_arch_params(fpga_definition: FpgaDefinition) -> Tuple[str, Dict]:
+    """Extracts the arch specific params of an fpga, Returns a tuple
+    with the field name and the field value."""
+    fpga_dict = proto_util.proto_to_json_dict(fpga_definition)
+    arch = fpga_dict["arch"]
+    field_name = arch + "-params"
+    field_value = fpga_dict[field_name]
+    return (field_name, field_value)
+
+
 def _collect_fpgas_entries(apio_ctx: ApioContext) -> List[Entry]:
     """Returns a sorted list of supported fpgas entries."""
 
@@ -56,15 +69,15 @@ def _collect_fpgas_entries(apio_ctx: ApioContext) -> List[Entry]:
 
     # -- Collect all entries.
     result: List[Entry] = []
-    for fpga_id, fpga_info in apio_ctx.definitions.fpgas.items():
+    for fpga_id, fpga_definition in apio_ctx.definitions.fpgas.items():
         # -- Construct the Entry for this fpga.
         board_count = boards_counts.get(fpga_id, 0)
-        fpga_arch = fpga_info.get("arch", "")
-        fpga_part_num = fpga_info.get("part-num", "")
-        fpga_size = fpga_info.get("size", "")
+        fpga_arch = fpga_definition.arch
+        fpga_part_num = fpga_definition.part_num
+        fpga_size = fpga_definition.size
 
         # -- Arch specific params summary string.
-        _, params = resource_util.get_fpga_arch_params(fpga_info)
+        _, params = _get_fpga_arch_params(fpga_definition)
         values = [f"\\[{v}]" for v in params.values()]
         fpga_params = " ".join(values)
 
@@ -73,7 +86,7 @@ def _collect_fpgas_entries(apio_ctx: ApioContext) -> List[Entry]:
             Entry(
                 fpga=fpga_id,
                 board_count=board_count,
-                fpga_arch=fpga_arch,
+                fpga_arch=ApioArch.Name(fpga_arch),
                 fpga_part_num=fpga_part_num,
                 fpga_size=fpga_size,
                 fpga_params=fpga_params,

apio/commands/apio_info.py

@@ -26,6 +26,7 @@
     ProjectPolicy,
     RemoteConfigPolicy,
 )
+from apio.common.proto.apio_common_pb2 import ApioArch
 from apio.utils.cmd_util import ApioGroup, ApioSubgroup, ApioCommand
 from apio.common.apio_themes import THEMES_TABLE, THEME_LIGHT
 from apio.managers.remote_config import (
@@ -265,9 +266,9 @@ def _project_cli(
     table.add_row("Board definition", board_definition)
     table.add_row("FPGA id", fpga_id)
     table.add_row("FPGA definition", fpga_definition)
-    table.add_row("FPGA part num", res.fpga_info.get("part-num", ""))
-    table.add_row("FPGA Architecture", res.fpga_info.get("arch", ""))
-    table.add_row("FPGA size", res.fpga_info.get("size", ""))
+    table.add_row("FPGA part num", res.fpga_definition.part_num)
+    table.add_row("FPGA Architecture", ApioArch.Name(res.fpga_definition.arch))
+    table.add_row("FPGA size", res.fpga_definition.size)
     table.add_row("Programmer id", programmer_id)
     table.add_row("Programmer definition", programmer_definition)
 

apio/common/proto/apio-common.proto

@@ -12,10 +12,11 @@ syntax = "proto2";
 package apio.common.proto;
 
 // The supported FPGA architectures.
+// Using lower case names to be json friendly.
 enum ApioArch {
-  ICE40 = 1;
-  ECP5 = 2;
-  GOWIN = 3;
-  XILINX = 4;
+  ice40 = 1;
+  ecp5 = 2;
+  gowin = 3;
+  xilinx = 4;
 }
 

apio/common/proto/apio_common_pb2.py

@@ -26,7 +26,7 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#apio/common/proto/apio-common.proto\x12\x11\x61pio.common.proto*6\n\x08\x41pioArch\x12\t\n\x05ICE40\x10\x01\x12\x08\n\x04\x45\x43P5\x10\x02\x12\t\n\x05GOWIN\x10\x03\x12\n\n\x06XILINX\x10\x04')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#apio/common/proto/apio-common.proto\x12\x11\x61pio.common.proto*6\n\x08\x41pioArch\x12\t\n\x05ice40\x10\x01\x12\x08\n\x04\x65\x63p5\x10\x02\x12\t\n\x05gowin\x10\x03\x12\n\n\x06xilinx\x10\x04')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)

apio/common/proto/apio_common_pb2.pyi

@@ -8,11 +8,11 @@ DESCRIPTOR: _descriptor.FileDescriptor
 
 class ApioArch(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
-    ICE40: _ClassVar[ApioArch]
-    ECP5: _ClassVar[ApioArch]
-    GOWIN: _ClassVar[ApioArch]
-    XILINX: _ClassVar[ApioArch]
-ICE40: ApioArch
-ECP5: ApioArch
-GOWIN: ApioArch
-XILINX: ApioArch
+    ice40: _ClassVar[ApioArch]
+    ecp5: _ClassVar[ApioArch]
+    gowin: _ClassVar[ApioArch]
+    xilinx: _ClassVar[ApioArch]
+ice40: ApioArch
+ecp5: ApioArch
+gowin: ApioArch
+xilinx: ApioArch

apio/managers/apio_definitions.py

@@ -17,7 +17,10 @@
 import json5
 from apio.utils import proto_util
 from apio.common.apio_console import cout, cerror
-from apio.common.proto.apio_definitions_pb2 import BoardDefinition
+from apio.common.proto.apio_definitions_pb2 import (
+    BoardDefinition,
+    FpgaDefinition,
+)
 
 # -- Boards definitions file name.
 BOARDS_JSONC = "boards.jsonc"
@@ -130,8 +133,7 @@ def __init__(
             self._project_definitions_dir,
         )
 
-        # -- Convert the board definition to BoardDefinition protos we keep
-        # -- in this object.
+        # -- Convert the board definition dicts to BoardDefinition protos.
         self.boards: Dict[str, BoardDefinition] = {}
         for board_id, definition_dict in boards_json.items():
             definition = proto_util.proto_from_json_dict(
@@ -149,34 +151,28 @@ def __init__(
                 sys.exit(1)
 
         # -- Read fpgas definitions.
-        self.fpgas, self.custom_fpgas_ids = self._load_definitions(
+        fpgas_json, self.custom_fpgas_ids = self._load_definitions(
             FPGAS_JSONC,
             self._package_definitions_dir,
             self._project_definitions_dir,
         )
 
-        # -- Validate fpgas definitions. Optional project custom definition
-        # -- supersede apio standard definitions.
-        for fpga_id, fpga_info in self.fpgas.items():
+        # -- Convert the fpgas definition dicts to FpgasDefinition protos.
+        self.fpgas: Dict[str, FpgaDefinition] = {}
+        for fpga_id, definition_dict in fpgas_json.items():
+            definition = proto_util.proto_from_json_dict(
+                definition_dict,
+                FpgaDefinition,
+                f"Failed to parse fpga definition '{fpga_id}",
+            )
+            self.fpgas[fpga_id] = definition
+
+        # -- Validate fpgas definitions.
+        for fpga_id, fpga_definition in self.fpgas.items():
             if not ID_FORMAT.match(fpga_id):
                 cerror(f"FPGA id has an invalid format: {fpga_id}")
                 sys.exit(1)
-            try:
-                validate(instance=fpga_info, schema=FPGA_SCHEMA)
-            except ValidationError as e:
-                cerror(f"Invalid fpga definition [{fpga_id}]: {e.message}")
-                sys.exit(1)
-
-            # -- Expecting a params field for the specified architecture.
-            params_pattern = re.compile(r".*-params$")
-            actual_params = [
-                key for key in fpga_info if params_pattern.match(key)
-            ]
-            expected_params = [fpga_info["arch"] + "-params"]
-            if actual_params != expected_params:
-                cerror(f"Unexpected params {actual_params} in fpga {fpga_id}")
-                sys.exit(1)
-            part_num = fpga_info["part-num"]
+            part_num = fpga_definition.part_num
             lc_part_num = part_num.lower().replace("/", "-")
             if fpga_id != lc_part_num and not fpga_id.startswith(
                 lc_part_num + "-"

apio/managers/examples.py

@@ -14,6 +14,7 @@
 from typing import Optional, List, Dict
 from apio.common.apio_console import cout, cstyle, cerror
 from apio.common.apio_styles import INFO, SUCCESS, EMPH3
+from apio.common.proto.apio_common_pb2 import ApioArch
 from apio.apio_context import ApioContext
 from apio.utils import util
 
@@ -122,10 +123,10 @@ def get_examples_infos(self) -> List[ExampleInfo]:
                 # -- Extract the fpga arch and part number, with "" as
                 # -- default value if not found.
                 fpga_id = board_definition.fpga_id
-                fpga_info = definitions.fpgas[fpga_id]
-                fpga_arch = fpga_info.get("arch", "")
-                fpga_part_num = fpga_info.get("part-num", "")
-                fpga_size = fpga_info.get("size", "")
+                fpga_definition = definitions.fpgas[fpga_id]
+                fpga_arch = ApioArch.Name(fpga_definition.arch)
+                fpga_part_num = fpga_definition.part_num
+                fpga_size = fpga_definition.size
 
                 # -- Append this example to the list.
                 example_info = ExampleInfo(