Merge pull request #988 from zapta/main

Transition the in-memory representation of programmer definitions fro…

Author: zapta

apio/commands/apio_api.py

@@ -406,7 +406,9 @@ def _get_project_cli(
             pr.programmer_id
         ),
     }
-    programmer_dict["definition"] = pr.programmer_info
+    programmer_dict["definition"] = proto_util.proto_to_json_dict(
+        pr.programmer_definition
+    )
     section_dict["programmer"] = programmer_dict
 
     # -- Add section
@@ -625,9 +627,11 @@ def _get_programmers_cli(
     section = {}
     for (
         programmer_id,
-        programmer_info,
+        programmer_definition,
     ) in apio_ctx.definitions.programmers.items():
-        section[programmer_id] = programmer_info
+        section[programmer_id] = proto_util.proto_to_json_dict(
+            programmer_definition
+        )
 
     top_dict["programmers"] = section
 

apio/managers/apio_definitions.py

@@ -12,14 +12,13 @@
 import re
 from pathlib import Path
 from typing import Dict, Set, Tuple, Optional
-from jsonschema import validate
-from jsonschema.exceptions import ValidationError
 import json5
 from apio.utils import proto_util
 from apio.common.apio_console import cout, cerror
 from apio.common.proto.apio_definitions_pb2 import (
     BoardDefinition,
     FpgaDefinition,
+    ProgrammerDefinition,
 )
 
 # -- Boards definitions file name.
@@ -32,83 +31,13 @@
 PROGRAMMERS_JSONC = "programmers.jsonc"
 
 # -- A regex for validating boards, fpgas, and programmers ids.
-ID_FORMAT = re.compile(r"^[a-z][a-z0-9-]*$")
-
-
-# -- JSON schema for validating a single fpga definition in fpga.jsonc.
-# -- The fields 'part-num' and 'size' are for information only.
-FPGA_SCHEMA = schema = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "type": "object",
-    "properties": {
-        "part-num": {"type": "string"},
-        "arch": {
-            "type": "string",
-            "enum": ["ice40", "ecp5", "gowin", "xilinx"],
-        },
-        "size": {"type": "string"},
-        "ice40-params": {
-            "type": "object",
-            "properties": {
-                "type": {"type": "string"},
-                "package": {"type": "string"},
-            },
-            "required": ["type", "package"],
-            "additionalProperties": False,
-        },
-        "ecp5-params": {
-            "type": "object",
-            "properties": {
-                "type": {"type": "string"},
-                "package": {"type": "string"},
-                "speed": {"type": "string"},
-            },
-            "required": ["type", "package", "speed"],
-            "additionalProperties": False,
-        },
-        "gowin-params": {
-            "type": "object",
-            "properties": {
-                "yosys-family": {"type": "string"},
-                "nextpnr-family": {"type": "string"},
-                "packer-device": {"type": "string"},
-            },
-            "required": ["yosys-family", "nextpnr-family", "packer-device"],
-            "additionalProperties": False,
-        },
-        "xilinx-params": {
-            "type": "object",
-            "properties": {
-                "family": {"type": "string"},
-                "yosys-arch": {"type": "string"},
-                "package": {"type": "string"},
-                "speed": {"type": "string"},
-            },
-            "required": ["family", "yosys-arch", "package", "speed"],
-            "additionalProperties": False,
-        },
-    },
-    "required": ["part-num", "arch", "size"],
-    "additionalProperties": False,
-}
-
-
-# -- JSON schema for validating a single programmer definition in
-# -- programmers.jsonc.
-PROGRAMMER_SCHEMA = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "type": "object",
-    "required": ["command", "args"],
-    "properties": {"command": {"type": "string"}, "args": {"type": "string"}},
-    "additionalProperties": False,
-}
+DEFINITION_ID_FORMAT = re.compile(r"^[a-z][a-z0-9-]*$")
 
 
 class ApioDefinitions:
     """Contains the apio definitions in the form of json dictionaries."""
 
     # pylint: disable=too-many-instance-attributes
-    # pylint: disable=too-many-locals
 
     def __init__(
         self,
@@ -146,7 +75,7 @@ def __init__(
         # -- Validate boards definitions. Optional project custom definition
         # -- supersede apio standard definitions.
         for board_id in self.boards:
-            if not ID_FORMAT.match(board_id):
+            if not DEFINITION_ID_FORMAT.match(board_id):
                 cerror(f"Board id has an invalid format: {board_id}")
                 sys.exit(1)
 
@@ -169,7 +98,7 @@ def __init__(
 
         # -- Validate fpgas definitions.
         for fpga_id, fpga_definition in self.fpgas.items():
-            if not ID_FORMAT.match(fpga_id):
+            if not DEFINITION_ID_FORMAT.match(fpga_id):
                 cerror(f"FPGA id has an invalid format: {fpga_id}")
                 sys.exit(1)
             proto_util.check_is_required(fpga_definition, "part_num")
@@ -185,25 +114,27 @@ def __init__(
 
         # -- Load programmers definitions. Optional project custom definition
         # -- supersede apio standard definitions.
-        self.programmers, self.custom_programmers_ids = self._load_definitions(
+        programmers_json, self.custom_programmers_ids = self._load_definitions(
             PROGRAMMERS_JSONC,
             self._package_definitions_dir,
             self._project_definitions_dir,
         )
 
+        # -- Convert the programmers definition dicts to FpgaDefinition protos.
+        self.programmers: Dict[str, ProgrammerDefinition] = {}
+        for programmer_id, definition_dict in programmers_json.items():
+            definition = proto_util.proto_from_json_dict(
+                definition_dict,
+                ProgrammerDefinition,
+                f"Failed to parse programmer definition '{programmer_id}",
+            )
+            self.programmers[programmer_id] = definition
+
         # -- Validate programmers definitions.
-        for programmer_id, programmer_info in self.programmers.items():
-            if not ID_FORMAT.match(programmer_id):
+        for programmer_id in self.programmers:
+            if not DEFINITION_ID_FORMAT.match(programmer_id):
                 cerror(f"Programmer id has an invalid format: {programmer_id}")
                 sys.exit(1)
-            try:
-                validate(instance=programmer_info, schema=PROGRAMMER_SCHEMA)
-            except ValidationError as e:
-                cerror(
-                    f"Invalid programmer definition [{programmer_id}]: "
-                    f"{e.message}"
-                )
-                sys.exit(1)
 
         # -- Check references from boards to fpga and programmers
         for board_id, board_definition in self.boards.items():

apio/managers/programmers.py

@@ -171,16 +171,19 @@ def _construct_cmd_template(apio_ctx: ApioContext) -> str:
 
     pr = apio_ctx.project_resources
     board_definition = pr.board_definition
-    programmer_info = pr.programmer_info
+    programmer_definition = pr.programmer_definition
 
     # -- Here when using the standard command.
 
     # -- Start building the template with the programmer binary name.
     # -- E.g. "openFPGAloader". "command" is a validated required field.
-    cmd_template = programmer_info["command"]
+    proto_util.check_is_required(programmer_definition, "command")
+    cmd_template = programmer_definition.command
 
-    # -- Append the optional args template from the programmer.
-    args = pr.programmer_info.get("args", "")
+    # -- Append the optional args template from the programmer. The 'args'
+    # -- field is required but may be empty.
+    proto_util.check_is_required(programmer_definition, "args")
+    args = programmer_definition.args
     if args:
         cmd_template += " "
         cmd_template += args

apio/utils/resource_util.py

@@ -1,7 +1,6 @@
 """Utilities related to the Apio resource files."""
 
 import sys
-from typing import Any, Dict
 from dataclasses import dataclass
 from jsonschema import validate
 from jsonschema.exceptions import ValidationError
@@ -11,6 +10,7 @@
 from apio.common.proto.apio_definitions_pb2 import (
     BoardDefinition,
     FpgaDefinition,
+    ProgrammerDefinition,
 )
 
 
@@ -23,7 +23,7 @@ class ProjectResources:
     fpga_id: str
     fpga_definition: FpgaDefinition
     programmer_id: str
-    programmer_info: Dict[str, Any]
+    programmer_definition: ProgrammerDefinition
 
 
 # -- JSON schema for validating config.jsonc.
@@ -127,10 +127,7 @@ def collect_project_resources(
 
     # -- Get programmer id and info.
     programmer_id = board_definition.programmer.id
-    programmer_info = definitions.programmers.get(programmer_id, None)
-    if programmer_info is None:
-        cerror(f"Unknown programmer id '{programmer_id}'.")
-        sys.exit(1)
+    programmer_definition = definitions.programmers[programmer_id]
 
     # -- Create the project resources bundle.
     project_resources = ProjectResources(
@@ -139,7 +136,7 @@ def collect_project_resources(
         fpga_id,
         fpga_definition,
         programmer_id,
-        programmer_info,
+        programmer_definition,
     )
 
     # -- All done

tests/unit_tests/managers/test_apio_definitions.py

@@ -228,9 +228,18 @@ def test_loading_with_custom_programmer(apio_runner: ApioRunner):
         assert "alhambra-ii" in definitions.boards
         assert "ice40hx4k-tq144-8k" in definitions.fpgas
 
-        assert definitions.programmers["iceprog"] == programmer_info1
+        programmer_definition1 = definitions.programmers["iceprog"]
         assert (
-            definitions.programmers["my-custom-programmer"] == programmer_info2
+            proto_util.proto_to_json_dict(programmer_definition1)
+            == programmer_info1
+        )
+
+        programmer_definition2 = definitions.programmers[
+            "my-custom-programmer"
+        ]
+        assert (
+            proto_util.proto_to_json_dict(programmer_definition2)
+            == programmer_info2
         )
 
         assert definitions.is_custom_programmer("iceprog")
@@ -343,7 +352,7 @@ def test_loading_invalid_custom_programmer(apio_runner: ApioRunner):
 
         # -- Verify
         assert e.value.code == 1
-        assert "'command' is a required property" in log.out
+        assert "Missing required field 'command'" in log.out
 
 
 def test_loading_invalid_custom_board_id(apio_runner: ApioRunner):