Transitioned the in-memory board definitions representation from json dicts to protocol buffer messages. Next will do the same for fpgas and programmers.

Author: zapta

apio/commands/apio_api.py

@@ -25,6 +25,7 @@
     usb_util,
     serial_util,
     util,
+    proto_util,
     apio_platforms,
     env_options,
 )
@@ -387,7 +388,9 @@ def _get_project_cli(
         "id": pr.board_id,
         "is-custom": apio_ctx.definitions.is_custom_board(pr.board_id),
     }
-    board_dict["definition"] = pr.board_info
+    board_dict["definition"] = proto_util.proto_to_json_dict(
+        pr.board_definition
+    )
     section_dict["board"] = board_dict
 
     fpga_dict = {
@@ -466,15 +469,15 @@ def _get_boards_cli(
 
     # -- Generate the boards section.
     section = {}
-    for board_id, board_info in apio_ctx.definitions.boards.items():
+    for board_id, board_definition in apio_ctx.definitions.boards.items():
         # -- The board output dict.
         board_dict = {}
 
         # -- Add board description
-        board_dict["description"] = board_info.get("description", None)
+        board_dict["description"] = board_definition.description
 
         # -- Add board's fpga information.
-        fpga_id = board_info.get("fpga-id", None)
+        fpga_id = board_definition.fpga_id
         fpga_info = apio_ctx.definitions.fpgas.get(fpga_id, {})
         assert "id" not in fpga_info
         fpga_dict = {"id": fpga_id}
@@ -483,7 +486,7 @@ def _get_boards_cli(
 
         # -- Add board's programmer information.
         programmer_dict = {}
-        programmer_id = board_info.get("programmer", {}).get("id", None)
+        programmer_id = board_definition.programmer.id
         programmer_dict["id"] = programmer_id
         board_dict["programmer"] = programmer_dict
 
@@ -944,8 +947,8 @@ def _scan_devices_cli(
     section = []
     for device in usb_devices:
         dev = {}
-        dev["vid"] = device.vendor_id
-        dev["pid"] = device.product_id
+        dev["vid"] = device.vid
+        dev["pid"] = device.pid
         dev["bus"] = device.bus
         dev["device"] = device.device
         dev["manufacturer"] = device.manufacturer
@@ -967,8 +970,8 @@ def _scan_devices_cli(
         dev = {}
         dev["port"] = device.port
         dev["port-name"] = device.port_name
-        dev["vendor-id"] = device.vendor_id
-        dev["product-id"] = device.product_id
+        dev["vendor-id"] = device.vid
+        dev["product-id"] = device.pid
         dev["manufacturer"] = device.manufacturer
         dev["product"] = device.product
         dev["serial-number"] = device.serial_number

apio/commands/apio_boards.py

@@ -58,16 +58,16 @@ def _collect_board_entries(apio_ctx) -> List[Entry]:
 
     # -- Collect the boards info into a list of entires, one per board.
     result: List[Entry] = []
-    for board, board_info in apio_ctx.definitions.boards.items():
-        fpga_id = board_info.get("fpga-id", "")
+    for board, board_definition in apio_ctx.definitions.boards.items():
+        fpga_id = board_definition.fpga_id
         fpga_info = apio_ctx.definitions.fpgas.get(fpga_id, {})
 
         examples_count = "   " + str(examples_counts.get(board, ""))
-        board_description = board_info.get("description", "")
+        board_description = board_definition.description
         fpga_arch = fpga_info.get("arch", "")
         fpga_size = fpga_info.get("size", "")
         fpga_part_num = fpga_info.get("part-num", "")
-        programmer_id = board_info.get("programmer", {}).get("id", "")
+        programmer_id = board_definition.programmer.id
 
         result.append(
             Entry(

apio/commands/apio_devices.py

@@ -57,7 +57,7 @@ def _list_usb_devices(apio_ctx: ApioContext) -> None:
     # -- Add a raw per device
     for device in devices:
         values = []
-        values.append(f"{device.vendor_id}:{device.product_id}")
+        values.append(f"{device.vid}:{device.pid}")
         values.append(f"{device.bus}:{device.device}")
         values.append(device.manufacturer)
         values.append(device.product)
@@ -141,7 +141,7 @@ def _list_serial_devices(apio_ctx: ApioContext) -> None:
     for device in devices:
         values = []
         values.append(device.port)
-        values.append(f"{device.vendor_id}:{device.product_id}")
+        values.append(f"{device.vid}:{device.pid}")
         values.append(device.manufacturer)
         values.append(device.product)
         values.append(device.serial_number)

apio/commands/apio_fpgas.py

@@ -49,11 +49,10 @@ def _collect_fpgas_entries(apio_ctx: ApioContext) -> List[Entry]:
 
     # -- Collect a sparse dict with fpga ids to board count.
     boards_counts: Dict[str, int] = {}
-    for board_info in apio_ctx.definitions.boards.values():
-        fpga_id = board_info.get("fpga-id", None)
-        if fpga_id:
-            old_count = boards_counts.get(fpga_id, 0)
-            boards_counts[fpga_id] = old_count + 1
+    for board_definition in apio_ctx.definitions.boards.values():
+        fpga_id = board_definition.fpga_id
+        old_count = boards_counts.get(fpga_id, 0)
+        boards_counts[fpga_id] = old_count + 1
 
     # -- Collect all entries.
     result: List[Entry] = []

apio/common/proto/apio-common.proto

@@ -5,12 +5,6 @@
 // 'invoke update-protos' to propagate them to the python stubs.
 //  Otherwise they do not take affect.
 
-// Online proto formatter at https://formatter.org/protobuf-formatter
-
-// NOTE: Since we use the the serialized proto data within a single invocation
-// of apio, protocol buffers text mode and binary mode backward compatibility
-// considerations do not apply.
-
 // Using proto2 for features such as 'has' and 'required'.
 syntax = "proto2";
 

apio/common/proto/apio-definitions.proto

@@ -5,12 +5,6 @@
 // 'invoke update-protos' to propagate them to the python stubs.
 //  Otherwise they do not take affect.
 
-// Online proto formatter at https://formatter.org/protobuf-formatter
-
-// NOTE: Since we use the the serialized proto data within a single invocation
-// of apio, protocol buffers text mode and binary mode backward compatibility
-// considerations do not apply.
-
 // Using proto2 for features such as 'has' and 'required'.
 syntax = "proto2";
 
@@ -28,8 +22,8 @@ message BoardProgrammerSection {
 
 // The board's USB info.
 message BoardUsbSection {
-  required string vid = 1;
-  required string pid = 2;
+  optional string vid = 1;
+  optional string pid = 2;
   optional string product_regex = 3 [json_name = "product-regex"];
 }
 

apio/common/proto/apio-scons.proto

@@ -4,16 +4,9 @@
 // 'invoke update-protos' to propagate them to the python stubs.
 //  Otherwise they do not take affect.
 
-// Online proto formatter at https://formatter.org/protobuf-formatter
-
-// NOTE: Since we use the the serialized proto data within a single invocation
-// of apio, protocol buffers text mode and binary mode backward compatibility
-// considerations do not apply.
-
 // Using proto2 for features such as 'has' and 'required'.
 syntax = "proto2";
 
-
 package apio.common.proto;
 
 import "apio/common/proto/apio-common.proto";

apio/common/proto/apio-testing.proto

@@ -0,0 +1,22 @@
+// Protocol buffers definitions used by apio tests only. The are not 
+// used by Apio runtime.
+//
+// IMPORTANT: After making changes in this file, run the repo task
+// 'invoke update-protos' to propagate them to the python stubs.
+//  Otherwise they do not take affect.
+
+// Using proto2 for features such as 'has' and 'required'.
+syntax = "proto2";
+
+package apio.common.proto;
+
+
+message MessageA {
+  required string field_a1 = 1 [json_name = "field-a1"];
+  optional string field_a2 = 2 [json_name = "field-a2"];
+}
+
+message MessageB {
+  required MessageA field_b1 = 1 [json_name = "field-b1"];
+  optional MessageA field_b2 = 2 [json_name = "field-b2"];
+}

apio/common/proto/apio_definitions_pb2.py

@@ -27,7 +27,7 @@
 from apio.common.proto import apio_common_pb2 as apio_dot_common_dot_proto_dot_apio__common__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(apio/common/proto/apio-definitions.proto\x12\x11\x61pio.common.proto\x1a#apio/common/proto/apio-common.proto\"D\n\x16\x42oardProgrammerSection\x12\n\n\x02id\x18\x01 \x02(\t\x12\x1e\n\nextra_args\x18\x02 \x01(\tR\nextra-args\"Q\n\x0f\x42oardUsbSection\x12\x0b\n\x03vid\x18\x01 \x02(\t\x12\x0b\n\x03pid\x18\x02 \x02(\t\x12$\n\rproduct_regex\x18\x03 \x01(\tR\rproduct-regex\"6\n\x14\x42oardTinyprogSection\x12\x1e\n\nname_regex\x18\x01 \x02(\tR\nname-regex\"\xeb\x01\n\x0f\x42oardDefinition\x12\x13\n\x0b\x64\x65scription\x18\x01 \x02(\t\x12\x18\n\x07\x66pga_id\x18\x02 \x02(\tR\x07\x66pga-id\x12=\n\nprogrammer\x18\x03 \x02(\x0b\x32).apio.common.proto.BoardProgrammerSection\x12/\n\x03usb\x18\x04 \x01(\x0b\x32\".apio.common.proto.BoardUsbSection\x12\x39\n\x08tinyprog\x18\x05 \x01(\x0b\x32\'.apio.common.proto.BoardTinyprogSection\"0\n\x0f\x46pgaIce40Params\x12\x0c\n\x04type\x18\x01 \x02(\t\x12\x0f\n\x07package\x18\x02 \x02(\t\">\n\x0e\x46pgaEcp5Params\x12\x0c\n\x04type\x18\x01 \x02(\t\x12\x0f\n\x07package\x18\x02 \x02(\t\x12\r\n\x05speed\x18\x03 \x02(\t\"\x83\x01\n\x0f\x46pgaGowinParams\x12\"\n\x0cyosys_family\x18\x01 \x02(\tR\x0cyosys-family\x12&\n\x0enextpnr_family\x18\x02 \x02(\tR\x0enextpnr-family\x12$\n\rpacker_device\x18\x03 \x02(\tR\rpacker-device\"b\n\x10\x46pgaXilinxParams\x12\x0e\n\x06\x66\x61mily\x18\x01 \x02(\t\x12\x1e\n\nyosys_arch\x18\x02 \x02(\tR\nyosys-arch\x12\x0f\n\x07package\x18\x03 \x02(\t\x12\r\n\x05speed\x18\x04 \x02(\t\"\x9c\x03\n\x0e\x46pgaDefinition\x12\x1a\n\x08part_num\x18\x01 \x02(\tR\x08part-num\x12)\n\x04\x61rch\x18\x02 \x02(\x0e\x32\x1b.apio.common.proto.ApioArch\x12\x0c\n\x04size\x18\x03 \x02(\t\x12H\n\x0cice40_params\x18\n \x01(\x0b\x32\".apio.common.proto.FpgaIce40ParamsH\x00R\x0cice40-params\x12\x45\n\x0b\x65\x63p5_params\x18\x0b \x01(\x0b\x32!.apio.common.proto.FpgaEcp5ParamsH\x00R\x0b\x65\x63p5-params\x12H\n\x0cgowin_params\x18\x0c \x01(\x0b\x32\".apio.common.proto.FpgaGowinParamsH\x00R\x0cgowin-params\x12K\n\rxilinx_params\x18\r \x01(\x0b\x32#.apio.common.proto.FpgaXilinxParamsH\x00R\rxilinx-paramsB\r\n\x0b\x61rch_params\"5\n\x14ProgrammerDefinition\x12\x0f\n\x07\x63ommand\x18\x01 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x02(\t')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(apio/common/proto/apio-definitions.proto\x12\x11\x61pio.common.proto\x1a#apio/common/proto/apio-common.proto\"D\n\x16\x42oardProgrammerSection\x12\n\n\x02id\x18\x01 \x02(\t\x12\x1e\n\nextra_args\x18\x02 \x01(\tR\nextra-args\"Q\n\x0f\x42oardUsbSection\x12\x0b\n\x03vid\x18\x01 \x01(\t\x12\x0b\n\x03pid\x18\x02 \x01(\t\x12$\n\rproduct_regex\x18\x03 \x01(\tR\rproduct-regex\"6\n\x14\x42oardTinyprogSection\x12\x1e\n\nname_regex\x18\x01 \x02(\tR\nname-regex\"\xeb\x01\n\x0f\x42oardDefinition\x12\x13\n\x0b\x64\x65scription\x18\x01 \x02(\t\x12\x18\n\x07\x66pga_id\x18\x02 \x02(\tR\x07\x66pga-id\x12=\n\nprogrammer\x18\x03 \x02(\x0b\x32).apio.common.proto.BoardProgrammerSection\x12/\n\x03usb\x18\x04 \x01(\x0b\x32\".apio.common.proto.BoardUsbSection\x12\x39\n\x08tinyprog\x18\x05 \x01(\x0b\x32\'.apio.common.proto.BoardTinyprogSection\"0\n\x0f\x46pgaIce40Params\x12\x0c\n\x04type\x18\x01 \x02(\t\x12\x0f\n\x07package\x18\x02 \x02(\t\">\n\x0e\x46pgaEcp5Params\x12\x0c\n\x04type\x18\x01 \x02(\t\x12\x0f\n\x07package\x18\x02 \x02(\t\x12\r\n\x05speed\x18\x03 \x02(\t\"\x83\x01\n\x0f\x46pgaGowinParams\x12\"\n\x0cyosys_family\x18\x01 \x02(\tR\x0cyosys-family\x12&\n\x0enextpnr_family\x18\x02 \x02(\tR\x0enextpnr-family\x12$\n\rpacker_device\x18\x03 \x02(\tR\rpacker-device\"b\n\x10\x46pgaXilinxParams\x12\x0e\n\x06\x66\x61mily\x18\x01 \x02(\t\x12\x1e\n\nyosys_arch\x18\x02 \x02(\tR\nyosys-arch\x12\x0f\n\x07package\x18\x03 \x02(\t\x12\r\n\x05speed\x18\x04 \x02(\t\"\x9c\x03\n\x0e\x46pgaDefinition\x12\x1a\n\x08part_num\x18\x01 \x02(\tR\x08part-num\x12)\n\x04\x61rch\x18\x02 \x02(\x0e\x32\x1b.apio.common.proto.ApioArch\x12\x0c\n\x04size\x18\x03 \x02(\t\x12H\n\x0cice40_params\x18\n \x01(\x0b\x32\".apio.common.proto.FpgaIce40ParamsH\x00R\x0cice40-params\x12\x45\n\x0b\x65\x63p5_params\x18\x0b \x01(\x0b\x32!.apio.common.proto.FpgaEcp5ParamsH\x00R\x0b\x65\x63p5-params\x12H\n\x0cgowin_params\x18\x0c \x01(\x0b\x32\".apio.common.proto.FpgaGowinParamsH\x00R\x0cgowin-params\x12K\n\rxilinx_params\x18\r \x01(\x0b\x32#.apio.common.proto.FpgaXilinxParamsH\x00R\rxilinx-paramsB\r\n\x0b\x61rch_params\"5\n\x14ProgrammerDefinition\x12\x0f\n\x07\x63ommand\x18\x01 \x02(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x02(\t')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)

apio/common/proto/apio_testing_pb2.py

@@ -0,0 +1,40 @@
+# pylint: disable=all
+
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: apio/common/proto/apio-testing.proto
+# Protobuf Python Version: 6.31.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    31,
+    1,
+    '',
+    'apio/common/proto/apio-testing.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$apio/common/proto/apio-testing.proto\x12\x11\x61pio.common.proto\"B\n\x08MessageA\x12\x1a\n\x08\x66ield_a1\x18\x01 \x02(\tR\x08\x66ield-a1\x12\x1a\n\x08\x66ield_a2\x18\x02 \x01(\tR\x08\x66ield-a2\"|\n\x08MessageB\x12\x37\n\x08\x66ield_b1\x18\x01 \x02(\x0b\x32\x1b.apio.common.proto.MessageAR\x08\x66ield-b1\x12\x37\n\x08\x66ield_b2\x18\x02 \x01(\x0b\x32\x1b.apio.common.proto.MessageAR\x08\x66ield-b2')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'apio.common.proto.apio_testing_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_MESSAGEA']._serialized_start=59
+  _globals['_MESSAGEA']._serialized_end=125
+  _globals['_MESSAGEB']._serialized_start=127
+  _globals['_MESSAGEB']._serialized_end=251
+# @@protoc_insertion_point(module_scope)