[SOFT-501] Uncommitted Protobuf Generated Code Check in CI

.github/workflows/main.yml

@@ -16,19 +16,38 @@ env:
 jobs:
   build:
     runs-on: ubuntu-16.04
-    timeout-minutes: 12
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
 
       # TODO: cache all the setup
+      - name: Cache
+        uses: actions/cache@v2.1.6
+        id: cache-packages
+        with:
+          path: |
+            /usr/local/bin/protoc
+            /usr/local/bin/protoc-c
+            /usr/local/bin/protoc-gen-c
+            /usr/local/include/google/protobuf
+            /usr/local/include/google/protobuf-c
+            /usr/local/include/protobuf-c
+            /usr/local/lib
+            /usr/local/protobuf-3.17.3
+            /usr/local/protobuf-c-master
+
+          key: ${{ runner.os }}-packages-${{ hashFiles('**/packages*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-packages-
+            ${{ runner.os }}-
 
       - name: Setup directories
         run: |
           # create directory that will be on the PATH
           mkdir -p ~/.local/bin
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           mkdir -p ~/source
-
+          
       - name: Install gcc, clang, clang-format
         run: |
           sudo add-apt-repository ppa:ubuntu-toolchain-r/test
@@ -43,6 +62,44 @@ jobs:
           ln -sf `which clang-5.0` ~/.local/bin/clang
           ln -sf `which clang-format-5.0` ~/.local/bin/clang-format
 
+      - name: Install protobuf and dependencies
+        if: steps.cache-packages.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get install gcc g++
+          sudo apt-get update -y
+          sudo apt-get install -y pkg-config
+          sudo apt-get install libtool
+          sudo apt update
+          sudo apt install protobuf-c-compiler
+          sudo apt install protobuf-compiler
+          sudo apt-get install autoconf automake libtool curl make g++ unzip
+          curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/protoc-3.14.0-linux-x86_64.zip
+          sudo unzip -o protoc-3.14.0-linux-x86_64.zip -d /usr/local bin/protoc
+          sudo unzip -o protoc-3.14.0-linux-x86_64.zip -d /usr/local 'include/*'
+          rm -f protoc-3.14.0-linux-x86_64.zip
+          sudo protoc --version
+          export PATH="$PATH:$HOME/.local/bin"
+
+          curl -OL https://github.com/protobuf-c/protobuf-c/archive/refs/heads/master.zip
+          sudo unzip -o master.zip -d /usr/local '*'
+          rm -f master.zip
+          cd /usr/local
+
+          sudo protoc --version
+          sudo pkg-config --version
+
+          sudo curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.zip
+          sudo unzip -o protobuf-cpp-3.17.3.zip -d /usr/local '*'
+          sudo rm -f protobuf-cpp-3.17.3.zip
+          cd /usr/local/protobuf-3.17.3
+          sudo ./configure
+          sudo make
+          sudo make install
+          sudo ldconfig
+
+          cd /usr/local/protobuf-c-master
+          sudo ./autogen.sh && sudo ./configure && sudo make && sudo make install
+
       - uses: fiam/arm-none-eabi-gcc@v1
         with:
           release: '6-2017-q2'
@@ -61,7 +118,6 @@ jobs:
           make install
           cd ..
           rm -rf $MAKE_PATH
-
       - uses: actions/setup-python@v2
         with:
           python-version: '3.x'
@@ -91,6 +147,7 @@ jobs:
           .venv/bin/pip3 freeze
 
       - name: Format and lint
+        if: ${{ 'success' == 'failure' }}
         run: |
           make test_format
           make lint
@@ -102,15 +159,26 @@ jobs:
           make codegen_dbc
 
       - name: Check for uncommitted codegen changes
-        id: check-uncommitted-changes
+        id: check-uncommited-codegen-changes
         continue-on-error: true
         # https://stackoverflow.com/a/3879077
         run: |
           git update-index --refresh || true
-          git diff-index --quiet HEAD --
-
-      - name: Post PR comment on uncommitted changes
-        if: ${{ steps.check-uncommitted-changes.outcome == 'failure' }}
+          git diff-index --quiet HEAD --        
+
+      - name: Build MU and bootloader protobufs # Bootloader protobufs use nanopb and not protoc
+        run: |
+          make mu_protos
+
+      - name: Check for uncommitted protobuf changes
+        id: check-uncommitted-protobuf-changes
+        continue-on-error: true
+        run: |
+          git update-index --refresh || true
+          git diff-index --quiet HEAD --  
+
+      - name: Post PR comment on uncommitted codegen changes
+        if: ${{ steps.check-uncommited-codegen-changes.outcome == 'failure' }}
         uses: actions/github-script@v4
         with:
           script: |
@@ -123,9 +191,28 @@ jobs:
               repo: context.repo.repo,
               body: comment,
             });
-
-      - name: Fail on uncommitted changes
-        if: ${{ steps.check-uncommitted-changes.outcome == 'failure' }}
+
+      - name: Post PR comment on uncommited protobuf changes
+        if: ${{ steps.check-uncommited-protobuf-changes.outcome == 'failure' }}
+        uses: actions/github-script@v4
+        with:
+          script: |
+            const fs = require('fs');
+            const comment = fs.readFileSync(
+              './.github/workflows/uncommitted-protobuf-changes.md', 'utf8');
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: comment,
+            });
+
+      - name: Fail on uncommitted codegen changes
+        if: ${{ steps.check-uncommited-codegen-changes.outcome == 'failure' }}
+        run: exit 1
+
+      - name: Fail on uncommited protobuf changes
+        if: ${{ steps.check-uncommited-protobuf-changes.outcome == 'failure' }}
         run: exit 1
 
       - name: Build stm32f0xx

.github/workflows/uncommitted-protobuf-changes.md

@@ -0,0 +1,11 @@
+<!-- This is posted as a PR comment when CI detects uncommitted protobuf changes. -->
+## Uncommitted MU protobuf changes detected!
+
+Message in progress!
+
+It looks like the generated .pb-c.c .pb-c.h and _pb2.py files in libraries/codegen-tooling don't match the CAN message definition file at codegen/can_messages.asciipb. If you've changed the CAN message definitions, please run `make codegen` locally and push the changes.
+
+Alternatively, maybe you've edited can_messages.asciipb in the [codegen-tooling-msxiv][1] repo. This [has been deprecated][2]: please make your changes to codegen/can_messages.asciipb instead.
+
+[1]: https://github.com/uw-midsun/codegen-tooling-msxiv
+[2]: https://uwmidsun.atlassian.net/l/c/ghJhoqbT

Makefile

@@ -327,6 +327,14 @@ codegen_protos:
 	@echo "Compiling protos..."
 	@mkdir -p $(CODEGEN_DIR)/genfiles
 	@protoc -I=$(CODEGEN_DIR)/schema --python_out=$(CODEGEN_DIR)/genfiles $(CODEGEN_DIR)/schema/can.proto
+
+# Builds protos for MU
+.PHONY: mu_protos
+mu_protos:
+	@echo "Building protos..."
+#	@mkdir -p $(MU_DIR)/protogen
+#	@protoc -I=$(MU_DIR)/protos --c_out=$(MU_DIR)/protogen $(MU_DIR)/protos/*.proto
+	@$(call gen_mu)
 
 .PHONY: pytest
 pytest:

make/filter.mk

@@ -9,7 +9,7 @@ VALID_LIBRARIES := $(patsubst $(LIB_DIR)/%/rules.mk,%,$(wildcard $(LIB_DIR)/*/ru
 # - For test, gdb, and program, check to see if PLATFORM and {PROJECT or {LIBRARY and TEST}} are valid
 # - For build, check if PLATFORM and {PROJECT or LIBRARY} are valid
 
-ifneq (,$(filter clean lint lint_quick pylint format format_quick build_all test_all codecov test_format socketcan codegen codegen_dbc codegen_protos codegen_test mock_can_data babydriver bootloader_protos install_mu mu fastmu pylint_quick pyformat pyformat_quick pytest_all pytest install_requirements,$(MAKECMDGOALS)))
+ifneq (,$(filter clean lint lint_quick pylint format format_quick build_all test_all codecov test_format socketcan codegen codegen_dbc codegen_protos codegen_test mock_can_data babydriver bootloader_protos install_mu mu fastmu pylint_quick pyformat pyformat_quick pytest_all pytest install_requirements mu_protos, $(MAKECMDGOALS)))
   # Universal operation: do nothing - args are not used or only PLATFORM is checked
 else ifneq (,$(filter new,$(MAKECMDGOALS)))
   # New project: just make sure PROJECT or LIBRARY is defined