kas-build-ci.yml

name: KAS Build CI

on:
  push:
    branches: [ main, develop ]
    paths:
      - 'recipes-**'
      - 'conf/**'
      - 'kas/**'
      - 'scripts/**'
      - 'classes/**'
      - 'meta-dynamicdevices-bsp/**'
      - 'meta-dynamicdevices-distro/**'
      - 'meta-lmp-base/**'
      - 'meta-dynamicdevices-bsp'
      - 'meta-dynamicdevices-distro'
      - 'meta-lmp-base'
      - '.github/workflows/kas-build-ci.yml'
  pull_request:
    branches: [ main, develop ]
    paths:
      - 'recipes-**'
      - 'conf/**'
      - 'kas/**'
      - 'scripts/**'
      - 'classes/**'
      - 'meta-dynamicdevices-bsp/**'
      - 'meta-dynamicdevices-distro/**'
      - 'meta-lmp-base/**'
      - 'meta-dynamicdevices-bsp'
      - 'meta-dynamicdevices-distro'
      - 'meta-lmp-base'
      - '.github/workflows/kas-build-ci.yml'
  workflow_dispatch:
    inputs:
      machine:
        description: 'Specific machine to build (optional)'
        required: false
        type: choice
        options:
          - 'all'
          - 'imx8mm-jaguar-sentai'
          - 'imx93-jaguar-eink'
        default: 'all'

jobs:
  # Layer validation job - runs once before builds
  validate:
    name: Validate Yocto Layer Compliance
    runs-on: self-hosted
    container:
      image: dynamicdevices/yocto-ci-build:latest
      options: --privileged --platform linux/amd64  -v /dev/net/tun:/dev/net/tun -v /dev/kvm:/dev/kvm
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Cache KAS layers
        uses: actions/cache@v4
        with:
          path: |
            build/layers
            build/cache
          key: kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}-${{ github.sha }}
          restore-keys: |
            kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}-
            kas-layers-

      - name: Cache Yocto downloads
        uses: actions/cache@v4
        with:
          path: ~/yocto/downloads
          key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}
          restore-keys: |
            yocto-downloads-${{ runner.os }}-

      - name: Validate Yocto Layers
        timeout-minutes: 10
        run: |
          echo "🏅 Validating Yocto layers using yocto-check-layer"
          echo "📋 Testing layers: meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro"
          
          # Initialize Yocto build environment using kas to get the right layers and setup
          echo "🔧 Setting up Yocto environment with kas (with timeout)..."
          timeout 300 kas checkout kas/lmp-dynamicdevices-base.yml || {
            echo "⚠️ kas checkout timed out or failed, setting up minimal environment..."
            
            # Create minimal directory structure
            mkdir -p build/layers/openembedded-core
            mkdir -p build/layers/meta-lmp/meta-lmp-base
            mkdir -p build/layers/meta-openembedded/meta-oe
            
            # Clone only essential repositories for layer validation
            echo "📦 Cloning essential repositories for validation..."
            git clone --depth 1 https://github.com/lmp-mirrors/openembedded-core build/layers/openembedded-core || echo "Failed to clone openembedded-core"
            git clone --depth 1 https://github.com/foundriesio/meta-lmp build/layers/meta-lmp || echo "Failed to clone meta-lmp"
            git clone --depth 1 https://github.com/lmp-mirrors/meta-openembedded build/layers/meta-openembedded || echo "Failed to clone meta-openembedded"
          }
          
          # Create a clean BitBake environment for validation only
          echo "🔧 Setting up clean BitBake environment for validation..."
          
          # Create a separate validation build directory
          mkdir -p validation-build/conf
          cd validation-build
          
          # Create minimal bblayers.conf with all required layers for meta-lmp-base validation
          echo 'LCONF_VERSION = "7"' > conf/bblayers.conf
          echo 'BBPATH = "${TOPDIR}"' >> conf/bblayers.conf
          echo 'BBFILES ?= ""' >> conf/bblayers.conf
          echo 'BBLAYERS ?= " \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/openembedded-core/meta \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-openembedded/meta-oe \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-openembedded/meta-filesystems \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-openembedded/meta-python \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-openembedded/meta-networking \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-openembedded/meta-perl \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-security \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-updater \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-virtualisation \' >> conf/bblayers.conf
          echo '  ${TOPDIR}/../build/layers/meta-clang \' >> conf/bblayers.conf
          echo '"' >> conf/bblayers.conf
          
          # Create minimal local.conf with proper distro
          echo 'MACHINE = "qemux86-64"' > conf/local.conf
          echo 'DISTRO = "nodistro"' >> conf/local.conf
          echo 'DISTRO_FEATURES = "security virtualization seccomp x11 wayland"' >> conf/local.conf
          echo 'PREFERRED_RPROVIDER_ssh = "openssh"' >> conf/local.conf
          
          # Source the oe-init-build-env to set up BitBake environment
          if [ -f "../build/layers/openembedded-core/oe-init-build-env" ]; then
            source ../build/layers/openembedded-core/oe-init-build-env .
            echo "✅ Clean BitBake environment initialized for validation"
          else
            echo "❌ oe-init-build-env not found"
            exit 1
          fi
          
          # Find yocto-check-layer script in the checked out layers
          YOCTO_CHECK_LAYER=$(find ../build/layers -name 'yocto-check-layer' -type f | head -1)
          if [ -z "$YOCTO_CHECK_LAYER" ]; then
            echo '❌ yocto-check-layer script not found'
            exit 1
          fi
          
          echo "✅ Found yocto-check-layer: $YOCTO_CHECK_LAYER"
          
          # Verify BitBake is available
          if ! python3 -c "import bb.tinfoil" 2>/dev/null; then
            echo "❌ BitBake environment not properly initialized"
            echo "PYTHONPATH: $PYTHONPATH"
            echo "PATH: $PATH"
            exit 1
          fi
          
          echo "✅ BitBake environment verified"
          
          # Remove BitBake test data to avoid collection conflicts
          echo "🔧 Removing BitBake test data to avoid collection conflicts..."
          
          # Debug: Show BitBake directory structure
          echo "🔧 Debugging BitBake directory structure..."
          find ../build/layers -name "bitbake" -type d 2>/dev/null | head -5
          find ../build/layers -path "*/layerindexlib/tests" -type d 2>/dev/null | head -5
          
          # Check multiple possible paths for BitBake test data
          testdata_found=false
          for testdata_path in \
            "../build/layers/bitbake/lib/layerindexlib/tests/testdata" \
            "../build/layers/openembedded-core/bitbake/lib/layerindexlib/tests/testdata" \
            "$(find ../build/layers -path "*/bitbake/lib/layerindexlib/tests/testdata" -type d 2>/dev/null | head -1)" \
            "$(find ../build/layers -path "*/layerindexlib/tests/testdata" -type d 2>/dev/null | head -1)"; do
            
            if [ -n "$testdata_path" ] && [ -d "$testdata_path" ]; then
              echo "🔧 Found BitBake test data at: $testdata_path"
              rm -rf "$testdata_path"
              echo "✅ Removed BitBake test data from $testdata_path"
              testdata_found=true
              break
            fi
          done
          
          if [ "$testdata_found" = false ]; then
            echo "⚠️ No BitBake test data found - this might be okay if it doesn't exist"
          fi
          
          # Create missing bbclass files that meta-lmp-base recipes depend on
          echo "🔧 Creating missing bbclass files for validation..."
          mkdir -p ../build/layers/meta-lmp/meta-lmp-base/classes
          if [ ! -f "../build/layers/meta-lmp/meta-lmp-base/classes/uuu_bootloader_tag.bbclass" ]; then
            echo "# Minimal uuu_bootloader_tag.bbclass for validation" > ../build/layers/meta-lmp/meta-lmp-base/classes/uuu_bootloader_tag.bbclass
            echo "# This is a stub for layer validation purposes only" >> ../build/layers/meta-lmp/meta-lmp-base/classes/uuu_bootloader_tag.bbclass
            echo 'UBOOT_BOOTLOADER_TAG ?= ""' >> ../build/layers/meta-lmp/meta-lmp-base/classes/uuu_bootloader_tag.bbclass
            echo "✅ Created minimal uuu_bootloader_tag.bbclass"
          fi
          
          # Check what layers kas checkout actually fetched
          echo "🔧 Checking layers fetched by kas checkout..."
          find ../build/layers -name "layer.conf" -type f | head -10
          
          # Skip meta-lmp-base check since we're not including it to avoid missing recipe errors
          echo "🔧 Skipping meta-lmp-base check to avoid missing recipe dependencies..."
          
          # Run yocto-check-layer for each layer individually in dependency order
          echo '🔧 Validating layers individually in dependency order:'
          
          # Strict validation approach - fail if dependencies can't be resolved
          # Build minimal but sufficient layer list to resolve dependencies
          
          # Essential layers for dependency resolution - minimal set to avoid conflicts
          CORE_LAYERS="../build/layers/openembedded-core/meta"
          # Include all required OE layers for meta-lmp-base dependencies
          OE_ESSENTIAL="../build/layers/meta-openembedded/meta-oe"
          OE_PYTHON="../build/layers/meta-openembedded/meta-python"
          OE_NETWORKING="../build/layers/meta-openembedded/meta-networking"
          OE_FILESYSTEMS="../build/layers/meta-openembedded/meta-filesystems"
          OE_PERL="../build/layers/meta-openembedded/meta-perl"
          # Include security and virtualization layers
          META_SECURITY="../build/layers/meta-security"
          META_UPDATER="../build/layers/meta-updater"
          META_VIRTUALIZATION="../build/layers/meta-virtualisation"
          META_CLANG="../build/layers/meta-clang"
          # Include LMP layers
          LMP_BASE="../build/layers/meta-lmp/meta-lmp-base"
          LMP_BSP="../build/layers/meta-lmp/meta-lmp-bsp"
          
          # Complete dependency set for meta-lmp-base validation
          # Exclude meta-lmp-base and meta-lmp-bsp to avoid missing recipe errors
          ALL_DEPS="$CORE_LAYERS $OE_ESSENTIAL $OE_PYTHON $OE_NETWORKING $OE_FILESYSTEMS $OE_PERL $META_SECURITY $META_UPDATER $META_VIRTUALIZATION $META_CLANG"
          
          # Check if essential dependency layers exist
          echo "🔍 Checking essential dependency layers..."
          echo "Current working directory: $(pwd)"
          echo "Contents of ../build/layers:"
          ls -la ../build/layers/ || echo "build/layers directory not accessible"
          echo ""
          
          missing_deps=""
          # Check core essential layers only (don't check every single dependency)
          essential_layers="$CORE_LAYERS $OE_ESSENTIAL $META_SECURITY $META_UPDATER $META_VIRTUALIZATION"
          for layer in $essential_layers; do
            layer_name=$(basename "$layer")
            echo "Checking: $layer"
            if [ -d "$layer" ] && [ -f "$layer/conf/layer.conf" ]; then
              echo "✅ $layer_name found at $layer"
            else
              echo "❌ $layer_name missing at $layer"
              echo "  Directory exists: $([ -d \"$layer\" ] && echo \"Yes\" || echo \"No\")"
              echo "  layer.conf exists: $([ -f \"$layer/conf/layer.conf\" ] && echo \"Yes\" || echo \"No\")"
              missing_deps="$missing_deps $layer_name"
            fi
          done
          
          if [ -n "$missing_deps" ]; then
            echo "❌ Critical dependency layers missing:$missing_deps"
            echo "Cannot perform proper layer validation without essential dependencies"
            exit 1
          fi
          
          # Show what layers we're including to debug collection conflicts
          echo "🔍 Dependency layers being used:"
          for layer in $ALL_DEPS; do
            if [ -f "$layer/conf/layer.conf" ]; then
              collection=$(grep "BBFILE_COLLECTIONS" "$layer/conf/layer.conf" | head -1 | sed 's/.*"\(.*\)".*/\1/' || echo "unknown")
              echo "  - $(basename $layer): $collection"
            fi
          done
          echo ""
          echo "🔧 Note: Excluding meta-lmp layers to avoid missing recipe dependencies and collection conflicts"
          echo ""
          
          # Validate with minimal dependencies - STRICT MODE
          echo '1️⃣ Validating meta-dynamicdevices-bsp (with minimal dependencies)...'
          YOCTO_CMD_BSP="python3 $YOCTO_CHECK_LAYER $ALL_DEPS ../meta-dynamicdevices-bsp"
          echo "Command: $YOCTO_CMD_BSP"
          if eval "$YOCTO_CMD_BSP"; then
            echo '✅ meta-dynamicdevices-bsp validation PASSED'
          else
            echo '❌ meta-dynamicdevices-bsp validation FAILED'
            echo 'This is a blocking error - layer has unresolved dependencies or configuration issues'
            exit 1
          fi
          
          echo ""
          
          # Validate distro layer with dependencies
          echo '2️⃣ Validating meta-dynamicdevices-distro (with all dependencies)...'
          YOCTO_CMD_DISTRO="python3 $YOCTO_CHECK_LAYER $ALL_DEPS ../meta-dynamicdevices-bsp ../meta-dynamicdevices-distro"
          echo "Command: $YOCTO_CMD_DISTRO"
          if eval "$YOCTO_CMD_DISTRO"; then
            echo '✅ meta-dynamicdevices-distro validation PASSED'
          else
            echo '❌ meta-dynamicdevices-distro validation FAILED'
            echo 'This is a blocking error - layer has unresolved dependencies or configuration issues'
            exit 1
          fi
          
          echo ""
          
          # Validate main layer with all dependencies
          echo '3️⃣ Validating meta-dynamicdevices (main, with all dependencies)...'
          YOCTO_CMD_MAIN="python3 $YOCTO_CHECK_LAYER $ALL_DEPS ../meta-dynamicdevices-bsp ../meta-dynamicdevices-distro .."
          echo "Command: $YOCTO_CMD_MAIN"
          if eval "$YOCTO_CMD_MAIN"; then
            echo '✅ meta-dynamicdevices validation PASSED'
          else
            echo '❌ meta-dynamicdevices validation FAILED'
            echo 'This is a blocking error - layer has unresolved dependencies or configuration issues'
            exit 1
          fi
          
          echo ""
          echo '4️⃣ Testing layer configuration parsing (basic validation)...'
          # At minimum, verify our layer.conf files are parseable
          for layer in "../meta-dynamicdevices-bsp" "../meta-dynamicdevices-distro" ".."; do
            layer_name=$(basename "$layer")
            if [ "$layer" = ".." ]; then layer_name="meta-dynamicdevices"; fi
            
            echo "Checking $layer_name layer.conf..."
            if [ -f "$layer/conf/layer.conf" ]; then
              # Basic syntax check - look for required variables
              if grep -q "BBFILE_COLLECTIONS" "$layer/conf/layer.conf" && grep -q "LAYER_CONF_VERSION" "$layer/conf/layer.conf"; then
                echo "✅ $layer_name layer.conf is valid"
              else
                echo "❌ $layer_name layer.conf missing required variables"
              fi
            else
              echo "❌ $layer_name layer.conf not found"
            fi
          done
          
          echo "✅ Layer validation completed successfully"

  # Build job with matrix for specific machines and targets
  build:
    name: Build ${{ matrix.machine }} - ${{ matrix.target }}
    runs-on: self-hosted
    container:
      image: dynamicdevices/yocto-ci-build:latest
      options: --privileged --platform linux/amd64  -v /dev/net/tun:/dev/net/tun -v /dev/kvm:/dev/kvm
    needs: [validate]
    strategy:
      fail-fast: false
      matrix:
        machine: [imx8mm-jaguar-sentai, imx93-jaguar-eink]
        target: [lmp-factory-image, mfgtool-files]
    
    env:
      KAS_MACHINE: ${{ matrix.machine }}
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Cache KAS layers
        uses: actions/cache@v4
        with:
          path: |
            build/layers
            build/cache
          key: kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml', 'kas/lmp-dynamicdevices-mfgtool.yml') }}-${{ github.sha }}
          restore-keys: |
            kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml', 'kas/lmp-dynamicdevices-mfgtool.yml') }}-
            kas-layers-

      - name: Cache Yocto downloads
        uses: actions/cache@v4
        with:
          path: ~/yocto/downloads
          key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml', 'kas/lmp-dynamicdevices-mfgtool.yml') }}
          restore-keys: |
            yocto-downloads-${{ runner.os }}-

      - name: Cache Yocto sstate
        uses: actions/cache@v4
        with:
          path: ~/yocto/sstate
          key: yocto-sstate-${{ runner.os }}-${{ matrix.machine }}-${{ matrix.target }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml', 'kas/lmp-dynamicdevices-mfgtool.yml') }}-${{ hashFiles('recipes-**') }}
          restore-keys: |
            yocto-sstate-${{ runner.os }}-${{ matrix.machine }}-${{ matrix.target }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml', 'kas/lmp-dynamicdevices-mfgtool.yml') }}-
            yocto-sstate-${{ runner.os }}-${{ matrix.machine }}-${{ matrix.target }}-
            yocto-sstate-${{ runner.os }}-${{ matrix.machine }}-
            yocto-sstate-${{ runner.os }}-

      - name: Cache build tmp directory
        uses: actions/cache@v4
        with:
          path: |
            ~/yocto/persistent
            build/tmp/cache
          key: yocto-build-cache-${{ runner.os }}-${{ matrix.machine }}-${{ matrix.target }}-${{ hashFiles('recipes-**') }}
          restore-keys: |
            yocto-build-cache-${{ runner.os }}-${{ matrix.machine }}-${{ matrix.target }}-
            yocto-build-cache-${{ runner.os }}-${{ matrix.machine }}-
            yocto-build-cache-${{ runner.os }}-

      - name: Setup build cache
        run: |
          echo "🗄️ Setting up persistent build cache"
          mkdir -p ~/yocto/{downloads,sstate,persistent}
          chmod 755 ~/yocto/{downloads,sstate,persistent}

      - name: Build with KAS
        timeout-minutes: 360
        run: |
          echo "🔨 Starting KAS build: ${{ matrix.machine }} (${{ matrix.target }})"
          
          # Select appropriate KAS configuration and build target
          if [ "${{ matrix.target }}" = "mfgtool-files" ]; then
            kas_config="kas/lmp-dynamicdevices-mfgtool.yml"
            build_target="mfgtool-files"
          else
            kas_config="kas/lmp-dynamicdevices-base.yml" 
            build_target="lmp-factory-image"
          fi
          
          echo "Using KAS config: $kas_config"
          echo "Build target: $build_target"
          echo "Machine: $KAS_MACHINE"
          
          # Run KAS build (directly in container)
          kas build "$kas_config:$build_target"
          
          echo "✅ Build completed successfully"

      - name: Collect build artifacts
        if: always()
        run: |
          echo "📦 Collecting build artifacts for ${{ matrix.machine }} (${{ matrix.target }})"
          
          # Create artifacts directory
          mkdir -p artifacts/${{ matrix.machine }}/${{ matrix.target }}
          
          # Find the deploy directory
          deploy_dir=$(find . -path "*/tmp*/deploy/images/${{ matrix.machine }}" -type d | head -1)
          
          if [ -d "$deploy_dir" ]; then
            echo "✅ Found deploy directory: $deploy_dir"
            
            # Copy key artifacts based on target
            if [ "${{ matrix.target }}" = "lmp-factory-image" ]; then
              # Factory image artifacts
              for pattern in "*.wic.gz" "*.wic.bmap" "imx-boot-*" "u-boot-*.itb"; do
                find "$deploy_dir" -name "$pattern" -exec cp {} artifacts/${{ matrix.machine }}/${{ matrix.target }}/ \; 2>/dev/null || true
              done
            else
              # MFGtool artifacts  
              for pattern in "imx-boot-mfgtool*" "u-boot-mfgtool*"; do
                find "$deploy_dir" -name "$pattern" -exec cp {} artifacts/${{ matrix.machine }}/${{ matrix.target }}/ \; 2>/dev/null || true
              done
            fi
            
            echo "📋 Artifacts collected:"
            ls -la artifacts/${{ matrix.machine }}/${{ matrix.target }}/
          else
            echo "❌ Deploy directory not found"
            echo "# Build Failed" > artifacts/${{ matrix.machine }}/${{ matrix.target }}/build-failed.txt
          fi

      - name: Upload build artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: build-artifacts-${{ matrix.machine }}-${{ matrix.target }}
          path: artifacts/
          retention-days: 30
          if-no-files-found: warn

  # Summary job
  summary:
    name: Build Summary
    runs-on: self-hosted
    needs: [validate, build]
    if: always()
    
    steps:
      - name: Generate build summary
        run: |
          echo "# KAS Build CI Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Workflow:** ${{ github.workflow }}" >> $GITHUB_STEP_SUMMARY
          echo "**Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
          echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY
          echo "**Branch:** ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Determine overall status
          if [ "${{ needs.validate.result }}" = "success" ] && [ "${{ needs.build.result }}" = "success" ]; then
            echo "## ✅ Build Status: SUCCESS" >> $GITHUB_STEP_SUMMARY
            echo "All layer validations and builds completed successfully!" >> $GITHUB_STEP_SUMMARY
          else
            echo "## ❌ Build Status: FAILED" >> $GITHUB_STEP_SUMMARY
            echo "- Layer validation: ${{ needs.validate.result }}" >> $GITHUB_STEP_SUMMARY
            echo "- Build: ${{ needs.build.result }}" >> $GITHUB_STEP_SUMMARY
          fi
          
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "## Build Matrix" >> $GITHUB_STEP_SUMMARY
          echo "- **Machines:** imx8mm-jaguar-sentai, imx93-jaguar-eink" >> $GITHUB_STEP_SUMMARY
          echo "- **Targets:** lmp-factory-image, mfgtool-files" >> $GITHUB_STEP_SUMMARY
          echo "- **Total builds:** 4 (2 machines × 2 targets)" >> $GITHUB_STEP_SUMMARY