Advanced CI/CD Pipeline #37

Workflow file for this run

.github/workflows/ci-cd-pipeline.yml at 80cb5c3

	name: Advanced CI/CD Pipeline

	on:
	push:
	branches: [ main, develop, feature/* ]
	pull_request:
	branches: [ main, develop ]
	schedule:
	# Daily at 2 AM UTC
	- cron: '0 2 * * *'
	workflow_dispatch:
	inputs:
	run_security_scan:
	description: 'Run security scanning'
	required: false
	default: 'true'
	type: boolean
	run_performance_tests:
	description: 'Run performance tests'
	required: false
	default: 'true'
	type: boolean
	deploy_to_staging:
	description: 'Deploy to staging environment'
	required: false
	default: 'false'
	type: boolean

	env:
	NODE_VERSION: '18'
	PYTHON_VERSION: '3.9'
	GO_VERSION: '1.21'

	jobs:
	# Code Quality and Static Analysis
	code-quality:
	runs-on: ubuntu-latest
	outputs:
	quality-score: ${{ steps.quality.outputs.score }}
	coverage: ${{ steps.coverage.outputs.percentage }}

	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Setup Node.js
	uses: actions/setup-node@v4
	with:
	node-version: ${{ env.NODE_VERSION }}
	cache: 'npm'

	- name: Install Python dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install -r requirements-dev.txt

	- name: Install Node.js dependencies
	run: \|
	cd frontend && npm ci

	- name: Run Python linting
	run: \|
	pip install flake8 black isort mypy
	flake8 ml-model-api/ --count --select=E9,F63,F7,F82 --show-source --statistics
	black --check ml-model-api/
	isort --check-only ml-model-api/
	mypy ml-model-api/ --ignore-missing-imports

	- name: Run JavaScript linting
	run: \|
	cd frontend && npm run lint

	- name: Run security scanning
	id: security
	run: \|
	pip install safety bandit semgrep trufflehog gitleaks
	echo "Running security scans..."

	# Dependency scanning
	safety check --json --output dependency-security-reports/python-dependencies.json \|\| true

	# Code security analysis
	bandit -r ml-model-api/ -f json -o code-security-reports/bandit-report.json \|\| true
	semgrep --config=auto --json --output=code-security-reports/semgrep-report.json ml-model-api/ \|\| true

	# Secret scanning
	trufflehog filesystem . --json --output=secrets-security-reports/trufflehog-report.json \|\| true
	gitleaks detect --json --output=secrets-security-reports/gitleaks-report.json \|\| true

	# Generate summary
	python scripts/security/generate_summary.py

	- name: Run unit tests with coverage
	id: coverage
	run: \|
	pip install pytest pytest-cov pytest-mock coverage
	pytest tests/unit/ --cov=ml_model_api --cov-report=json --cov-report=html --cov-report=xml --junitxml=reports/unit-tests.xml
	COVERAGE=$(python -c "import json; print(json.load(open('coverage.json')['totals']['percent_covered']))")
	echo "percentage=$COVERAGE" >> $GITHUB_OUTPUT

	- name: Quality gate evaluation
	id: quality
	run: \|
	python -c "
	import json
	coverage = ${{ steps.coverage.outputs.percentage }}
	quality_score = min(100, coverage + 10) # Simple quality score calculation
	print(f'score={quality_score}')
	print(f'score={quality_score}') >> $GITHUB_OUTPUT
	"

	- name: Upload coverage reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: coverage-reports
	path: \|
	coverage.xml
	htmlcov/
	coverage.json
	retention-days: 30

	- name: Upload security reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: security-reports
	path: \|
	dependency-security-reports/
	code-security-reports/
	secrets-security-reports/
	retention-days: 30

	# Integration Tests
	integration-tests:
	runs-on: ubuntu-latest
	needs: code-quality
	services:
	postgres:
	image: postgres:15
	env:
	POSTGRES_PASSWORD: postgres
	POSTGRES_DB: flavorsnap_test
	options: >-
	--health-cmd pg_isready
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 5432:5432

	redis:
	image: redis:7-alpine
	options: >-
	--health-cmd "redis-cli ping"
	--health-interval 10s
	--health-timeout 5s
	--health-retries 5
	ports:
	- 6379:6379

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install -r requirements-test.txt

	- name: Wait for services
	run: \|
	timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done'
	timeout 60 bash -c 'until nc -z localhost 6379; do sleep 1; done'

	- name: Run database migrations
	run: \|
	python manage.py migrate
	env:
	DATABASE_URL: postgresql://postgres:postgres@localhost:5432/flavorsnap_test
	REDIS_URL: redis://localhost:6379

	- name: Run integration tests
	run: \|
	pytest tests/integration/ --junitxml=reports/integration-tests.xml -v
	env:
	DATABASE_URL: postgresql://postgres:postgres@localhost:5432/flavorsnap_test
	REDIS_URL: redis://localhost:6379

	- name: Upload integration test reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: integration-test-reports
	path: reports/integration-tests.xml
	retention-days: 30

	# Performance Tests
	performance-tests:
	runs-on: ubuntu-latest
	needs: code-quality
	if: github.event.inputs.run_performance_tests == 'true' \|\| github.event_name == 'schedule'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install locust pytest-benchmark

	- name: Start application
	run: \|
	python manage.py runserver &
	sleep 10

	- name: Run performance benchmarks
	run: \|
	pytest tests/performance/ --benchmark-json=performance-results.json --benchmark-only

	- name: Run load tests
	run: \|
	locust --headless --users 100 --spawn-rate 10 --run-time 60s --host http://localhost:8000 tests/performance/locustfile.py

	- name: Upload performance reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: performance-reports
	path: \|
	performance-results.json
	locust_report.html
	retention-days: 30

	# API Tests
	api-tests:
	runs-on: ubuntu-latest
	needs: integration-tests

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt
	pip install pytest-httpx

	- name: Start application
	run: \|
	python manage.py runserver &
	sleep 10

	- name: Run API tests
	run: \|
	pytest tests/api/ --junitxml=reports/api-tests.xml -v

	- name: Upload API test reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: api-test-reports
	path: reports/api-tests.xml
	retention-days: 30

	# Security Scanning (Enhanced)
	security-scan:
	runs-on: ubuntu-latest
	needs: code-quality
	if: github.event.inputs.run_security_scan == 'true' \|\| github.event_name == 'schedule'

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Run comprehensive security scan
	run: \|
	python scripts/security/vulnerability_scanner.py

	- name: Generate remediation scripts
	run: \|
	python scripts/security/generate_remediation.py

	- name: Check for critical vulnerabilities
	run: \|
	python scripts/security/check_critical_vulns.py --fail-threshold high

	- name: Upload security scan results
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: enhanced-security-reports
	path: \|
	security-reports/
	remediation-scripts/
	retention-days: 30

	# Container Security
	container-security:
	runs-on: ubuntu-latest
	needs: code-quality

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build Docker image
	uses: docker/build-push-action@v5
	with:
	context: .
	push: false
	tags: flavorsnap:test
	cache-from: type=gha
	cache-to: type=gha,mode=max

	- name: Run Docker security scan
	run: \|
	# Install security tools
	sudo apt-get update
	sudo apt-get install -y docker.io trivy hadolint

	# Dockerfile linting
	hadolint Dockerfile > container-security-reports/hadolint-report.json \|\| true

	# Container image scanning
	trivy image --format json --output container-security-reports/trivy-report.json flavorsnap:test \|\| true

	# Generate summary
	python scripts/security/check_headers.py --output container-security-reports/headers-report.json

	- name: Upload container security reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: container-security-reports
	path: container-security-reports/
	retention-days: 30

	# Quality Gates
	quality-gates:
	runs-on: ubuntu-latest
	needs: [code-quality, integration-tests, api-tests]
	outputs:
	gates-passed: ${{ steps.gates.outputs.passed }}
	overall-score: ${{ steps.gates.outputs.score }}

	steps:
	- name: Download all test reports
	uses: actions/download-artifact@v3
	with:
	path: all-reports/

	- name: Evaluate quality gates
	id: gates
	run: \|
	python ml-model-api/quality_gates.py

	- name: Generate comprehensive report
	run: \|
	python ml-model-api/reporting.py

	- name: Upload quality reports
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: quality-reports
	path: reports/
	retention-days: 30

	# Build and Package
	build:
	runs-on: ubuntu-latest
	needs: quality-gates
	if: needs.quality-gates.outputs.gates-passed == 'true'

	strategy:
	matrix:
	component: [api, frontend, worker]

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	if: matrix.component == 'api' \|\| matrix.component == 'worker'
	uses: actions/setup-python@v4
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: 'pip'

	- name: Setup Node.js
	if: matrix.component == 'frontend'
	uses: actions/setup-node@v4
	with:
	node-version: ${{ env.NODE_VERSION }}
	cache: 'npm'

	- name: Build API
	if: matrix.component == 'api'
	run: \|
	pip install -r requirements.txt
	pip install -r requirements-build.txt
	python setup.py sdist bdist_wheel

	- name: Build Frontend
	if: matrix.component == 'frontend'
	run: \|
	cd frontend
	npm ci
	npm run build
	npm run test:unit

	- name: Build Worker
	if: matrix.component == 'worker'
	run: \|
	pip install -r requirements.txt
	pip install -r requirements-build.txt
	python setup.py sdist bdist_wheel

	- name: Upload build artifacts
	uses: actions/upload-artifact@v3
	with:
	name: build-${{ matrix.component }}
	path: \|
	dist/
	frontend/dist/
	retention-days: 7

	# Deploy to Staging
	deploy-staging:
	runs-on: ubuntu-latest
	needs: [build, quality-gates]
	if: \|
	needs.quality-gates.outputs.gates-passed == 'true' &&
	(github.event.inputs.deploy_to_staging == 'true' \|\| github.ref == 'refs/heads/develop')
	environment: staging

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Download build artifacts
	uses: actions/download-artifact@v3
	with:
	path: build-artifacts/

	- name: Deploy to staging
	run: \|
	echo "Deploying to staging environment..."
	# Add deployment logic here
	# This would typically involve:
	# - Pushing Docker images to registry
	# - Updating Kubernetes deployments
	# - Running database migrations
	# - Health checks

	- name: Run smoke tests
	run: \|
	echo "Running smoke tests..."
	# Add smoke test logic here

	- name: Notify deployment
	uses: 8398a7/action-slack@v3
	if: always()
	with:
	status: ${{ job.status }}
	channel: '#deployments'
	webhook_url: ${{ secrets.SLACK_WEBHOOK }}

	# Deploy to Production
	deploy-production:
	runs-on: ubuntu-latest
	needs: [build, quality-gates, deploy-staging]
	if: \|
	needs.quality-gates.outputs.gates-passed == 'true' &&
	github.ref == 'refs/heads/main'
	environment: production

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Download build artifacts
	uses: actions/download-artifact@v3
	with:
	path: build-artifacts/

	- name: Deploy to production
	run: \|
	echo "Deploying to production environment..."
	# Add production deployment logic here

	- name: Run production health checks
	run: \|
	echo "Running production health checks..."
	# Add health check logic here

	- name: Create GitHub Release
	if: startsWith(github.ref, 'refs/tags/')
	uses: actions/create-release@v1
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	with:
	tag_name: ${{ github.ref }}
	release_name: Release ${{ github.ref }}
	draft: false
	prerelease: false

	# Monitoring and Alerting
	monitoring:
	runs-on: ubuntu-latest
	needs: [deploy-staging, deploy-production]
	if: always() && (needs.deploy-staging.result == 'success' \|\| needs.deploy-production.result == 'success')

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup monitoring
	run: \|
	pip install -r requirements.txt
	pip install prometheus-client grafana-api

	- name: Update monitoring dashboards
	run: \|
	python ml-model-api/monitoring_system.py

	- name: Check system health
	run: \|
	python -c "
	import requests
	import sys
	try:
	response = requests.get('https://staging.flavorsnap.ai/health', timeout=10)
	if response.status_code == 200:
	print('Health check passed')
	else:
	print(f'Health check failed: {response.status_code}')
	sys.exit(1)
	except Exception as e:
	print(f'Health check error: {e}')
	sys.exit(1)
	"

	- name: Send notifications
	if: failure()
	uses: 8398a7/action-slack@v3
	with:
	status: failure
	channel: '#alerts'
	webhook_url: ${{ secrets.SLACK_WEBHOOK }}

	# Cleanup
	cleanup:
	runs-on: ubuntu-latest
	needs: [deploy-staging, deploy-production]
	if: always()

	steps:
	- name: Cleanup old artifacts
	uses: actions/github-script@v6
	with:
	script: \|
	const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
	owner: context.repo.owner,
	repo: context.repo.repo,
	run_id: context.runId,
	});

	const oldArtifacts = artifacts.data.artifacts.filter(artifact => {
	const created = new Date(artifact.created_at);
	const daysOld = (Date.now() - created) / (1000 * 60 * 60 * 24);
	return daysOld > 30;
	});

	for (const artifact of oldArtifacts) {
	await github.rest.actions.deleteArtifact({
	owner: context.repo.owner,
	repo: context.repo.repo,
	artifact_id: artifact.id,
	});
	console.log(`Deleted artifact: ${artifact.name}`);
	}

Navigation Menu

Advanced CI/CD Pipeline #37

Workflow file

Advanced CI/CD Pipeline #37

Uh oh!

Workflow file for this run