Run ASIM tests on "ASIM-SchemaDataTester-GithubShared" workspace

Update checkpoint parser to handle logs with space within DeviceVendor #4281

Workflow file for this run

.github/workflows/runAsimSchemaAndDataTesters.yaml at 5b9930d

	# Each pull request that updates ASIM parsers triggers the script.
	# The script runs ASIM Schema and Data testers on the "eco-connector-test" workspace.
	name: Run ASIM tests on "ASIM-SchemaDataTester-GithubShared" workspace
	on:
	pull_request_target:
	types: [opened, edited, reopened, synchronize, labeled]
	branches:
	- master
	paths:
	- "Parsers/ASimDns/Parsers/**"
	- "Parsers/ASimNetworkSession/Parsers/**"
	- "Parsers/ASimWebSession/Parsers/**"
	- "Parsers/ASimProcessEvent/Parsers/**"
	- "Parsers/ASimAuditEvent/Parsers/**"
	- "Parsers/ASimAuthentication/Parsers/**"
	- "Parsers/ASimFileEvent/Parsers/**"
	- "Parsers/ASimRegistryEvent/Parsers/**"
	- "Parsers/ASimUserManagement/Parsers/**"
	- "Parsers/ASimDhcpEvent/Parsers/**"
	- "Parsers/ASimAlertEvent/Parsers/**"
	- "Parsers/ASimAssetEntity/Parsers/**"

	# Allows you to run this workflow manually from the Actions tab
	workflow_dispatch:

	permissions:
	contents: read

	concurrency:
	# group: asim-tests-${{ github.event.pull_request.number \|\| github.sha }}
	# cancel-in-progress: true
	group: >-
	asim-tests-${{ github.event.pull_request.head.repo.fork
	&& format('pr-{0}', github.event.pull_request.number)
	\|\| format('run-{0}', github.run_id) }}
	cancel-in-progress: ${{ github.event.pull_request.head.repo.fork }}

	jobs:
	# Security gate: Fork PRs require manual approval via "SafeToRun" label
	# Internal PRs (same repo) can proceed without labels
	security-gate:
	name: Security approval gate for fork PRs
	runs-on: ubuntu-latest
	timeout-minutes: 15
	permissions:
	contents: read
	pull-requests: write
	issues: write
	outputs:
	approved: ${{ steps.check-approval.outputs.approved }}
	steps:
	- name: Checkout pull request branch
	if: github.event.pull_request != null
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	fetch-depth: 0

	- name: Check if PR needs approval
	id: check-approval
	run: \|
	# Function to log with consistent formatting
	log_info() { echo "ℹ️ $1"; }
	log_success() { echo "✅ $1"; }

	log_info "Starting PR approval check..."

	# Check if this is a fork PR
	is_fork="${{ github.event.pull_request.head.repo.fork }}"
	log_info "Fork PR: $is_fork"

	if [ "$is_fork" = "true" ]; then
	log_info "FORK PR DETECTED - Proceeding with security checks"

	# Check if "SafeToRun" label is present
	labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
	log_info "Available labels: $labels"

	if echo "$labels" \| grep -q "SafeToRun"; then
	log_success "'SafeToRun' label found - checking if re-approval needed"

	# Check if this workflow was triggered by new commits (synchronize event)
	trigger_event="${{ github.event.action }}"
	log_info "Workflow triggered by: $trigger_event"

	if [ "$trigger_event" = "synchronize" ]; then
	log_info "New commits detected on fork PR with existing 'SafeToRun' label"
	log_info "Maintainer must remove and re-add the 'SafeToRun' label for security"
	echo "approved=false" >> $GITHUB_OUTPUT
	echo "needs_approval=false" >> $GITHUB_OUTPUT
	echo "comment_needed=true" >> $GITHUB_OUTPUT
	exit 1
	else
	log_success "Label approval granted (no new commits)"
	echo "approved=true" >> $GITHUB_OUTPUT
	echo "needs_approval=false" >> $GITHUB_OUTPUT
	echo "comment_needed=false" >> $GITHUB_OUTPUT
	fi
	else
	log_info "'SafeToRun' label not found - approval required"
	echo "approved=false" >> $GITHUB_OUTPUT
	echo "needs_approval=true" >> $GITHUB_OUTPUT
	echo "comment_needed=true" >> $GITHUB_OUTPUT
	exit 1
	fi
	else
	log_success "Internal PR - auto-approved"
	echo "approved=true" >> $GITHUB_OUTPUT
	echo "needs_approval=false" >> $GITHUB_OUTPUT
	echo "comment_needed=false" >> $GITHUB_OUTPUT
	fi

	- name: Prevent fork modifications to test infrastructure
	if: github.event.pull_request.head.repo.fork == true
	shell: bash
	run: \|
	set -euo pipefail

	log_info() { echo "ℹ️ $1"; }
	log_error() { echo "❌ $1"; }
	log_success() { echo "✅ $1"; }

	log_info "Checking for modifications to asimParsersTest folder in fork PR..."

	# We are currently checked out at the fork's HEAD SHA (actions/checkout did that).
	# Add the base repo as a remote and fetch the base branch, so we can diff reliably.
	git remote remove upstream 2>/dev/null \|\| true
	git remote add upstream "https://github.com/${{ github.repository }}.git"

	log_info "Fetching base branch ${{ github.event.pull_request.base.ref }} from upstream..."
	git fetch --no-tags --prune upstream "${{ github.event.pull_request.base.ref }}"

	# Diff base branch (FETCH_HEAD) -> current HEAD (fork head)
	modified_files="$(git diff --name-only "FETCH_HEAD...HEAD")"

	if echo "$modified_files" \| grep -E "\.script/tests/asimParsersTest/" > /dev/null; then
	log_error "Fork PRs cannot modify the asimParsersTest test infrastructure folder"
	log_error "Modified test files detected:"
	echo "$modified_files" \| grep "\.script/tests/asimParsersTest/" \| sed 's/^/ - /'
	exit 1
	fi

	log_success "No modifications to asimParsersTest folder detected"

	- name: Comment on fork PR for approval guidance
	if: \|
	always() &&
	github.event.pull_request.head.repo.fork == true &&
	steps.check-approval.outputs.comment_needed == 'true'
	uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410
	with:
	script: \|
	// Helper function for consistent logging
	const log = (level, message) => {
	const icons = { info: 'ℹ️', success: '✅', warning: '⚠️', error: '❌' };
	console.log(`${icons[level] \|\| 'ℹ️'} ${message}`);
	};

	log('info', 'Comment step triggered for fork PR approval guidance');

	try {
	// Fetch existing comments
	const { data: comments } = await github.rest.issues.listComments({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	});

	// Look for existing security guidance comments
	const botComments = comments.filter(comment =>
	comment.user.login === 'github-actions[bot]' &&
	(comment.body.includes('🔒 Security Approval Required') \|\|
	comment.body.includes('🔒 Security Re-approval Required'))
	);

	// Create approval message
	const timestamp = new Date().toISOString();

	let commentBody;
	if ('${{ steps.check-approval.outputs.needs_approval }}' === 'true') {
	// Initial approval scenario
	commentBody = `🔒 Security Approval Required

	This fork PR requires manual approval before automated testing can run.

	For security, a maintainer must:
	1. 📝 Review the code changes carefully
	2. ✅ Verify file types - This PR should only contain \`.yml\`, \`.yaml\`, or \`.json\` files. Check for any executable scripts (.ps1, .py, .sh, .exe, etc.) which are not allowed in this context.
	3. 🏷️ Add the \`SafeToRun\` label if the changes are safe to execute

	Note: If new commits are added later, simply remove and re-add the \`SafeToRun\` label.

	---
	🤖 Automated security check • Created: ${timestamp}
	Learn more: [GitHub Security Lab - Preventing PWN Requests](https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/)`;
	} else {
	// Re-approval scenario (new commits with existing label)
	commentBody = `🔒 Security Re-approval Required

	⚠️ New commits detected: This fork PR has been updated with new commits while the \`SafeToRun\` label was present.

	For security, a maintainer must:
	1. 📝 Review the latest commits carefully for any security concerns
	2. ✅ Verify file types - Ensure new commits only contain \`.yml\`, \`.yaml\`, or \`.json\` files. Reject if any executable scripts (.ps1, .py, .sh, .exe, etc.) are included.
	3. 🏷️ Remove the \`SafeToRun\` label
	4. 🏷️ Re-add the \`SafeToRun\` label if the new commits are safe

	This simple process ensures that all commits have been properly reviewed before testing with repository secrets.

	---
	🤖 Automated security check • Updated: ${timestamp}
	Learn more: [GitHub Security Lab - Preventing PWN Requests](https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/)`;
	}

	// Always create a new comment for maximum visibility
	// Keep existing comments for audit trail - don't delete them
	log('info', 'Creating new security guidance comment (preserving audit trail)');
	await github.rest.issues.createComment({
	owner: context.repo.owner,
	repo: context.repo.repo,
	issue_number: context.issue.number,
	body: commentBody
	});
	log('success', 'Created new security guidance comment');

	log('success', 'Comment operation completed successfully');

	} catch (error) {
	log('error', `Failed to post comment: ${error.message}`);
	if (error.response) {
	log('error', `API Response: ${error.response.status} - ${error.response.data?.message \|\| 'Unknown error'}`);
	}
	// Don't fail the step if comment posting fails
	log('warning', 'Comment posting failed, but continuing workflow...');
	}

	Run-ASim-TemplateValidation:
	name: Run ASim Template Validation tests
	needs: security-gate
	if: needs.security-gate.outputs.approved == 'true'
	runs-on: ubuntu-latest
	steps:
	- name: Checkout pull request branch
	uses: actions/checkout@v3
	with:
	ref: ${{github.event.pull_request.head.sha}}
	repository: ${{github.event.pull_request.head.repo.full_name}}
	persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
	fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
	- name: Setup git config
	run: \|
	git config --local user.name "github-actions[bot]"
	git config --local user.email "<>"
	- name: Merge master into pull request branch
	run: \|
	git merge origin/master
	Conflicts=$(git ls-files -u \| wc -l)
	if [ "$Conflicts" -gt 0 ] ; then
	echo "There is a merge conflict. Aborting"
	git merge --abort
	exit 1
	fi
	- name: Set up Python
	uses: actions/setup-python@v2
	with:
	python-version: "3.x"
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install requests
	pip install PyYAML
	pip install tabulate
	- name: Run ASim parsers template validations python script
	run: \|
	filePath=".script/tests/asimParsersTest/VerifyASimParserTemplate.py"
	url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/VerifyASimParserTemplate.py"
	# Check if file exists and delete if it does
	if [ -f "$filePath" ]; then
	rm -f "$filePath"
	fi
	# Download the file
	echo "Downloading script from the master: $url"
	curl -o "$filePath" "$url"
	# Execute the script
	python "$filePath"
	Run-ASim-Sample-Data-Ingest:
	needs: Run-ASim-TemplateValidation
	name: Run ASim Sample Data Ingestion
	runs-on: ubuntu-latest
	permissions:
	id-token: write
	contents: read
	steps:
	- name: Checkout pull request branch
	uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.sha}}
	repository: ${{github.event.pull_request.head.repo.full_name}}
	persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
	fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
	- name: Setup git config
	run: \|
	git config --local user.name "github-actions[bot]"
	git config --local user.email "<>"
	- name: Merge master into pull request branch
	run: \|
	git merge origin/master
	Conflicts=$(git ls-files -u \| wc -l)
	if [ "$Conflicts" -gt 0 ] ; then
	echo "There is a merge conflict. Aborting"
	git merge --abort
	exit 1
	fi
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.x"
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install azure-identity
	pip install requests
	pip install PyYAML
	pip install azure-monitor-ingestion
	pip install azure-core
	- name: Login to Azure Public Cloud
	uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5
	with:
	client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	allow-no-subscriptions: true
	- name: Asim Sample Log Ingestion
	id: Ingestlogs
	run: \|
	filePath=".script/tests/asimParsersTest/ingestASimSampleData.py"
	url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/ingestASimSampleData.py"
	# Check if file exists and delete if it does
	if [ -f "$filePath" ]; then
	rm -f "$filePath"
	fi
	# Download the file
	echo "Downloading script from the master: $url"
	curl -o "$filePath" "$url"
	chmod +x "$filePath"
	# Execute the script
	python "$filePath" "${{ github.event.pull_request.number }}"
	Run-ASim-Schema-Data-tests:
	needs: Run-ASim-Sample-Data-Ingest
	name: Run ASim Schema and Data tests
	runs-on: ubuntu-latest
	permissions:
	id-token: write
	contents: read
	steps:
	- name: Checkout pull request branch
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	persist-credentials: false
	fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.

	- name: Login to Azure Public Cloud with AzPowershell
	uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5
	with:
	client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	enable-AzPSSession: true
	allow-no-subscriptions: true

	- name: Setup git config
	run: \|
	git config --local user.name "github-actions[bot]"
	git config --local user.email "<>"

	- name: Merge master into pull request branch
	run: \|
	git merge origin/master
	Conflicts=$(git ls-files -u \| wc -l)
	if [ "$Conflicts" -gt 0 ] ; then
	echo "There is a merge conflict. Aborting"
	git merge --abort
	exit 1
	fi
	- name: Run ASIM Schema and Data tests PowerShell script
	uses: azure/powershell@53dd145408794f7e80f97cfcca04155c85234709
	with:
	inlineScript: \|
	$filePath = ".script/tests/asimParsersTest/runAsimTesters.ps1"
	$url = "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/runAsimTesters.ps1"
	# Check if file exists and delete if it does
	if (Test-Path $filePath) {
	Remove-Item $filePath -Force
	}
	# Download the runAsimTesters file
	Write-Host "Downloading script from the master: $url"
	Invoke-WebRequest -Uri $url -OutFile $filePath
	# download the convertYamlToObject.ps1 script form master
	$filePath_convert_yaml = ".script/tests/asimParsersTest/convertYamlToObject.ps1"
	$url_convert_yaml = "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/convertYamlToObject.ps1"
	# Check if file exists and delete if it does
	if (Test-Path $filePath_convert_yaml) {
	Remove-Item $filePath_convert_yaml -Force
	}
	# Download the convertYamlToObject file
	Write-Host "Downloading script from the master: $url_convert_yaml"
	Invoke-WebRequest -Uri $url_convert_yaml -OutFile $filePath_convert_yaml
	# Execute the script
	& $filePath
	azPSVersion: "latest"
	Run-ASim-Parser-Filtering-Tests:
	needs: Run-ASim-Sample-Data-Ingest
	name: Run ASim Parser Filtering tests
	runs-on: ubuntu-latest
	permissions:
	id-token: write
	contents: read
	steps:
	- name: Checkout pull request branch
	uses: actions/checkout@v3
	with:
	ref: ${{ github.event.pull_request.head.sha }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token.
	fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
	- name: Setup git config
	run: \|
	git config --local user.name "github-actions[bot]"
	git config --local user.email "<>"
	- name: Merge master into pull request branch
	run: \|
	git merge origin/master
	Conflicts=$(git ls-files -u \| wc -l)
	if [ "$Conflicts" -gt 0 ] ; then
	echo "There is a merge conflict. Aborting"
	git merge --abort
	exit 1
	fi
	- name: Setup Python
	uses: actions/setup-python@v2
	with:
	python-version: "3.x"
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install PyYAML
	pip install azure-identity
	pip install azure-monitor-query
	- name: Login to Azure Public Cloud
	uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5
	with:
	client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
	tenant-id: ${{ secrets.AZURE_TENANT_ID }}
	allow-no-subscriptions: true
	- name: Run ASim parsers filtering tests python script
	run: \|
	filePath=".script/tests/asimParsersTest/ASimFilteringTest.py"
	url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/ASimFilteringTest.py"
	# Check if file exists and delete if it does
	if [ -f "$filePath" ]; then
	rm -f "$filePath"
	fi
	# Download the file
	echo "Downloading script from the master: $url"
	curl -o "$filePath" "$url"
	# Execute the script
	python "$filePath"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update checkpoint parser to handle logs with space within DeviceVendor #4281

Workflow file

Update checkpoint parser to handle logs with space within DeviceVendor #4281

Uh oh!

Workflow file for this run