Skip to content

Commit 18871df

Browse files
bosesunehabosesuneha
committed
Move private-cluster integration test to 1ES pipeline
GitHub-repo Federated Identity Credentials are being removed, so the private-cluster integration test moves from GitHub Actions to an Azure DevOps 1ES pipeline that authenticates via a WIF service connection. - Add .pipelines/1es-integration-tests-private.yml (extends 1ES Unofficial template; runs on staging-pool-amd64-mariner-2). - Invoke the node24 action via `node lib/index.js` with INPUT_* env vars. - Use a pre-created, shared resource group; create/delete only a per-build cluster (no az group create/delete). - Add optional resourceGroup= arg to k8s-deploy-test.py (defaults to cluster name) so verification targets the shared RG. - Delete the GitHub workflow whose FIC-based azure/login no longer authenticates.
1 parent c56f17f commit 18871df

3 files changed

Lines changed: 162 additions & 86 deletions

File tree

‎.azure-pipelines/1es-integration-tests-private.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
# Private-cluster integration test (1ES pipeline).
2+
#
3+
# Runs the k8s-deploy action against a real private AKS cluster. Lives in Azure
4+
# DevOps (not GitHub Actions) because GitHub-repo Federated Identity Credentials
5+
# are being removed; auth uses a governed WIF service connection instead.
6+
#
7+
# The node24 JS action is invoked as `node lib/index.js` with inputs supplied via
8+
# INPUT_* env vars. There is no action.yml default injection here, so keep INPUT_*
9+
# in sync with action.yml; skip-tls-verify (getBooleanInput) and GITHUB_WORKFLOW
10+
# must be set or the action throws.
11+
#
12+
# The pipeline does not create/delete the resource group (it must already exist);
13+
# it only creates and deletes a per-build cluster inside it.
14+
15+
trigger:
16+
branches:
17+
include:
18+
- main
19+
20+
pr:
21+
branches:
22+
include:
23+
- releases/*
24+
25+
resources:
26+
repositories:
27+
- repository: 1esPipelines
28+
type: git
29+
name: 1ESPipelineTemplates/1ESPipelineTemplates
30+
ref: refs/tags/release
31+
32+
variables:
33+
serviceConnection: k8s-deploy-integration-oidc
34+
resourceGroup: k8s-deploy-intg-rg
35+
location: eastus2
36+
clusterName: test-$(Build.BuildId)
37+
namespace: test-$(Build.BuildId)
38+
workflowFriendlyName: k8s-deploy-integration-tests-private
39+
40+
extends:
41+
template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines
42+
parameters:
43+
pool:
44+
name: staging-pool-amd64-mariner-2
45+
image: 1es-azlinux-3-amd64-custom-disk
46+
os: linux
47+
hostArchitecture: amd64
48+
sdl:
49+
sourceAnalysisPool:
50+
name: staging-pool-amd64-mariner-2
51+
image: azcu-agent-amd64-windows-22-img
52+
os: windows
53+
hostArchitecture: amd64
54+
stages:
55+
- stage: integration_test
56+
displayName: Private cluster integration tests
57+
jobs:
58+
- job: run_integration_test
59+
displayName: Run Private Cluster Integration Tests
60+
timeoutInMinutes: 60
61+
steps:
62+
- task: NodeTool@0
63+
displayName: Install Node.js
64+
inputs:
65+
versionSpec: '24.x'
66+
67+
- script: |
68+
rm -rf node_modules/
69+
npm install
70+
displayName: Install dependencies
71+
72+
- script: npm run build
73+
displayName: Build
74+
75+
- task: AzureCLI@2
76+
displayName: Create private AKS cluster and set context
77+
inputs:
78+
azureSubscription: $(serviceConnection)
79+
scriptType: bash
80+
scriptLocation: inlineScript
81+
inlineScript: |
82+
set -e
83+
set +x
84+
# RG must already exist; do not run `az group create`.
85+
az aks create --name $(clusterName) --resource-group $(resourceGroup) --enable-private-cluster --generate-ssh-keys
86+
az aks get-credentials --resource-group $(resourceGroup) --name $(clusterName)
87+
88+
- task: AzureCLI@2
89+
displayName: Create namespace to run tests
90+
inputs:
91+
azureSubscription: $(serviceConnection)
92+
scriptType: bash
93+
scriptLocation: inlineScript
94+
inlineScript: |
95+
set -e
96+
az aks command invoke --resource-group $(resourceGroup) --name $(clusterName) --command "kubectl create ns $(namespace)"
97+
98+
- task: UsePythonVersion@0
99+
displayName: Install Python
100+
inputs:
101+
versionSpec: '3.x'
102+
103+
- task: AzureCLI@2
104+
displayName: Executing deploy action for pod
105+
inputs:
106+
azureSubscription: $(serviceConnection)
107+
scriptType: bash
108+
scriptLocation: inlineScript
109+
inlineScript: |
110+
set -e
111+
node lib/index.js
112+
env:
113+
KUBECONFIG: $(HOME)/.kube/config
114+
# Inputs supplied via INPUT_* (no action.yml injection here).
115+
INPUT_ACTION: deploy
116+
INPUT_STRATEGY: basic
117+
INPUT_TRAFFIC-SPLIT-METHOD: pod
118+
INPUT_SKIP-TLS-VERIFY: 'false'
119+
INPUT_NAMESPACE: $(namespace)
120+
INPUT_IMAGES: nginx:1.14.2
121+
INPUT_MANIFESTS: |
122+
test/integration/manifests/test.yml
123+
test/integration/manifests/test2.yml
124+
INPUT_PRIVATE-CLUSTER: 'true'
125+
INPUT_RESOURCE-GROUP: $(resourceGroup)
126+
INPUT_NAME: $(clusterName)
127+
INPUT_RESOURCE-TYPE: Microsoft.ContainerService/managedClusters
128+
# Required by the action; also the workflowFriendlyName label.
129+
GITHUB_WORKFLOW: $(workflowFriendlyName)
130+
131+
- task: AzureCLI@2
132+
displayName: Checking if deployments and services were created
133+
inputs:
134+
azureSubscription: $(serviceConnection)
135+
scriptType: bash
136+
scriptLocation: inlineScript
137+
inlineScript: |
138+
set -e
139+
python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment containerName=nginx:1.14.2 labels=app:nginx,workflow:actions.github.qkg1.top-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx
140+
python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service labels=workflow:actions.github.qkg1.top-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx
141+
142+
python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment2 containerName=nginx:1.14.2 labels=app:nginx2,workflow:actions.github.qkg1.top-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2
143+
python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service2 labels=workflow:actions.github.qkg1.top-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2
144+
145+
- task: AzureCLI@2
146+
displayName: Clean up AKS cluster
147+
condition: always()
148+
inputs:
149+
azureSubscription: $(serviceConnection)
150+
scriptType: bash
151+
scriptLocation: inlineScript
152+
inlineScript: |
153+
# Delete only the cluster; the shared RG persists.
154+
echo "deleting AKS cluster $(clusterName)"
155+
az aks delete --yes --resource-group $(resourceGroup) --name $(clusterName) || true

‎.github/workflows/run-integration-tests-private.yml‎

Lines changed: 0 additions & 85 deletions
This file was deleted.

‎test/integration/k8s-deploy-test.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
ingressServicesKey = "ingressServices"
2222
tsServicesKey = "tsServices"
2323
privateKey = "private"
24+
resourceGroupKey = "resourceGroup"
2425

2526

2627
def parseArgs(sysArgs):
@@ -211,7 +212,12 @@ def main():
211212
try:
212213
if privateKey in parsedArgs:
213214
uniqueName = parsedArgs[privateKey]
214-
azPrefix = f"az aks command invoke --resource-group {uniqueName} --name {uniqueName} --command "
215+
# The resource group defaults to the cluster's unique name for
216+
# backward compatibility (older workflows named the RG and cluster
217+
# identically). When the RG is shared/pre-created and differs from
218+
# the cluster name, pass it explicitly via resourceGroup=<rg>.
219+
resourceGroup = parsedArgs.get(resourceGroupKey, uniqueName)
220+
azPrefix = f"az aks command invoke --resource-group {resourceGroup} --name {uniqueName} --command "
215221
cmd = azPrefix + "'" + cmd + "'"
216222
outputString = os.popen(cmd).read()
217223
successExit = "exitcode=0"

0 commit comments

Comments
 (0)