feat(azure): End to end Azure CICD (#6574)

* feat: add experimental azurerm remote state backend

Co-Authored-By: Olof Mattsson <olof@devopsninja.nu>

* feat: azure backend unit tests

* chore: tests cleanup

* chore: azure backend fixes

* azure backend

* chore: code cleanup

* chore: PR comments

* chore: secrets fetching

* chore: venv usage

* chore: secrets cleanup

* erorr handling simplifications

* docs cleanup

* chore: azure PR cleanup

* chore: tests cleanup

* chore: PR comments

* fix(azure): refuse cross-cloud state migration

* chore: state fixes

* chore: azurehelper remote state

* chore: azure integraiton tests

* chore: env variables setup for cicd tests

* chore: azure cicd fixes

* chore: azure backned fixes

* chore:  azure tests checks

* CICD simplifications

* resources cleanup

* chore: azure cicd fixes

* git commit -m "ci(azure): pass ARM_ACCESS_KEY so tofu init can authenticate"

* chore: CI fixes

* integration test cleanup

* chore: ARM key update

* chore: blob test update

* chore: contributing update

* chore(azure): address review feedback on build tags, deref, and secret quoting

* chore: cicd updates

* chore: build test tag update

* Secrets cleanup

* chore: tests cleanup

* chore: PR review notes

* chore: changelog cleanup

* fix(ci): provide credentials to Azure internal tests

---------

Co-authored-by: Olof Mattsson <olof@devopsninja.nu>

Author: denis256

.github/scripts/setup/generate-secrets.sh

@@ -70,6 +70,16 @@ for SECRET in $SECRETS; do
 		printf "export TG_AZURE_TEST_STORAGE_ACCOUNT='%s'\n" "${TG_AZURE_TEST_STORAGE_ACCOUNT}" >>"$ENV_FILE"
 	elif [[ "$SECRET" == "TG_AZURE_TEST_SUBSCRIPTION_ID" && -n "${TG_AZURE_TEST_SUBSCRIPTION_ID:-}" ]]; then
 		printf "export TG_AZURE_TEST_SUBSCRIPTION_ID='%s'\n" "${TG_AZURE_TEST_SUBSCRIPTION_ID}" >>"$ENV_FILE"
+	elif [[ "$SECRET" == "TG_AZURE_TEST_RESOURCE_GROUP" && -n "${TG_AZURE_TEST_RESOURCE_GROUP:-}" ]]; then
+		printf "export TG_AZURE_TEST_RESOURCE_GROUP='%s'\n" "${TG_AZURE_TEST_RESOURCE_GROUP}" >>"$ENV_FILE"
+	elif [[ "$SECRET" == "ARM_CLIENT_ID" && -n "${ARM_CLIENT_ID:-}" ]]; then
+		printf "export ARM_CLIENT_ID='%s'\n" "${ARM_CLIENT_ID}" >>"$ENV_FILE"
+	elif [[ "$SECRET" == "ARM_CLIENT_SECRET" && -n "${ARM_CLIENT_SECRET:-}" ]]; then
+		printf "export ARM_CLIENT_SECRET='%s'\n" "${ARM_CLIENT_SECRET}" >>"$ENV_FILE"
+	elif [[ "$SECRET" == "ARM_TENANT_ID" && -n "${ARM_TENANT_ID:-}" ]]; then
+		printf "export ARM_TENANT_ID='%s'\n" "${ARM_TENANT_ID}" >>"$ENV_FILE"
+	elif [[ "$SECRET" == "ARM_SUBSCRIPTION_ID" && -n "${ARM_SUBSCRIPTION_ID:-}" ]]; then
+		printf "export ARM_SUBSCRIPTION_ID='%s'\n" "${ARM_SUBSCRIPTION_ID}" >>"$ENV_FILE"
 	elif [[ "$SECRET" == "TG_OCI_TEST_ECR_REPOSITORY" && -n "${TG_OCI_TEST_ECR_REPOSITORY:-}" ]]; then
 		printf "export TG_OCI_TEST_ECR_REPOSITORY='%s'\n" "${TG_OCI_TEST_ECR_REPOSITORY}" >>"$ENV_FILE"
 	elif [[ "$SECRET" == "TG_OCI_TEST_GHCR_REPOSITORY" && -n "${TG_OCI_TEST_GHCR_REPOSITORY:-}" ]]; then

.github/workflows/integration-test.yml

@@ -114,6 +114,25 @@ jobs:
                 AZURE_TENANT_ID,
                 TG_AZURE_TEST_STORAGE_ACCOUNT,
                 TG_AZURE_TEST_SUBSCRIPTION_ID,
+                ARM_CLIENT_ID,
+                ARM_CLIENT_SECRET,
+                ARM_TENANT_ID,
+                ARM_SUBSCRIPTION_ID,
+              ]
+          - name: Azure Integration
+            os: ubuntu
+            target: ./test
+            tags: azure
+            run: ^TestAzure
+            secrets:
+              [
+                TG_AZURE_TEST_STORAGE_ACCOUNT,
+                TG_AZURE_TEST_SUBSCRIPTION_ID,
+                TG_AZURE_TEST_RESOURCE_GROUP,
+                ARM_CLIENT_ID,
+                ARM_CLIENT_SECRET,
+                ARM_TENANT_ID,
+                ARM_SUBSCRIPTION_ID,
               ]
           - name: OCI Registries
             os: ubuntu
@@ -242,6 +261,11 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           TG_AZURE_TEST_STORAGE_ACCOUNT: ${{ secrets.TG_AZURE_TEST_STORAGE_ACCOUNT }}
           TG_AZURE_TEST_SUBSCRIPTION_ID: ${{ secrets.TG_AZURE_TEST_SUBSCRIPTION_ID }}
+          TG_AZURE_TEST_RESOURCE_GROUP: ${{ secrets.TG_AZURE_TEST_RESOURCE_GROUP }}
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
           TG_OCI_TEST_ECR_REPOSITORY: ${{ secrets.TG_OCI_TEST_ECR_REPOSITORY }}
           TG_OCI_TEST_GHCR_REPOSITORY: ${{ secrets.TG_OCI_TEST_GHCR_REPOSITORY }}
           TG_OCI_TEST_GHCR_USERNAME: ${{ secrets.TG_OCI_TEST_GHCR_USERNAME }}

.golangci.yml

@@ -202,7 +202,7 @@ linters:
 
       - linters:
           - forbidigo
-        path: ^(internal/cli/commands/browse/tui/model\.go|internal/cli/commands/catalog/cli\.go|internal/cli/commands/catalog/tui/form\.go|internal/cli/commands/catalog/tui/tags_layout\.go|internal/cli/commands/commands\.go|internal/cli/commands/hcl/validate/validate\.go|internal/cli/commands/help/cli\.go|internal/cli/commands/render/render\.go|internal/cli/commands/scaffold/cli\.go|internal/cli/commands/scaffold/scaffold\.go)$
+        path: ^(internal/cli/commands/browse/tui/model\.go|internal/cli/commands/catalog/cli\.go|internal/view/tui/form/form\.go|internal/cli/commands/catalog/tui/tags_layout\.go|internal/cli/commands/commands\.go|internal/cli/commands/hcl/validate/validate\.go|internal/cli/commands/help/cli\.go|internal/cli/commands/render/render\.go|internal/cli/commands/scaffold/cli\.go|internal/cli/commands/scaffold/scaffold\.go)$
       # Env-var-backed flags resolve against the process environment rather
       # than the venv's Env map, so every flag parsed from an env var bypasses
       # the seam.

docs/src/content/docs/05-community/01-contributing.mdx

@@ -268,6 +268,61 @@ gcloud projects add-iam-policy-binding <gcp-project> \
   --role="roles/iam.serviceAccountTokenCreator"
 ```
 
+#### Azure tests
+
+Terragrunt also includes end-to-end tests for the Azure Storage (`azurerm`) remote-state backend. These are prefixed with `TestAzure*` and use the `azure` build tag, because they create and destroy real Azure resources. The `internal/azurehelper` tests use the same build tag; their live round-trip skips when credentials are absent.
+
+```bash
+GOFLAGS='-tags=azure' go test ./internal/azurehelper/...
+GOFLAGS='-tags=azure' go test -run 'TestAzure*' ./test
+```
+
+Unlike some other suites, these fail rather than skip when credentials are missing, so a misconfigured environment is visible instead of silently reporting success.
+
+To successfully run the Azure tests, you must set the following environment variables:
+
+- `TG_AZURE_TEST_SUBSCRIPTION_ID`: The subscription hosting the test storage account. `ARM_SUBSCRIPTION_ID` or `AZURE_SUBSCRIPTION_ID` also work.
+- `TG_AZURE_TEST_STORAGE_ACCOUNT`: The name of a pre-existing storage account the tests create containers in. `ARM_STORAGE_ACCOUNT_NAME` also works.
+- `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`: Service principal credentials. The `ARM_*` spelling of each is also accepted, and these can be omitted entirely when authenticating with `az login`.
+
+With ARM-capable authentication, the resource group is looked up from the storage account. Set `TG_AZURE_TEST_RESOURCE_GROUP` when using an access key or SAS token, because data-plane credentials cannot perform that lookup.
+
+```bash
+export TG_AZURE_TEST_SUBSCRIPTION_ID="00000000-0000-0000-0000-000000000000"
+export TG_AZURE_TEST_STORAGE_ACCOUNT="tgteststate"
+az login
+```
+
+The identity used for Azure tests must hold the following roles:
+
+- `Storage Blob Data Contributor`, because the tests authorize blob access with Microsoft Entra rather than an account key.
+- `Storage Account Contributor`, because blob versioning and soft delete are management-plane properties of the account. Without it, `TestAzureBackendVersioningConverges` cannot converge them.
+
+You can assign these roles using the following Azure CLI commands:
+
+```bash
+RG="/subscriptions/<subscription-id>/resourceGroups/<resource-group>"
+
+az role assignment create \
+  --assignee "<client-id>" \
+  --role "Storage Blob Data Contributor" \
+  --scope "$RG"
+
+az role assignment create \
+  --assignee "<client-id>" \
+  --role "Storage Account Contributor" \
+  --scope "$RG"
+```
+
+Scoping both roles to the resource group also lets the tests resolve it from the storage account name, so no subscription-wide assignment is needed.
+
+An account key (`ARM_ACCESS_KEY`) authenticates the blob data plane and is enough for
+most of the suite, but it cannot reach the management plane at all, so
+`TestAzureBackendVersioningConverges` skips under it. Only an Entra identity runs the
+whole suite.
+
+The tests create and delete blob containers with unique names on each run; they never create or delete the storage account itself.
+
 #### Race tests
 
 Given that Terragrunt is a tool that frequently involves concurrently running multiple things at once, there's always a risk for race conditions to occur. As such, there are dedicated tests that are run with the `-race` flag in CI to use golang's built-in tooling for identifying race conditions.

docs/src/data/experiments/azure-backend.mdx

@@ -88,8 +88,9 @@ To transition the `azure-backend` feature to a stable release, the following mus
 - [x] `internal/azurehelper` package wrapping the Azure SDK with a builder pattern matching `awshelper`/`gcphelper`.
 - [x] Bootstrap of storage accounts and blob containers, including versioning and soft delete convergence.
 - [ ] Optional RBAC role assignment for `use_azuread_auth` during bootstrap (the data-plane role must currently be granted manually).
-- [x] Delete and migrate operations for state blobs and containers with confirmation prompts.
+- [x] Delete operations for state blobs and containers, with confirmation prompts, and state migration within a storage account.
 - [ ] Direct state file reads from Azure blobs for `--dependency-fetch-output-from-state`.
 - [x] Documentation covering authentication methods, configuration keys, and troubleshooting.
-- [ ] End-to-end live coverage against a real subscription for resource group and storage account bootstrap, policy convergence, backend delete, and state migration. The existing live blob round-trip covers only the data-plane helper; hermetic unit tests ship behind the `azure` build tag.
+- [x] End-to-end live coverage against a real subscription for container bootstrap, blob versioning convergence, and backend delete, behind the `azure` build tag.
+- [ ] End-to-end live coverage for resource group and storage account creation, soft-delete retention convergence, and state migration. The current live tests assume a pre-existing storage account.
 - [ ] Community feedback on real-world usage.

internal/azurehelper/blob_test.go

@@ -184,9 +184,9 @@ func TestBlobClient_CopyBlob_RequiresArgs(t *testing.T) {
 func TestBlob_LiveRoundTrip(t *testing.T) {
 	t.Parallel()
 
-	env := venv.OSVenv().Env
-	account := env["TG_AZURE_TEST_STORAGE_ACCOUNT"]
-	sub := env["TG_AZURE_TEST_SUBSCRIPTION_ID"]
+	testVenv := venv.OSVenv()
+	account := testVenv.Env["TG_AZURE_TEST_STORAGE_ACCOUNT"]
+	sub := testVenv.Env["TG_AZURE_TEST_SUBSCRIPTION_ID"]
 
 	if account == "" || sub == "" {
 		t.Skip("TG_AZURE_TEST_STORAGE_ACCOUNT and TG_AZURE_TEST_SUBSCRIPTION_ID are required for live test")
@@ -199,8 +199,9 @@ func TestBlob_LiveRoundTrip(t *testing.T) {
 		WithSessionConfig(&azurehelper.AzureSessionConfig{
 			SubscriptionID:     sub,
 			StorageAccountName: account,
-			UseAzureADAuth:     true,
+			UseAzureADAuth:     new(true),
 		}).
+		WithVenv(testVenv).
 		Build(log.New())
 	require.NoError(t, err, "Build config")