DC-250: Provision rotation app-user across all target databases (#495)

Author: spokenbird

tofu/modules/sebt_application/locals.tf

@@ -21,6 +21,16 @@ locals {
     DD_API_KEY = data.aws_secretsmanager_secret.datadog_key["this"].arn
   } : {}
 
+  # tofu-modules-aws-fargate-service (module.api) doesn't expose a
+  # service_name output — only cluster_name. Its cluster and service happen
+  # to share the same name today (HENNGE/ecs/aws: join("-", compact([project,
+  # environment, service]))), but that's an implementation detail of the
+  # upstream module, not a contract. Reconstruct it explicitly here so a
+  # future naming-scheme change in that module doesn't silently break the
+  # rotation Lambda's ecs:UpdateService call. If the module ever adds a
+  # service_name output, switch to that instead.
+  api_ecs_service_name = join("-", compact(["${var.project}-${var.state}", var.environment, "api"]))
+
   # Allow the AWS Security Agent penetration test to bypass the WAF in the
   # development environment by matching its unique User-Agent. "allow" is a
   # terminating action, so matching requests skip all subsequent rules

tofu/modules/sebt_application/main.tf

@@ -192,9 +192,10 @@ module "database" {
   ingress_security_groups = [module.api.security_group_id]
   ingress_cidrs           = var.db_ingress_cidrs
 
-  db_name          = "SebtPortal"
-  ecs_cluster_name = module.api.cluster_name
-  ecs_service_name = module.api.cluster_name
+  db_name             = "SebtPortal"
+  additional_db_names = var.dc_source_db_name != "" ? [var.dc_source_db_name] : []
+  ecs_cluster_name    = module.api.cluster_name
+  ecs_service_name    = local.api_ecs_service_name
 
   skip_final_snapshot = var.skip_final_snapshot
   apply_immediately   = var.apply_immediately

tofu/modules/sebt_database/lambda/rotate_db_credentials.py

@@ -7,13 +7,17 @@
 credential staleness window entirely.
 
 Environment variables (set by OpenTofu):
-    ADMIN_SECRET_ARN — ARN of the Secrets Manager secret holding RDS admin
-                       credentials (the RDS-managed master-user secret).
-    DB_HOST          — RDS SQL Server hostname.
-    DB_PORT          — SQL Server port (default: "1433").
-    DB_NAME          — Target database name for user creation and connection tests.
-    ECS_CLUSTER      — ECS cluster name to redeploy after finishSecret.
-    ECS_SERVICE      — ECS service name to redeploy after finishSecret.
+    ADMIN_SECRET_ARN     — ARN of the Secrets Manager secret holding RDS admin
+                           credentials (the RDS-managed master-user secret).
+    DB_HOST              — RDS SQL Server hostname.
+    DB_PORT              — SQL Server port (default: "1433").
+    DB_NAME              — Target database name for user creation and connection tests.
+    ADDITIONAL_DB_NAMES  — Comma-separated list of extra databases on the same RDS
+                           instance where the app user also needs a database-level
+                           user provisioned (e.g. DC's DcSource database). Empty/unset
+                           means no extras — the typical case for CO.
+    ECS_CLUSTER          — ECS cluster name to redeploy after finishSecret.
+    ECS_SERVICE          — ECS service name to redeploy after finishSecret.
 """
 
 import json
@@ -105,7 +109,6 @@ def set_secret(client, secret_arn, token):
 
     host = os.environ["DB_HOST"]
     port = int(os.environ.get("DB_PORT", "1433"))
-    dbname = pending["dbname"]
 
     # Server-level operations (CREATE/ALTER LOGIN) require master db context.
     master_conn = pymssql.connect(
@@ -126,40 +129,44 @@ def set_secret(client, secret_arn, token):
     finally:
         master_conn.close()
 
-    # Database-level operations (CREATE USER, role grant) require the target db.
-    db_conn = pymssql.connect(
-        server=host,
-        port=port,
-        user=admin["username"],
-        password=admin["password"],
-        database=dbname,
-        tds_version="7.4",
-    )
-    try:
-        cursor = db_conn.cursor()
-        _ensure_db_user_exists(cursor, pending["username"])
-        db_conn.commit()
-    finally:
-        db_conn.close()
+    # Database-level operations (CREATE USER, role grant) run against every
+    # database the app user needs access to, not just the primary one.
+    for dbname in _target_db_names(pending["dbname"]):
+        db_conn = pymssql.connect(
+            server=host,
+            port=port,
+            user=admin["username"],
+            password=admin["password"],
+            database=dbname,
+            tds_version="7.4",
+        )
+        try:
+            cursor = db_conn.cursor()
+            _ensure_db_user_exists(cursor, pending["username"])
+            db_conn.commit()
+            logger.info("Ensured db user exists in %s", dbname)
+        finally:
+            db_conn.close()
 
 
 def test_secret(client, secret_arn, token):
-    """Verify the AWSPENDING credentials by opening a test connection."""
+    """Verify the AWSPENDING credentials by opening a test connection to every target database."""
     pending = _get_secret_dict(client, secret_arn, stage="AWSPENDING", version_id=token)
 
-    conn = pymssql.connect(
-        server=os.environ["DB_HOST"],
-        port=int(os.environ.get("DB_PORT", "1433")),
-        user=pending["username"],
-        password=pending["password"],
-        database=pending["dbname"],
-        tds_version="7.4",
-    )
-    try:
-        conn.cursor().execute("SELECT 1")
-        logger.info("Test connection succeeded for pending login")
-    finally:
-        conn.close()
+    for dbname in _target_db_names(pending["dbname"]):
+        conn = pymssql.connect(
+            server=os.environ["DB_HOST"],
+            port=int(os.environ.get("DB_PORT", "1433")),
+            user=pending["username"],
+            password=pending["password"],
+            database=dbname,
+            tds_version="7.4",
+        )
+        try:
+            conn.cursor().execute("SELECT 1")
+            logger.info("Test connection succeeded for pending login against %s", dbname)
+        finally:
+            conn.close()
 
 
 def finish_secret(client, secret_arn, token):
@@ -189,6 +196,19 @@ def finish_secret(client, secret_arn, token):
 # Private helpers
 # ---------------------------------------------------------------------------
 
+def _target_db_names(primary_dbname):
+    """Return every database the app user login should be provisioned in.
+
+    Combines the primary database (from the secret) with any additional
+    state-specific databases configured via ADDITIONAL_DB_NAMES (e.g. DC's
+    DcSource database, which lives on the same RDS instance but doesn't
+    exist for other states like CO).
+    """
+    additional = os.environ.get("ADDITIONAL_DB_NAMES", "")
+    extras = [name.strip() for name in additional.split(",") if name.strip()]
+    return [primary_dbname] + extras
+
+
 def _other_user(username):
     """Return the inactive app user (the one not currently active)."""
     if username not in _USERS:
@@ -278,10 +298,14 @@ def _restart_ecs_service():
         logger.info("Triggered rolling ECS restart for %s/%s", cluster, service)
     except Exception:
         # The secret is already rotated; running tasks remain on the active
-        # user's valid credentials. A manual redeploy will pick up the new
-        # secret at next launch.
+        # user's valid credentials until the next natural redeploy. Not
+        # fatal to rotation, but silent — log a distinct, alertable marker
+        # so an external monitor (e.g. a Datadog log monitor) can page on
+        # it instead of this failing invisibly.
         logger.exception(
-            "Failed to trigger ECS restart for %s/%s — manual redeploy required",
+            "DB_ROTATION_ECS_RESTART_FAILED cluster=%s service=%s — "
+            "credentials rotated successfully but the ECS restart failed; "
+            "manual redeploy required to pick up the new secret",
             cluster,
             service,
         )

tofu/modules/sebt_database/lambda/test_rotate_db_credentials.py

@@ -54,6 +54,20 @@ def _make_client(token, current_user="appuser"):
 # _other_user
 # ---------------------------------------------------------------------------
 
+def test_target_db_names_returns_primary_only_when_no_additional():
+    assert rot._target_db_names("SebtPortal") == ["SebtPortal"]
+
+
+@patch.dict(os.environ, {"ADDITIONAL_DB_NAMES": "DcSource"})
+def test_target_db_names_includes_additional_dbs():
+    assert rot._target_db_names("SebtPortal") == ["SebtPortal", "DcSource"]
+
+
+@patch.dict(os.environ, {"ADDITIONAL_DB_NAMES": " DcSource , , OtherDb "})
+def test_target_db_names_strips_whitespace_and_drops_empties():
+    assert rot._target_db_names("SebtPortal") == ["SebtPortal", "DcSource", "OtherDb"]
+
+
 def test_other_user_returns_clone_when_given_primary():
     assert rot._other_user("appuser") == "appuser_clone"
 
@@ -147,6 +161,27 @@ def test_set_secret_opens_connections_and_commits(mock_pymssql):
     mock_conn.close.assert_called()
 
 
+@patch.dict(os.environ, {"ADMIN_SECRET_ARN": "admin-arn", "DB_HOST": "db", "DB_PORT": "1433",
+                         "DB_NAME": "SebtPortal", "ADDITIONAL_DB_NAMES": "DcSource",
+                         "ECS_CLUSTER": "cluster", "ECS_SERVICE": "svc"})
+@patch("rotate_db_credentials.pymssql")
+def test_set_secret_provisions_db_user_in_additional_databases(mock_pymssql):
+    mock_conn = MagicMock()
+    mock_pymssql.connect.return_value = mock_conn
+
+    client = MagicMock()
+    client.get_secret_value.side_effect = [
+        {"SecretString": json.dumps({"username": "appuser_clone", "password": "NewPw1!",
+                                     "host": "db", "port": "1433", "dbname": "SebtPortal"})},
+        {"SecretString": json.dumps({"username": "admin", "password": "AdminPw1!"})},
+    ]
+
+    rot.set_secret(client, "secret-arn", "new-token")
+
+    called_dbs = [kwargs["database"] for _, kwargs in mock_pymssql.connect.call_args_list]
+    assert called_dbs == ["master", "SebtPortal", "DcSource"]
+
+
 # ---------------------------------------------------------------------------
 # test_secret
 # ---------------------------------------------------------------------------
@@ -170,6 +205,27 @@ def test_test_secret_succeeds_on_valid_connection(mock_pymssql):
     mock_conn.close.assert_called_once()
 
 
+@patch.dict(os.environ, {"ADMIN_SECRET_ARN": "admin-arn", "DB_HOST": "db", "DB_PORT": "1433",
+                         "DB_NAME": "SebtPortal", "ADDITIONAL_DB_NAMES": "DcSource",
+                         "ECS_CLUSTER": "cluster", "ECS_SERVICE": "svc"})
+@patch("rotate_db_credentials.pymssql")
+def test_test_secret_tests_connection_to_every_target_database(mock_pymssql):
+    mock_conn = MagicMock()
+    mock_pymssql.connect.return_value = mock_conn
+
+    client = MagicMock()
+    client.get_secret_value.return_value = {
+        "SecretString": json.dumps({"username": "appuser_clone", "password": "NewPw1!",
+                                    "host": "db", "port": "1433", "dbname": "SebtPortal"})
+    }
+
+    rot.test_secret(client, "secret-arn", "new-token")
+
+    called_dbs = [kwargs["database"] for _, kwargs in mock_pymssql.connect.call_args_list]
+    assert called_dbs == ["SebtPortal", "DcSource"]
+    assert mock_conn.cursor.return_value.execute.call_count == 2
+
+
 @patch.dict(os.environ, {"ADMIN_SECRET_ARN": "admin-arn", "DB_HOST": "db", "DB_PORT": "1433",
                          "DB_NAME": "SebtPortal", "ECS_CLUSTER": "cluster", "ECS_SERVICE": "svc"})
 @patch("rotate_db_credentials.pymssql")

tofu/modules/sebt_database/rotation.tf

@@ -226,12 +226,13 @@ resource "aws_lambda_function" "rotation" {
 
   environment {
     variables = {
-      ADMIN_SECRET_ARN = aws_db_instance.main.master_user_secret[0].secret_arn
-      DB_HOST          = aws_db_instance.main.address
-      DB_PORT          = tostring(local.port)
-      DB_NAME          = var.db_name
-      ECS_CLUSTER      = var.ecs_cluster_name
-      ECS_SERVICE      = var.ecs_service_name
+      ADMIN_SECRET_ARN    = aws_db_instance.main.master_user_secret[0].secret_arn
+      DB_HOST             = aws_db_instance.main.address
+      DB_PORT             = tostring(local.port)
+      DB_NAME             = var.db_name
+      ADDITIONAL_DB_NAMES = join(",", var.additional_db_names)
+      ECS_CLUSTER         = var.ecs_cluster_name
+      ECS_SERVICE         = var.ecs_service_name
     }
   }
 

tofu/modules/sebt_database/variables.tf

@@ -1,3 +1,9 @@
+variable "additional_db_names" {
+  type        = list(string)
+  description = "Additional databases on this RDS instance (beyond db_name) where the app-user login also needs a database-level user provisioned, e.g. DC's DcSource database. Empty means only db_name is provisioned — the typical case for CO, which has no equivalent database."
+  default     = []
+}
+
 variable "allocated_storage" {
   type        = number
   description = "Allocated storage in GB."