[DEV-12155] Ensure tests run on separate buckets than your local

Author: dpb-bah

brus_backend_common/config.py

@@ -210,6 +210,11 @@ def JDBC_METASTORE_URL(self):
     MINIO_DATA_DIR: str = ""
 
 
+# TODO: Update CONFIG to use BaseModel with nested environment variables which can help generate this list automatically
+#       (ex. BUCKETS__DATA_ARCHIVE -> CONFIG.BUCKETS.DATA_ARCHIVE)
+CONFIG_BUCKETS = [attr for attr, value in DefaultConfig().__dict__.items() if attr.endswith("_BUCKET")]
+
+
 def pull_ssm_config() -> dict:
     """This function lives in the liminal space between CONFIG and helpers.aws, having a hand in both.
     While this is essentially more helpers.aws based, that file imports and uses CONFIG values from this file,
@@ -245,3 +250,11 @@ def set_brus_config(config: dict):
 if not CONFIG.IS_LOCAL:
     ssm_config = pull_ssm_config()
     CONFIG = DefaultConfig(**ssm_config)
+
+# Overwrite any values if running in a test to prevent conflicting with your local environment
+# https://docs.pytest.org/en/stable/example/simple.html#detect-if-running-from-within-a-pytest-run
+if os.environ.get("PYTEST_VERSION") is not None:
+    # Rename the buckets in the test environment to not overwrite your local buckets
+    CONFIG = DefaultConfig(
+        **{bucket_config: f"test-{getattr(CONFIG, bucket_config)}" for bucket_config in CONFIG_BUCKETS}
+    )

brus_backend_common/models/__init__.py

@@ -1,14 +1,6 @@
 from brus_backend_common.models.lakehouse_model import LakeHouseCurrentMigration, ExternalDataLoadDate
 
 from brus_backend_common.models.reference import DEFCBronze, DEFCGold, DEFCGroup, FONBronze, FONGold
-from brus_backend_common.config import CONFIG
-
-LAKEHOUSE_BUCKETS = {
-    "broker": CONFIG.LAKEHOUSE_BROKER_BUCKET,
-    "reference": CONFIG.LAKEHOUSE_REFERENCE_BUCKET,
-    "usas": CONFIG.LAKEHOUSE_USAS_BUCKET,
-}
-LAKEHOUSE_BUCKET_NAMES = list(LAKEHOUSE_BUCKETS.values())
 
 LAKEHOUSE_MODEL_CLASSES = [
     DEFCBronze,

brus_backend_common/tests/conftest_spark.py

@@ -10,10 +10,10 @@
     is_spark_context_stopped,
     stop_spark_context,
 )
-from brus_backend_common.config import CONFIG
+from brus_backend_common.config import CONFIG, CONFIG_BUCKETS
 from brus_backend_common.helpers.aws import _get_boto3
 from brus_backend_common.helpers.configs import LOCAL_BASIC_EXTRA_CONF  # LOCAL_EXTENDED_EXTRA_CONF
-from brus_backend_common.models import LAKEHOUSE_MODELS, LAKEHOUSE_BUCKET_NAMES
+from brus_backend_common.models import LAKEHOUSE_MODELS
 from brus_backend_common.scripts.create_migrate_lakehouse_model import main as create_migrate
 
 if TYPE_CHECKING:
@@ -208,7 +208,7 @@ def delta_lake_unittest_schema(spark: "SparkSession", hive_unittest_metastore_db
 
 @pytest.fixture(scope="session")
 def setup_teardown_buckets_session() -> Generator[List[str], None, None]:
-    buckets = LAKEHOUSE_BUCKET_NAMES + [CONFIG.PUBLIC_FILES_BUCKET, CONFIG.METRICS_BUCKET, CONFIG.DATA_SOURCES_BUCKET]
+    buckets = [getattr(CONFIG, bucket) for bucket in CONFIG_BUCKETS]
     for bucket in buckets:
         s3_unittest_data_bucket_setup(bucket)