Merge pull request #6 from fedspendingtransparency/mod/dev-12155-funding-opportunity-loaders

[DEV-12155] Funding Opportunity Extractor and Loader

Author: dpb-bah

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

brus_backend_common/config.py

@@ -20,8 +20,14 @@
       Pydantic's dotenv will automagically populate it based on it
 """
 
+import logging
 import os
 import pathlib
+
+import boto3
+
+from dotenv import dotenv_values
+from io import StringIO
 from pydantic import BaseSettings, SecretStr
 
 from brus_backend_common.helpers.uri import get_jdbc_url_from_pg_uri
@@ -34,6 +40,8 @@
 
 ENV_FILE_PATH = os.path.join(_PROJECT_ROOT_DIR, ".env")
 
+logger = logging.getLogger(__name__)
+
 
 class DefaultConfig(BaseSettings):
     """Top-level config that defines all configuration variables, and their default, overridable values
@@ -130,6 +138,10 @@ def AWS_S3_ENDPOINT(self):
     def AWS_STS_ENDPOINT(self):
         return f"sts.{self.AWS_REGION}.amazonaws.com" if not self.IS_LOCAL else f"{self.MINIO_HOST}:{self.MINIO_PORT}"
 
+    @property
+    def AWS_SSM_ENDPOINT(self):
+        return f"ssm.{self.AWS_REGION}.amazonaws.com" if not self.IS_LOCAL else f"{self.MINIO_HOST}:{self.MINIO_PORT}"
+
     # Buckets
     DATA_ARCHIVE_BUCKET: str = ""
     DATA_EXTRACTS_BUCKET: str = ""
@@ -198,10 +210,51 @@ def JDBC_METASTORE_URL(self):
     MINIO_DATA_DIR: str = ""
 
 
+# TODO: Update CONFIG to use BaseModel with nested environment variables which can help generate this list automatically
+#       (ex. BUCKETS__DATA_ARCHIVE -> CONFIG.BUCKETS.DATA_ARCHIVE)
+CONFIG_BUCKETS = [attr for attr, value in DefaultConfig().__dict__.items() if attr.endswith("_BUCKET")]
+
+
+def pull_ssm_config() -> dict:
+    """This function lives in the liminal space between CONFIG and helpers.aws, having a hand in both.
+    While this is essentially more helpers.aws based, that file imports and uses CONFIG values from this file,
+    which'd result in a circular dependency.
+
+    Likewise, we could re-use the helper function below but that'd also run into a circular dependency.
+    ssm_client = _get_boto3('client', 'ssm')
+    """
+    env_group = "prod" if CONFIG.ENV_CODE == "prod" else "nonprod"
+
+    # TODO: Post-FAPC Cleanup
+    non_fapc_path = f"/{env_group}/brus-backend-common/{CONFIG.ENV_CODE}/.env"
+    fapc_path = "/kc-dtas/brus/broker/secrets"
+    secrets_param_name = fapc_path if CONFIG.FAPC else non_fapc_path
+
+    ssm_client = boto3.client("ssm", region_name=CONFIG.AWS_REGION)
+    secrets_yaml_param = ssm_client.get_parameter(Name=secrets_param_name, WithDecryption=True)
+    return dotenv_values(stream=StringIO(secrets_yaml_param["Parameter"]["Value"]))
+
+
 CONFIG = DefaultConfig()
 
 
-def set_brus_config(config):
+def set_brus_config(config: dict) -> None:
     """Takes in a config dict of the attributes to override"""
     for attr, value in config.items():
         setattr(CONFIG, attr, value)
+
+
+# Overwrite any values with ones pulled from SSM if not local
+# Note: DefaultConfig() can take the argument, but we need the initial default values to look up the right
+# Parameter values, so we're updating them after the initial pull.
+if not CONFIG.IS_LOCAL:
+    ssm_config = pull_ssm_config()
+    CONFIG = DefaultConfig(**ssm_config)
+
+# Overwrite any values if running in a test to prevent conflicting with your local environment
+# https://docs.pytest.org/en/stable/example/simple.html#detect-if-running-from-within-a-pytest-run
+if os.environ.get("PYTEST_VERSION") is not None:
+    # Rename the buckets in the test environment to not overwrite your local buckets
+    CONFIG = DefaultConfig(
+        **{bucket_config: f"test-{getattr(CONFIG, bucket_config)}" for bucket_config in CONFIG_BUCKETS}
+    )

brus_backend_common/helpers/aws.py

@@ -44,6 +44,9 @@ def get_storage_options() -> dict:
         "aws_sts_endpoint": (
             f"http://{CONFIG.AWS_STS_ENDPOINT}" if CONFIG.IS_LOCAL else f"https://{CONFIG.AWS_STS_ENDPOINT}"
         ),
+        "aws_ssm_endpoint": (
+            f"http://{CONFIG.AWS_SSM_ENDPOINT}" if CONFIG.IS_LOCAL else f"https://{CONFIG.AWS_SSM_ENDPOINT}"
+        ),
         "region": CONFIG.AWS_REGION,
         "allow_http": "true",
         "aws_conditional_put": "etag",
@@ -134,6 +137,9 @@ def _get_boto3(method_name: str, *args, region_name=CONFIG.AWS_REGION, **kwargs)
     """
     attr = getattr(boto3, method_name)
     kwargs.update({"region_name": region_name})
+    endpoint = None
+    if len(args) > 0 and args[0].upper() in ("S3", "SSM", "STS"):
+        endpoint = getattr(CONFIG, f"AWS_{args[0].upper()}_ENDPOINT")
 
     if callable(attr):
         if CONFIG.IS_LOCAL:
@@ -143,9 +149,8 @@ def _get_boto3(method_name: str, *args, region_name=CONFIG.AWS_REGION, **kwargs)
                 aws_secret_access_key=CONFIG.AWS_SECRET_KEY.get_secret_value(),
             )
             attr = getattr(session, method_name)
-            kwargs.update({"endpoint_url": f"http://{CONFIG.AWS_S3_ENDPOINT}"})
-        else:
-            kwargs.update({"endpoint_url": f"https://{CONFIG.AWS_S3_ENDPOINT}"})
+        if endpoint:
+            kwargs.update({"endpoint_url": f"http{'s' if not CONFIG.IS_LOCAL else ''}://{endpoint}"})
         return attr(*args, **kwargs)
     return attr
 

brus_backend_common/models/__init__.py

@@ -1,19 +1,13 @@
 from brus_backend_common.models.lakehouse_model import LakeHouseCurrentMigration, ExternalDataLoadDate
 
-from brus_backend_common.models.reference import DEFCBronze, DEFCGold, DEFCGroup
-from brus_backend_common.config import CONFIG
-
-LAKEHOUSE_BUCKETS = {
-    "broker": CONFIG.LAKEHOUSE_BROKER_BUCKET,
-    "reference": CONFIG.LAKEHOUSE_REFERENCE_BUCKET,
-    "usas": CONFIG.LAKEHOUSE_USAS_BUCKET,
-}
-LAKEHOUSE_BUCKET_NAMES = list(LAKEHOUSE_BUCKETS.values())
+from brus_backend_common.models.reference import DEFCBronze, DEFCGold, DEFCGroup, FONBronze, FONGold
 
 LAKEHOUSE_MODEL_CLASSES = [
     DEFCBronze,
     DEFCGroup,
     DEFCGold,
+    FONBronze,
+    FONGold,
     LakeHouseCurrentMigration,
     ExternalDataLoadDate,
 ]