Merge pull request #8 from fedspendingtransparency/mod/dev-12162-agency-loader

[DEV-12162] Agency loader

Author: alburde1

brus_backend_common/models/__init__.py

@@ -1,8 +1,22 @@
 from brus_backend_common.models.lakehouse_model import LakeHouseCurrentMigration, ExternalDataLoadDate
 
-from brus_backend_common.models.reference import DEFCBronze, DEFCGold, DEFCGroup, FONBronze, FONGold
+from brus_backend_common.models.reference import (
+    DEFCBronze,
+    DEFCGold,
+    DEFCGroup,
+    FONBronze,
+    FONGold,
+    AgencyBronze,
+    CGACGold,
+    FRECGold,
+    SubTierAgencyGold,
+)
 
 LAKEHOUSE_MODEL_CLASSES = [
+    AgencyBronze,
+    CGACGold,
+    FRECGold,
+    SubTierAgencyGold,
     DEFCBronze,
     DEFCGroup,
     DEFCGold,

brus_backend_common/models/reference.py

@@ -2,6 +2,110 @@
 from brus_backend_common.models.lakehouse_model import BaseSchema, CSVModel, LakeHouseDatabase, SchemaField, SchemaType
 
 
+class AgencyBronze(CSVModel):
+    BUCKET_NAME = CONFIG.LAKEHOUSE_REFERENCE_BUCKET
+    DATABASE_NAME = LakeHouseDatabase.BRONZE
+    TABLE_NAME = "agency_codes"
+    DESCRIPTION = "Raw Agency CSV placed in S3"
+    CSV_NAME = "agency_codes.csv"
+    UNIQUE_CONSTRAINTS = [("CGAC AGENCY CODE", "FREC", "SUBTIER CODE")]
+    MIGRATION_HISTORY = []
+
+    STRUCTURE = BaseSchema(
+        [
+            SchemaField("CGAC AGENCY CODE", SchemaType.STRING, False),
+            SchemaField("AGENCY NAME", SchemaType.STRING, False),
+            SchemaField("AGENCY ABBREVIATION", SchemaType.STRING, True),
+            SchemaField("FREC", SchemaType.STRING, True),
+            SchemaField("FREC Entity Description", SchemaType.STRING, True),
+            SchemaField("FREC ABBREVIATION", SchemaType.STRING, True),
+            SchemaField("SUBTIER CODE", SchemaType.STRING, True),
+            SchemaField("SUBTIER NAME", SchemaType.STRING, True),
+            SchemaField("SUBTIER ABBREVIATION", SchemaType.STRING, True),
+            SchemaField("Admin Org Name", SchemaType.STRING, True),
+            SchemaField("ADMIN_ORG", SchemaType.STRING, True),
+            SchemaField("TOPTIER_FLAG", SchemaType.BOOLEAN, False),
+            SchemaField("IS_FREC", SchemaType.BOOLEAN, False),
+            SchemaField("FREC CGAC ASSOCIATION", SchemaType.BOOLEAN, False),
+            SchemaField("USER SELECTABLE ON USASPENDING.GOV", SchemaType.BOOLEAN, False),
+            SchemaField("MISSION", SchemaType.STRING, True),
+            SchemaField("ABOUT AGENCY DATA", SchemaType.STRING, True),
+            SchemaField("WEBSITE", SchemaType.STRING, True),
+            SchemaField("CONGRESSIONAL JUSTIFICATION", SchemaType.STRING, True),
+            SchemaField("ICON FILENAME", SchemaType.STRING, True),
+            SchemaField("COMMENT", SchemaType.STRING, True),
+        ]
+    )
+
+
+class CGACGold(CSVModel):
+    BUCKET_NAME = CONFIG.LAKEHOUSE_REFERENCE_BUCKET
+    DATABASE_NAME = LakeHouseDatabase.GOLD
+    TABLE_NAME = "cgac"
+    DESCRIPTION = "CGAC Data after processing"
+    CSV_NAME = "cgac.csv"
+    PK = "cgac_id"
+    UNIQUE_CONSTRAINTS = ["cgac_code"]
+    MIGRATION_HISTORY = []
+
+    STRUCTURE = BaseSchema(
+        [
+            SchemaField("cgac_id", SchemaType.INTEGER, False),
+            SchemaField("cgac_code", SchemaType.STRING, False),
+            SchemaField("agency_name", SchemaType.STRING, False),
+            SchemaField("agency_abbreviation", SchemaType.STRING, True),
+            SchemaField("display_name", SchemaType.STRING, False),
+            SchemaField("icon_name", SchemaType.STRING, True),
+        ]
+    )
+
+
+class FRECGold(CSVModel):
+    BUCKET_NAME = CONFIG.LAKEHOUSE_REFERENCE_BUCKET
+    DATABASE_NAME = LakeHouseDatabase.GOLD
+    TABLE_NAME = "frec"
+    DESCRIPTION = "FREC Data after processing"
+    CSV_NAME = "frec.csv"
+    PK = "frec_id"
+    UNIQUE_CONSTRAINTS = ["frec_code"]
+    MIGRATION_HISTORY = []
+
+    STRUCTURE = BaseSchema(
+        [
+            SchemaField("frec_id", SchemaType.INTEGER, False),
+            SchemaField("frec_code", SchemaType.STRING, False),
+            SchemaField("agency_name", SchemaType.STRING, False),
+            SchemaField("agency_abbreviation", SchemaType.STRING, True),
+            SchemaField("display_name", SchemaType.STRING, False),
+            SchemaField("cgac_code", SchemaType.STRING, False),
+            SchemaField("icon_name", SchemaType.STRING, True),
+        ]
+    )
+
+
+class SubTierAgencyGold(CSVModel):
+    BUCKET_NAME = CONFIG.LAKEHOUSE_REFERENCE_BUCKET
+    DATABASE_NAME = LakeHouseDatabase.GOLD
+    TABLE_NAME = "subtier_agency"
+    DESCRIPTION = "Sub Tier Agency Data after processing"
+    CSV_NAME = "subtier_agency.csv"
+    PK = "subtier_agency_id"
+    UNIQUE_CONSTRAINTS = ["subtier_code"]
+    MIGRATION_HISTORY = []
+
+    STRUCTURE = BaseSchema(
+        [
+            SchemaField("subtier_agency_id", SchemaType.INTEGER, False),
+            SchemaField("subtier_code", SchemaType.STRING, False),
+            SchemaField("subtier_name", SchemaType.STRING, False),
+            SchemaField("cgac_code", SchemaType.STRING, False),
+            SchemaField("frec_code", SchemaType.STRING, False),
+            SchemaField("priority", SchemaType.INTEGER, False),
+            SchemaField("is_frec", SchemaType.BOOLEAN, False),
+        ]
+    )
+
+
 class DEFCBronze(CSVModel):
     BUCKET_NAME = CONFIG.LAKEHOUSE_REFERENCE_BUCKET
     DATABASE_NAME = LakeHouseDatabase.BRONZE