Merge pull request #4091 from bcgov/fix/924-attachments-file-fixtures

924 Fix attachments file fixtures

Author: pbastia

bc_obps/registration/apps.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.db.models.signals import pre_migrate
 from django.apps import AppConfig
 from django.db import connection
@@ -16,3 +17,6 @@ class RegistrationConfig(AppConfig):
     def ready(self):
         pre_migrate.connect(create_erc_schemas, sender=self)
         from .signals import signals, consumers  # noqa: F401
+
+        if settings.ENVIRONMENT != 'prod':
+            from .signals import post_save_fixtures  # noqa: F401

bc_obps/registration/signals/post_save_fixtures.py

@@ -0,0 +1,29 @@
+import logging
+from typing import Any, Type
+from django.core.files.base import ContentFile
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from registration.models.document import Document
+from django.core.files.storage import default_storage
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=Document)
+def handle_save_registration_document_fixture(
+    sender: Type[Document], instance: Document, **kwargs: dict[str, Any]
+) -> None:
+    """
+    Handler that creates registration documents on the storage medium (cloud or local storage)
+    when the fixture is loaded
+    """
+
+    if not kwargs['raw']:
+        # raw=True is passed only for a fixture
+        # https://docs.djangoproject.com/en/5.2/topics/db/fixtures/#how-fixtures-are-saved-to-the-database
+        return
+
+    logger.info(f"Post-save: saving file fixture to storage {instance.file.name}")
+    default_storage.save(
+        instance.file.name, content=ContentFile(f"Document Fixture {instance.file.name}".encode("utf-8"))
+    )

bc_obps/reporting/apps.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.db.models.signals import pre_migrate
 from django.apps import AppConfig
 from django.db import connection
@@ -17,3 +18,6 @@ def ready(self):
         pre_migrate.connect(create_erc_schemas, sender=self)
         from .signals import consumers  # noqa: F401
         from .signals import signals  # noqa: F401
+
+        if settings.ENVIRONMENT != 'prod':
+            from .signals import post_save_fixtures  # noqa: F401

bc_obps/reporting/fixtures/mock/report_attachment.json

@@ -6,7 +6,7 @@
       "created_at": "2025-05-26T11:20:58.144081-04:00",
       "updated_at": null,
       "archived_at": null,
-      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_OIOdlgX.pdf",
+      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_1.pdf",
       "attachment_type": "verification_statement",
       "attachment_name": "VERIFICATION_ATTACHMENT.pdf",
       "archived_by_id": null,
@@ -23,7 +23,7 @@
       "created_at": "2025-05-26T11:20:58.144081-04:00",
       "updated_at": null,
       "archived_at": null,
-      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_OIOdlgX.pdf",
+      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_2.pdf",
       "attachment_type": "verification_statement",
       "attachment_name": "VERIFICATION_ATTACHMENT.pdf",
       "archived_by_id": null,
@@ -40,7 +40,7 @@
       "created_at": "2025-05-26T11:20:58.144081-04:00",
       "updated_at": null,
       "archived_at": null,
-      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_OIOdlgX.pdf",
+      "attachment": "report_attachments/2025/VERIFICATION_ATTACHMENT_3.pdf",
       "attachment_type": "verification_statement",
       "attachment_name": "VERIFICATION_ATTACHMENT.pdf",
       "archived_by_id": null,

bc_obps/reporting/signals/post_save_fixtures.py

@@ -0,0 +1,30 @@
+import logging
+from typing import Any, Type
+from django.core.files.base import ContentFile
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from reporting.models.report_attachment import ReportAttachment
+from django.core.files.storage import default_storage
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=ReportAttachment)
+def handle_save_report_attachment_fixture(
+    sender: Type[ReportAttachment], instance: ReportAttachment, **kwargs: dict[str, Any]
+) -> None:
+    """
+    Handler that creates attachments on the storage medium (cloud or local storage)
+    when the fixture is loaded
+    """
+
+    if not kwargs['raw']:
+        # raw=True is passed only for a fixture
+        # https://docs.djangoproject.com/en/5.2/topics/db/fixtures/#how-fixtures-are-saved-to-the-database
+        return
+
+    logger.info(f"Post-save: saving file fixture to storage {instance.attachment.name}")
+    default_storage.save(
+        instance.attachment.name,
+        content=ContentFile(f"Attachment Fixture {instance.attachment.name}".encode('utf-8')),
+    )

dags/bc_obps_reset_data.py

@@ -31,9 +31,14 @@
     doc_md=RESET_DAG_DOC,
 )
 def reset_data():
+
+    @task
+    def reset_attachment_storage_task():
+        trigger_k8s_cronjob("reset-attachment-storage", BCIERS_NAMESPACE)
+
     @task
     def reset_data_task():
-        trigger_k8s_cronjob('reset-database', BCIERS_NAMESPACE)
+        trigger_k8s_cronjob("reset-database", BCIERS_NAMESPACE)
 
     cycle_backend_pod_task = KubernetesJobOperator(
         task_id="cycle_backend_pod",
@@ -52,7 +57,7 @@ def reset_data_task():
         trigger_dag_id=WAIT_FOR_BACKEND_ROLLOUT_DAG_NAME,
     )
 
-    reset_data_task >> cycle_backend_pod_task >> trigger_wait_for_backend_rollout
+    ([reset_attachment_storage_task(), reset_data_task()] >> cycle_backend_pod_task >> trigger_wait_for_backend_rollout)
 
 
 ROLLOUT_DAG_DOC = """

helm/cas-bciers/templates/cronJobs/reset-attachment-storage.yaml

@@ -0,0 +1,64 @@
+{{- if not (hasSuffix "-prod" .Release.Namespace) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: reset-attachment-storage
+  labels: {{- include "cas-bciers.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": "pre-install,pre-upgrade"
+    "helm.sh/hook-weight": "-10"
+spec:
+  suspend: true # triggered manually via Airflow
+  schedule: "0 * * * *"
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 600
+      template:
+        metadata:
+          labels: {{ include "cas-bciers.labels" . | nindent 12 }}
+            component: job-with-database-access
+        spec:
+          activeDeadlineSeconds: 600
+          restartPolicy: Never
+          serviceAccountName: deployer
+          containers:
+            - name: reset-attachment-storage
+              image: "gcr.io/google.com/cloudsdktool/google-cloud-cli:stable"
+              imagePullPolicy: "Always"
+              env:
+                - name: GOOGLE_APPLICATION_CREDENTIALS
+                  value: "/attachment-credentials/attachment-credentials.json"
+                - name: GS_CLEAN_BUCKET_NAME
+                  value: {{ .Release.Namespace }}-bciers-attach-clean
+              command:
+                - /usr/bin/env
+                - bash
+                - -c
+                - |
+                  set -x;
+
+                  gcloud auth login --cred-file=$GOOGLE_APPLICATION_CREDENTIALS;
+                  gcloud storage rm --recursive "gs://$GS_CLEAN_BUCKET_NAME/report_attachments/**";
+                  gcloud storage rm --recursive "gs://$GS_CLEAN_BUCKET_NAME/documents/**";
+
+                  exit 0;
+
+              volumeMounts:
+                - mountPath: "/attachment-credentials"
+                  name: gcs-attachment-credentials
+                  readOnly: true
+                - mountPath: "/.config"
+                  name: config-volume
+          volumes:
+            - name: gcs-attachment-credentials
+              secret:
+                secretName: gcp-{{ .Release.Namespace }}-bciers-attach-service-account-key
+                items:
+                - key: credentials.json
+                  path: attachment-credentials.json
+            - name: config-volume
+              emptyDir:
+                sizeLimit: 50Mi
+{{- end}}