Merge pull request #214 from maykinmedia/feature/oaf-188-csv-data-dump

add csv option to data dump script

Author: stevenbal

.github/workflows/bin-check.yml

@@ -39,6 +39,7 @@ jobs:
           python src/manage.py migrate
           src/manage.py loaddata demodata
           SCRIPTPATH=bin DUMP_FILE=dump.sql bin/dump_data.sh --combined
+          SCRIPTPATH=bin TAR_FILE=dump.tar bin/dump_data.sh --csv
         env:
           DB_PASSWORD: ""
           DB_USER: postgres
@@ -56,3 +57,10 @@ jobs:
         run: |
           createdb -h localhost -U postgres test
           psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d test -f dump.sql
+
+      - name: validate csv dump
+        run: |
+          tar -xf dump.tar
+          test -f core_objecttype.csv || exit 1
+          ! test -f auth_group.csv || exit 1
+          grep "id,uuid,name,name_plural,contact_email,description,maintainer_department,maintainer_organization,data_classification,contact_person,documentation_url,labels,provider_organization,source,update_frequency,created_at,modified_at,allow_geometry" core_objecttype.csv

Dockerfile

@@ -1,13 +1,13 @@
 # Stage 1 - Compile needed python dependencies
-FROM python:3.12-slim-bookworm AS build
+FROM python:3.12-slim-trixie AS build
 
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
         pkg-config \
         build-essential \
         libpq-dev \
         # required for (log) routing support in uwsgi
-        libpcre3 \
-        libpcre3-dev \
+        libpcre2-8-0 \
+        libpcre2-dev \
         git \
     && rm -rf /var/lib/apt/lists/*
 
@@ -19,7 +19,7 @@ RUN pip install -r requirements/production.txt
 
 
 # Stage 2 - build frontend
-FROM node:24-alpine AS frontend-build
+FROM node:24-trixie-slim AS frontend-build
 
 WORKDIR /app
 
@@ -35,7 +35,7 @@ RUN npm run build
 
 
 # Stage 3 - Build docker image suitable for execution and deployment
-FROM python:3.12-slim-bookworm AS production
+FROM python:3.12-slim-trixie AS production
 
 # Stage 3.1 - Set up the needed production dependencies
 # install all the dependencies for GeoDjango
@@ -44,11 +44,11 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
         procps \
         vim \
         # serve correct Content-Type headers
-        mime-support \
+        media-types \
         # (geo) django dependencies
         postgresql-client \
         gettext \
-        libpcre3 \
+        libpcre2-8-0 \
         binutils \
         libproj-dev \
         gdal-bin \

bin/dump_data.sh

@@ -10,49 +10,58 @@
 # or --combined which appends the data dump to the schema dump.
 # The schema dump could not use -t to filter tables because this excludes extensions like postgis in the dump.
 # pg_dump also does not add related tables automatically, so `dump_data.sh` does not add related data from accounts to the dump.
+#
+# with --csv a csv dump can be created for all tables in the given components. The csv files will be generated in the temporary directory csv_dumps
+# and combined into a single TAR archive csv_dumps.
 
-
-set -e
+set -euo pipefail
 
 DEFAULT_APPS=(core)
 
 export PGHOST=${DB_HOST:-db}
 export PGPORT=${DB_PORT:-5432}
-export PGUSER=${DB_USER:-objectypes}
-export PGDATABASE=${DB_NAME:-objectypes}
+export PGUSER=${DB_USER:-objecttypes}
+export PGDATABASE=${DB_NAME:-objecttypes}
 export PGPASSWORD=${DB_PASSWORD:-""}
 
 SCRIPT=$(readlink -f "$0")
 SCRIPTPATH=$(dirname "$SCRIPT")
 
 ${SCRIPTPATH}/wait_for_db.sh
 
-DUMP_FILE=${DUMP_FILE:-"dump_$(date +'%Y-%m-%d_%H-%M-%S').sql"}
+DEFAULT_FILE_NAME="dump_$(date +'%Y-%m-%d_%H-%M-%S')"
+DUMP_FILE=${DUMP_FILE:-"$DEFAULT_FILE_NAME.sql"}
+TAR_FILE=${TAR_FILE:-"$DEFAULT_FILE_NAME.tar"}
+CSV_OUTPUT_DIR="csv_dumps"
 
+CSV=false
 SCHEMA=true
 DATA=true
 COMBINED=false
+APPS=()
 
 for arg in "$@"; do
-  case "$arg" in
+    case "$arg" in
+    --csv) CSV=true ;;
     --schema-only) DATA=false ;;
-    --data-only)   SCHEMA=false ;;
-    --combined)    COMBINED=true ;;
+    --data-only) SCHEMA=false ;;
+    --combined) COMBINED=true ;;
     --*)
-      echo "Unknown flag: $arg"
-      exit 1
-      ;;
+        echo "Unknown flag: $arg"
+        exit 1
+        ;;
     *)
-      APPS+=("$arg") ;;
-  esac
+        APPS+=("$arg")
+        ;;
+    esac
 done
 
 # export given apps or export DEFAULT_APPS
 if [ "${#APPS[@]}" -eq 0 ]; then
-  APPS=("${DEFAULT_APPS[@]}")
+    APPS=("${DEFAULT_APPS[@]}")
 fi
 
->&2 echo "exporting: ${APPS[*]}"
+echo >&2 "exporting: ${APPS[*]}"
 
 # create -t flags for each app
 INCLUDES=()
@@ -61,32 +70,59 @@ for app in "${APPS[@]}"; do
 done
 
 dump_schema() {
-  echo "Dumping schema to $1..."
-  pg_dump --schema-only -f "$1"
+    echo "Dumping schema to $1..."
+    pg_dump --schema-only -f "$1"
 }
 
 dump_data() {
-  echo "Dumping data to $1..."
-  pg_dump "${INCLUDES[@]}" --disable-triggers --data-only > "$1"
+    echo "Dumping data to $1..."
+    pg_dump "${INCLUDES[@]}" --disable-triggers --data-only >"$1"
 }
 
 append_data() {
-  echo "Appending data to $1..."
-  pg_dump "${INCLUDES[@]}" --disable-triggers --data-only \
-    | sed '/^SET\|^SELECT pg_catalog.set_config/d' >> "$1"
+    echo "Appending data to $1..."
+    pg_dump "${INCLUDES[@]}" --disable-triggers --data-only |
+        sed '/^SET\|^SELECT pg_catalog.set_config/d' >>"$1"
 }
 
+dump_csv() {
+    mkdir -p $CSV_OUTPUT_DIR
+    echo "Dumping data to csv..."
+
+    WHERE_CLAUSE=""
+    for app in "${APPS[@]}"; do
+        if [ -n "$WHERE_CLAUSE" ]; then
+            WHERE_CLAUSE+=" OR "
+        fi
+        WHERE_CLAUSE+="tablename LIKE '${app}_%'"
+    done
+
+    TABLES=$(psql -Atc "SELECT tablename FROM pg_tables WHERE schemaname='public' AND ($WHERE_CLAUSE);")
+
+    for table in $TABLES; do
+        echo "dumping $table..."
+        psql -c "\copy $table TO '$CSV_OUTPUT_DIR/$table.csv' WITH CSV HEADER"
+    done
+
+    tar -cf "$TAR_FILE" -C "$CSV_OUTPUT_DIR" .
+    rm -rf "$CSV_OUTPUT_DIR"
+}
+
+if $CSV; then
+    dump_csv
+    exit 0
+fi
 
 if $COMBINED; then
-  dump_schema "$DUMP_FILE"
-  append_data "$DUMP_FILE"
-  exit 0
+    dump_schema "$DUMP_FILE"
+    append_data "$DUMP_FILE"
+    exit 0
 fi
 
 if $SCHEMA; then
-  dump_schema "schema__$DUMP_FILE"
+    dump_schema "schema__$DUMP_FILE"
 fi
 
 if $DATA; then
-  dump_data "data__$DUMP_FILE"
+    dump_data "data__$DUMP_FILE"
 fi

docker-compose.yml

@@ -6,12 +6,12 @@ services:
       POSTGRES_PASSWORD: objecttypes
     command: postgres -c max_connections=300 -c log_min_messages=LOG
     networks:
-      - objecttypes-api-dev 
+      - objecttypes-api-dev
 
   redis:
     image: redis
     networks:
-      - objecttypes-api-dev 
+      - objecttypes-api-dev
 
   web:
     image: maykinmedia/objecttypes-api:latest
@@ -39,8 +39,8 @@ services:
       - app=objecttypes-api
       - service=api
     networks:
-      - objecttypes-api-dev 
-    
+      - objecttypes-api-dev
+
 
   web-init:
     image: maykinmedia/objecttypes-api:latest
@@ -57,12 +57,12 @@ services:
     depends_on:
       - db
     networks:
-      - objecttypes-api-dev 
+      - objecttypes-api-dev
 
 volumes:
   db:
   log:
 
 networks:
   objecttypes-api-dev:
-    name: objecttypes-api-dev
+    name: objecttypes-api-dev

requirements/base.txt

@@ -353,7 +353,7 @@ urllib3==2.5.0
     #   elastic-apm
     #   requests
     #   sentry-sdk
-uwsgi==2.0.29
+uwsgi==2.0.31
     # via open-api-framework
 vine==5.1.0
     # via

requirements/ci.txt

@@ -663,7 +663,7 @@ urllib3==2.5.0
     #   requests
     #   sentry-sdk
     #   vcrpy
-uwsgi==2.0.29
+uwsgi==2.0.31
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt

requirements/dev.txt

@@ -814,7 +814,7 @@ urllib3==2.5.0
     #   requests
     #   sentry-sdk
     #   vcrpy
-uwsgi==2.0.29
+uwsgi==2.0.31
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt

src/objecttypes/conf/docker.py

@@ -1,9 +1,9 @@
 import os
 
 os.environ.setdefault("DB_HOST", "db")
-os.environ.setdefault("DB_NAME", "postgres")
-os.environ.setdefault("DB_USER", "postgres")
-os.environ.setdefault("DB_PASSWORD", "")
+os.environ.setdefault("DB_NAME", "objecttypes")
+os.environ.setdefault("DB_USER", "objecttypes")
+os.environ.setdefault("DB_PASSWORD", "objecttypes")
 os.environ.setdefault("DB_CONN_MAX_AGE", "60")
 
 os.environ.setdefault("ENVIRONMENT", "docker")