feat(ci): two-step vagrant tests reusing a configurator export

.gitlab-ci.yml

@@ -19,6 +19,7 @@ stages:
   - build_artifacts
   - sign
   - publish
+  - pretest
   - test
   - deploy
   - build_pf_img
@@ -322,21 +323,28 @@ variables:
 # .test_branches_only_rules into a single template. For feature-branch runs
 # (not devel, not maintenance, no tag) it emits VAGRANT_COMMON_DOTFILE_PATH
 # as a rule variable so the job uses the devel-compatible Vagrant common state.
+# On devel/maintenance it overrides BOX_NAME to pfdeb12dev (flat bucket layout).
 # NOTE: TEST_ONLY filtering is done in .test_script_job via shell script
 # because GitLab CI doesn't expand variables inside regex patterns.
 .test_rules:
   rules:
-    # Run on devel branch (push, web, not schedule)
+    # Run on devel branch (push, web, not schedule) — use flat pfdeb12dev box
     - if: '$CI_COMMIT_REF_NAME == "devel" && $CI_PIPELINE_SOURCE != "schedule" && ( $TEST == "yes" || $CI_COMMIT_MESSAGE =~ /test=yes/ || $TEST_ONLY != "" )'
-    # Run on maintenance branches (web/api only)
+      variables:
+        BOX_NAME: pfdeb12dev
+    # Run on maintenance branches (web/api only) — use flat pfdeb12dev box
     - if: '$CI_COMMIT_REF_NAME =~ /^maintenance\/[[:digit:]]+\.[[:digit:]]+$/ && $CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api") && ( $TEST == "yes" || $CI_COMMIT_MESSAGE =~ /test=yes/ || $TEST_ONLY != "" )'
-    # Run on devel/maintenance with schedule
+      variables:
+        BOX_NAME: pfdeb12dev
+    # Run on devel/maintenance with schedule — use flat pfdeb12dev box
     - if: '( $CI_COMMIT_REF_NAME == "devel" || $CI_COMMIT_REF_NAME =~ /^maintenance\/[[:digit:]]+\.[[:digit:]]+$/) && $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_TAG == null && ( $TEST == "yes" || $CI_COMMIT_MESSAGE =~ /test=yes/ || $TEST_ONLY != "" )'
-    # Run on feature branches (web/api) — emit VAGRANT_COMMON_DOTFILE_PATH override
+      variables:
+        BOX_NAME: pfdeb12dev
+    # Run on feature branches (web/api) — pfdeb12branch box + VAGRANT_COMMON_DOTFILE_PATH override
     - if: '$CI_COMMIT_REF_NAME != "devel" && $CI_COMMIT_REF_NAME !~ /^maintenance\/[[:digit:]]+\.[[:digit:]]+$/ && $CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api") && ( $TEST == "yes" || $CI_COMMIT_MESSAGE =~ /test=yes/ || $TEST_ONLY != "" )'
       variables:
         VAGRANT_COMMON_DOTFILE_PATH: /var/local/gitlab-runner/vagrant/vagrant-common-devel
-    # Run on feature branches via schedule — emit VAGRANT_COMMON_DOTFILE_PATH override
+    # Run on feature branches via schedule — pfdeb12branch box + VAGRANT_COMMON_DOTFILE_PATH override
     - if: '$CI_COMMIT_REF_NAME != "devel" && $CI_COMMIT_REF_NAME !~ /^maintenance\/[[:digit:]]+\.[[:digit:]]+$/ && $CI_COMMIT_TAG == null && $CI_PIPELINE_SOURCE == "schedule" && ( $TEST == "yes" || $CI_COMMIT_MESSAGE =~ /test=yes/ || $TEST_ONLY != "" )'
       variables:
         VAGRANT_COMMON_DOTFILE_PATH: /var/local/gitlab-runner/vagrant/vagrant-common-devel
@@ -574,6 +582,43 @@ variables:
   script:
     - make -e build_deb
 
+# Gating templates: downstream test scenarios that previously ran the
+# `configurator` venom suite as a prep step now import the matching arch's
+# exportable backup. Each consumer extends the matching template so its
+# job only starts after the producer has uploaded.
+.needs_configurator_deb12:
+  needs:
+    - job: configurator_deb12
+      artifacts: false
+
+.needs_configurator_el8:
+  needs:
+    - job: configurator_el8
+      artifacts: false
+
+# Producer template: run the configurator scenario, then export a
+# PacketFence backup tarball (addons/exportable-backup.sh) to Linode only
+# if the suite passes. Applied to both configurator_deb12 and
+# configurator_el8 — the bucket key is suffixed with the arch (last
+# underscore-segment of CI_JOB_NAME) so deb12 and el8 don't collide.
+# Downstream test jobs import the matching tarball instead of running
+# configurator again.
+# Runs in `pretest` (before the test stage) — listed last in each producer's
+# extends so this stage overrides the `stage: test` from .test_job.
+.configurator_producer_job:
+  stage: pretest
+  script:
+    # Producer always runs when .test_rules selects the pipeline (TEST/TEST_ONLY
+    # gate lives there). Both arches are baked unconditionally; cheaper than
+    # parsing TEST_ONLY tokens to decide which export downstream needs.
+    - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${TESTDIR} MAKE_TARGET=run ${CI_JOB_NAME} || EXIT_CODE=$?
+    # Export only on success; capture upload failure so teardown still runs.
+    - |
+      if [ -z "${EXIT_CODE:-}" ]; then
+        ${VAGRANT_LIB_DIR}/upload-pf-export.sh || EXIT_CODE=$?
+      fi
+    # Teardown always runs (get_logs + destroy VM); job still fails via JOB_STATUS.
+    - JOB_STATUS=${EXIT_CODE:-} timeout ${PIPELINE_TIMEOUT_CLEANUP} ${TESTCIDIR}/clean-test-environment.sh
 
 .test_script_job:
   variables:
@@ -1055,6 +1100,7 @@ unit_tests_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 unit_tests_deb12:
   variables:
@@ -1063,6 +1109,7 @@ unit_tests_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 configurator_el8:
   variables:
@@ -1071,6 +1118,7 @@ configurator_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .configurator_producer_job
 
 configurator_deb12:
   variables:
@@ -1079,6 +1127,7 @@ configurator_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .configurator_producer_job
 
 pfappserver_el8:
   variables:
@@ -1103,6 +1152,7 @@ dot1x_eap_peap_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 dot1x_eap_peap_deb12:
   variables:
@@ -1111,6 +1161,7 @@ dot1x_eap_peap_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 mac_auth_el8:
   variables:
@@ -1119,6 +1170,7 @@ mac_auth_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 mac_auth_deb12:
   variables:
@@ -1127,6 +1179,7 @@ mac_auth_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 dot1x_eap_tls_el8:
   variables:
@@ -1135,6 +1188,7 @@ dot1x_eap_tls_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 dot1x_eap_tls_deb12:
   variables:
@@ -1143,6 +1197,7 @@ dot1x_eap_tls_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 fingerbank_invalid_db_deb12:
   variables:
@@ -1151,6 +1206,7 @@ fingerbank_invalid_db_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 security_events_deb12:
   variables:
@@ -1159,6 +1215,7 @@ security_events_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 cli_login_deb12:
   variables:
@@ -1167,6 +1224,7 @@ cli_login_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 cli_login_el8:
   variables:
@@ -1175,6 +1233,7 @@ cli_login_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 external_integrations_deb12:
   variables:
@@ -1183,6 +1242,7 @@ external_integrations_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 captive_portal_el8:
   variables:
@@ -1191,6 +1251,7 @@ captive_portal_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 captive_portal_deb12:
   variables:
@@ -1199,6 +1260,7 @@ captive_portal_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 inline_deb12:
   variables:
@@ -1207,6 +1269,7 @@ inline_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 inline_el8:
   variables:
@@ -1215,6 +1278,7 @@ inline_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 mac_auth_virtualswitch_el8:
   variables:
@@ -1223,6 +1287,7 @@ mac_auth_virtualswitch_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 mac_auth_virtualswitch_deb12:
   variables:
@@ -1231,6 +1296,7 @@ mac_auth_virtualswitch_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 device_profiling_virtualswitch_el8:
   variables:
@@ -1239,6 +1305,7 @@ device_profiling_virtualswitch_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 device_profiling_virtualswitch_deb12:
   variables:
@@ -1247,6 +1314,7 @@ device_profiling_virtualswitch_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 dot1x_wired_computer_auth_virtualswitch_el8:
   variables:
@@ -1255,6 +1323,7 @@ dot1x_wired_computer_auth_virtualswitch_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 dot1x_wired_computer_auth_virtualswitch_deb12:
   variables:
@@ -1263,6 +1332,7 @@ dot1x_wired_computer_auth_virtualswitch_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 pfacct_bandwidth_virtualswitch_el8:
   variables:
@@ -1271,6 +1341,7 @@ pfacct_bandwidth_virtualswitch_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 pfacct_bandwidth_virtualswitch_deb12:
   variables:
@@ -1279,6 +1350,7 @@ pfacct_bandwidth_virtualswitch_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 security_event_virtualswitch_el8:
   variables:
@@ -1287,6 +1359,7 @@ security_event_virtualswitch_el8:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_el8
 
 security_event_virtualswitch_deb12:
   variables:
@@ -1295,6 +1368,7 @@ security_event_virtualswitch_deb12:
     - .test_job
     - .test_script_job
     - .test_rules
+    - .needs_configurator_deb12
 
 ### release
 configurator_el8_pristine:
@@ -1453,15 +1527,17 @@ build_pf_img_vagrant_maintenance_debian_bookworm:
     - if: '$CI_COMMIT_REF_NAME =~ /^maintenance\/[[:digit:]]+\.[[:digit:]]+$/ && $CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api")'
 
 # Feature branches: packages come from the pipeline PPA (publish_ppa); all
-# feature branches share the same Linode prefix, the pipeline ID (BOX_VERSION)
-# distinguishes builds and lets a user pin a specific pipeline's box.
+# feature branches share the same Linode prefix. Each branch uploads under its
+# own <BOX_NAME>/<CI_COMMIT_REF_SLUG>/ sub-prefix via REF_SLUG=<slug>; the
+# pipeline ID (BOX_VERSION) versions builds and lets a user pin a specific one.
 build_pf_img_vagrant_branch_debian_bookworm:
   extends:
     - .build_pf_img_vagrant_job
   script:
     - export PF_MINOR_RELEASE="$(perl -ne 'print $1 if (m/.*?(\d+\.\d+)/)' conf/pf-release)"
     - export BOX_VERSION="${PF_MINOR_RELEASE}-ci-${CI_PIPELINE_ID}"
     - export BOX_NAME="pfdeb12branch"
+    - export REF_SLUG="${CI_COMMIT_REF_SLUG}"
     - export BOX_DESC="PacketFence ${PF_MINOR_RELEASE} - ${CI_COMMIT_REF_SLUG} - ${BOX_VERSION}"
     - export CI_PAGES_URL="http://inverse.ca/downloads/PacketFence/gitlab/${CI_PIPELINE_ID}"
     - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${VAGRANT_IMG_DIR}

addons/full-import/export.sh

@@ -55,8 +55,35 @@ if echo "$last_db_dump" | grep '\.sql.gz$' >/dev/null; then
     mariadb_args="$mariadb_args -p$mariadb_root_pass"
   fi
 
+  db_name=$(/usr/local/pf/bin/get_pf_conf database db)
+  db_name=${db_name:-pf}
+
   echo "Database dump uses mysqldump. Exporting the grants from the database. Enter the MariaDB root password if prompted to"
   mysql $mariadb_args --skip-column-names -A -e"SELECT CONCAT('SHOW GRANTS FOR ''',user,'''@''',host,''';') FROM mysql.user WHERE user<>'' AND user<>'PUBLIC'" | mysql $mariadb_args --skip-column-names -A | sed 's/$/;/g' > grants.sql
+
+  # pf-user dump omits triggers (no TRIGGER privilege); dump them as root.
+  # Routines are skipped: the base pf dump already carries them.
+  echo "Exporting the triggers from the database"
+  mysqldump $mariadb_args --single-transaction --no-create-info --no-data --skip-add-drop-table \
+    --triggers "$db_name" > triggers.sql
+
+  # The nightly dump omits large history tables via --ignore-table, but triggers
+  # write to them (e.g. the locationlog trigger inserts into locationlog_history),
+  # so export their structure only to recreate the empty tables on import.
+  # Keep this list in sync with the --ignore-table list in backup-and-maintenance.sh;
+  # only tables that exist are dumped, so obsolete names (pre-10.3) are ignored.
+  excluded_tables="locationlog_history iplog_archive"
+  existing_excluded=""
+  for t in $excluded_tables; do
+    if [ -n "$(mysql $mariadb_args -N -B -e "SHOW TABLES LIKE '$t'" "$db_name" 2>/dev/null)" ]; then
+      existing_excluded="$existing_excluded $t"
+    fi
+  done
+  if [ -n "$existing_excluded" ]; then
+    echo "Exporting the structure of the tables excluded from the dump:$existing_excluded"
+    mysqldump $mariadb_args --no-data --skip-triggers --skip-add-drop-table \
+      "$db_name" $existing_excluded > excluded_tables_schema.sql
+  fi
 fi
 
 main_splitter