test: take testutils and the compose files from mender-testkit

Drops the mender_server submodule. testutils, the Server facade and
mender-server's compose files now come from the mender-testkit package,
which vendors all three from a pinned mender-server commit.

conftest materialises the compose tree to tests/mender_server, the path the
submodule occupied, so tests/compose/*.yml keep working unchanged -- both
the include: paths and the project_directory that makes the relative bind
mounts resolve. Gitignored; regenerated per run. GATEWAY_HOSTNAME is set by
the package's pytest plugin, which loads before any conftest, so the
ordering dance around that import goes away.

CI no longer fetches submodules.

Depends on mender-testkit being installable; python-requirements.in still
needs it added once it is published.

Ticket: QA-1702
Signed-off-by: Rewan Rashid <rewan.rashid@northern.tech>

Author: rewanrashid-boop

.gitignore

@@ -34,3 +34,6 @@ venv
 # direnv Python virtualenv setup
 *.direnv
 *.envrc
+
+# Materialised from the mender-testkit package by tests/conftest.py
+tests/mender_server/

.gitlab-ci-default-pipeline.yml

@@ -7,7 +7,7 @@ include:
     file: '.gitlab-ci-check-python3-format.yml'
 
 variables:
-  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./(extra/gitdm|tests/mender_server).*'
+  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./extra/gitdm.*'
 
 test:extra-tools:
   image: "python:3"

.gitlab-ci-full-integration-template.yml

@@ -16,11 +16,9 @@ test:integration:$CI_NODE_INDEX:
   # default budget.
   timeout: 2h
   variables:
-    # The backend composition comes from the mender-server submodule at
-    # tests/mender_server; without this the compose files are simply absent and
-    # every environment fails to start.
-    GIT_SUBMODULE_STRATEGY: recursive
-    GIT_SUBMODULE_DEPTH: 1
+    # No submodules needed: the backend composition arrives with the
+    # mender-testkit package, which conftest materialises before any test runs.
+    GIT_SUBMODULE_STRATEGY: none
   before_script:
     # These two variables would be set by GitLab CI "parallel" feature, and are used by our Pytest
     # plugin to split the tests among the CI jobs. They get substituted by the generator.
@@ -35,14 +33,10 @@ test:integration:$CI_NODE_INDEX:
     - apk add py3-virtualenv screen
     - python -m virtualenv /.venv
     - source /.venv/bin/activate
-    # Integration keeps its own testutils, so its own requirements file is the
-    # right one -- nothing here imports from the mender_server submodule, which
-    # is used for its compose files only.
+    # mender-testkit brings testutils, the Server facade and mender-server's
+    # compose files; this file carries everything else.
     - pip3 install -r ./tests/requirements-python/python-requirements.txt
     - pip3 install pyyaml
-    # Fail early and loudly if the submodule did not come through.
-    - test -f ./tests/mender_server/docker-compose.yml
-      || { echo "mender_server submodule missing"; exit 1; }
     # Gitlab CI tends to set these DOCKER_ variables internally.
     # dind also creates the unix socket at /var/run/docker.sock
     - unset DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "extra/gitdm/gitdm"]
 	path = extra/gitdm/gitdm
 	url = git://git.lwn.net/gitdm.git
-[submodule "tests/mender_server"]
-	path = tests/mender_server
-	url = https://github.com/mendersoftware/mender-server.git

pyproject.toml

@@ -23,7 +23,6 @@ exclude = '''
     | build
     | dist
     | extra/gitdm
-    | tests/mender_server
   )/
 )
 '''

renovate.json5

@@ -1,36 +1,46 @@
+// renovate.json5 - Mender dependency update configuration for integration
+// See Documentation/dependency-updates.md in mender-qa for the process.
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+
   "extends": [
     "config:recommended",
     ":gitSignOff",
     ":rebaseStalePrs"
   ],
-  "reviewers": ["team:qa-dependabot-reviewers"],
 
-  "packageRules": [
-    {
-      // TODO: remove some of these when we ditch dependabot
-      "matchCategories": ["python", "docker", "ci"],
-      "enabled": false
-    },
-    {
-      // The backend composition lives in this submodule, so keeping it current is
-      // how integration picks up new/changed services. Grouped and labelled so the
-      // bump is obvious: it can change the topology under the whole test suite.
-      "matchManagers": ["git-submodules"],
-      "matchDepNames": ["tests/mender_server"],
-      "enabled": true,
-      "groupName": "mender-server compose files",
-      "commitMessageTopic": "mender-server submodule",
-      "schedule": ["before 6am on monday"],
-    },
-  ],
+  // CODEOWNERS drives reviewer assignment - keep its dependency file entries current.
+  "reviewersFromCodeOwners": true,
 
-  "git-submodules": {
+  // fix(deps): <desc>, per renovate.json5.sample in mender-qa.
+  "semanticCommits": "enabled",
+  "semanticCommitType": "fix",
+  "semanticCommitScope": "deps",
+
+  // Fallback window for weeks with no merges; the renovate CI job is the primary trigger.
+  "schedule": ["after 10pm on Monday", "before 6am on Tuesday"],
+
+  // Was prHourlyLimit 0 (unlimited) while Dependabot still owned most ecosystems.
+  "prHourlyLimit": 2,
+  "prConcurrentLimit": 5,
+
+  // master plus 3.8.x, which is still live (last commit 2026-06-15). The older
+  // maintenance branches are dormant - 3.7.x last moved 2025-03, 3.6.x 2024-08,
+  // 3.5.x 2023-04 - so they are left out.
+  "baseBranchPatterns": ["master", "3.8.x"],
+
+  // Security PRs bypass the schedule and automerge when CI is green.
+  "vulnerabilityAlerts": {
     "enabled": true,
+    "schedule": ["at any time"],
+    "automerge": true,
+    "automergeType": "pr",
+    "labels": ["security"]
   },
 
+  // MENDER_ARTIFACT_VERSION is pinned in two places in this repo.
   "customManagers": [
+    // .gitlab-ci.yml, as a variable block with value:
     {
       "customType": "regex",
       "managerFilePatterns": ["/.gitlab-ci.yml/"],
@@ -42,6 +52,7 @@
       "extractVersionTemplate": "^v?(?<version>.*)$",
       "versioningTemplate": "semver"
     },
+    // .env, as a plain assignment
     {
       "customType": "regex",
       "managerFilePatterns": ["/.env/"],
@@ -52,8 +63,53 @@
       "datasourceTemplate": "github-releases",
       "extractVersionTemplate": "^v?(?<version>.*)$",
       "versioningTemplate": "semver"
-    },
+    }
   ],
 
-  "prHourlyLimit": 0
+  "packageRules": [
+    // config:recommended pulls in :semanticPrefixFixDepsChoreOthers, which forces
+    // semanticCommitType back to "chore" for any dep whose manager does not tag it
+    // with an npm/pip style depType - that is every docker and compose dep here.
+    // Must stay BEFORE the gitlabci rule: rules apply in order and the last match
+    // wins, so gitlabci keeps its own type.
+    {
+      "matchPackageNames": ["*"],
+      "semanticCommitType": "fix"
+    },
+
+    // CI image bumps get ci: instead of fix(deps):
+    {
+      "matchManagers": ["gitlabci", "gitlabci-include"],
+      "semanticCommitType": "ci",
+      "semanticCommitScope": ""
+    },
+
+    // Python: one PR for tests/requirements-python and extra/requirements.txt.
+    {
+      "matchManagers": ["pip_requirements"],
+      "groupName": "python-dependencies",
+      "groupSlug": "python-dependencies"
+    },
+
+    // 17 docker-compose files at the root, so group them into one PR. Only literal
+    // image tags are tracked - the ${MENDER_SERVER_*} ones are release tooling's job.
+    {
+      "matchManagers": ["docker-compose"],
+      "groupName": "docker-compose",
+      "groupSlug": "docker-compose"
+    },
+
+    // extra/gitdm/gitdm is a third-party submodule Dependabot never updated either.
+    {
+      "matchManagers": ["git-submodules"],
+      "enabled": false
+    },
+
+    // No major bumps on the maintenance branch - that needs a backport decision.
+    {
+      "matchBaseBranches": ["/^\\d+\\.\\d+\\.x$/"],
+      "matchUpdateTypes": ["major"],
+      "enabled": false
+    }
+  ]
 }

tests/MenderAPI/authentication.py

@@ -21,7 +21,7 @@
 from . import get_container_manager
 from .requests_helpers import requests_retry
 
-from testutils.infra.cli import CliUseradm, CliTenantadm
+from mender_testkit.testutils.infra.cli import CliUseradm, CliTenantadm
 
 
 class Authentication:

tests/MenderAPI/deviceconnect.py

@@ -12,8 +12,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-from testutils.util import websockets
-from testutils.api import deviceconnect
+from mender_testkit.testutils.util import websockets
+from mender_testkit.testutils.api import deviceconnect
 from . import api_version
 from . import get_container_manager
 

tests/MenderAPI/requests_helpers.py

@@ -16,7 +16,7 @@
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
-from testutils.api.client import GATEWAY_HOSTNAME
+from mender_testkit.testutils.api.client import GATEWAY_HOSTNAME
 
 
 # Will retry on server errors (5xx)

tests/common_setup.py

@@ -20,9 +20,9 @@
 
 from .MenderAPI import authentication, auth, devauth, reset_mender_api, DeviceAuthV2
 
-from testutils.common import User, new_tenant_client
-from testutils.infra.cli import CliTenantadm
-from testutils.infra.device import MenderDevice, MenderDeviceGroup
+from mender_testkit.testutils.common import User, new_tenant_client
+from mender_testkit.testutils.infra.cli import CliTenantadm
+from mender_testkit.testutils.infra.device import MenderDevice, MenderDeviceGroup
 from .container_manager import factory
 
 container_factory = factory.get_factory()