test: take testutils and the compose files from mender-testkit

Drops the mender_server submodule. testutils, the Server facade and
mender-server's compose files now come from the mender-testkit package,
which vendors all three from a pinned mender-server commit.

conftest materialises the compose tree to tests/mender_server, the path the
submodule occupied, so tests/compose/*.yml keep working unchanged -- both
the include: paths and the project_directory that makes the relative bind
mounts resolve. Gitignored; regenerated per run. GATEWAY_HOSTNAME is set by
the package's pytest plugin, which loads before any conftest, so the
ordering dance around that import goes away.

CI no longer fetches submodules.

Depends on mender-testkit being installable; python-requirements.in still
needs it added once it is published.

Ticket: QA-1702
Signed-off-by: Rewan Rashid <rewan.rashid@northern.tech>

Author: rewanrashid-boop

.gitignore

@@ -34,3 +34,6 @@ venv
 # direnv Python virtualenv setup
 *.direnv
 *.envrc
+
+# Materialised from the mender-testkit package by tests/conftest.py
+tests/mender_server/

.gitlab-ci-default-pipeline.yml

@@ -7,7 +7,7 @@ include:
     file: '.gitlab-ci-check-python3-format.yml'
 
 variables:
-  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./(extra/gitdm|tests/mender_server).*'
+  LICENSE_HEADERS_IGNORE_FILES_REGEXP: '\./extra/gitdm.*'
 
 test:extra-tools:
   image: "python:3"

.gitlab-ci-full-integration-template.yml

@@ -16,11 +16,9 @@ test:integration:$CI_NODE_INDEX:
   # default budget.
   timeout: 2h
   variables:
-    # The backend composition comes from the mender-server submodule at
-    # tests/mender_server; without this the compose files are simply absent and
-    # every environment fails to start.
-    GIT_SUBMODULE_STRATEGY: recursive
-    GIT_SUBMODULE_DEPTH: 1
+    # No submodules needed: the backend composition arrives with the
+    # mender-testkit package, which conftest materialises before any test runs.
+    GIT_SUBMODULE_STRATEGY: none
   before_script:
     # These two variables would be set by GitLab CI "parallel" feature, and are used by our Pytest
     # plugin to split the tests among the CI jobs. They get substituted by the generator.
@@ -35,14 +33,10 @@ test:integration:$CI_NODE_INDEX:
     - apk add py3-virtualenv screen
     - python -m virtualenv /.venv
     - source /.venv/bin/activate
-    # Integration keeps its own testutils, so its own requirements file is the
-    # right one -- nothing here imports from the mender_server submodule, which
-    # is used for its compose files only.
+    # mender-testkit brings testutils, the Server facade and mender-server's
+    # compose files; this file carries everything else.
     - pip3 install -r ./tests/requirements-python/python-requirements.txt
     - pip3 install pyyaml
-    # Fail early and loudly if the submodule did not come through.
-    - test -f ./tests/mender_server/docker-compose.yml
-      || { echo "mender_server submodule missing"; exit 1; }
     # Gitlab CI tends to set these DOCKER_ variables internally.
     # dind also creates the unix socket at /var/run/docker.sock
     - unset DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "extra/gitdm/gitdm"]
 	path = extra/gitdm/gitdm
 	url = git://git.lwn.net/gitdm.git
-[submodule "tests/mender_server"]
-	path = tests/mender_server
-	url = https://github.com/mendersoftware/mender-server.git

pyproject.toml

@@ -23,7 +23,6 @@ exclude = '''
     | build
     | dist
     | extra/gitdm
-    | tests/mender_server
   )/
 )
 '''

tests/MenderAPI/authentication.py

@@ -21,7 +21,7 @@
 from . import get_container_manager
 from .requests_helpers import requests_retry
 
-from testutils.infra.cli import CliUseradm, CliTenantadm
+from mender_testkit.testutils.infra.cli import CliUseradm, CliTenantadm
 
 
 class Authentication:

tests/MenderAPI/deviceconnect.py

@@ -12,8 +12,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-from testutils.util import websockets
-from testutils.api import deviceconnect
+from mender_testkit.testutils.util import websockets
+from mender_testkit.testutils.api import deviceconnect
 from . import api_version
 from . import get_container_manager
 

tests/MenderAPI/requests_helpers.py

@@ -16,7 +16,7 @@
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
-from testutils.api.client import GATEWAY_HOSTNAME
+from mender_testkit.testutils.api.client import GATEWAY_HOSTNAME
 
 
 # Will retry on server errors (5xx)

tests/common_setup.py

@@ -20,9 +20,9 @@
 
 from .MenderAPI import authentication, auth, devauth, reset_mender_api, DeviceAuthV2
 
-from testutils.common import User, new_tenant_client
-from testutils.infra.cli import CliTenantadm
-from testutils.infra.device import MenderDevice, MenderDeviceGroup
+from mender_testkit.testutils.common import User, new_tenant_client
+from mender_testkit.testutils.infra.cli import CliTenantadm
+from mender_testkit.testutils.infra.device import MenderDevice, MenderDeviceGroup
 from .container_manager import factory
 
 container_factory = factory.get_factory()

tests/conftest.py

@@ -23,34 +23,28 @@
 
 import multiprocessing
 
-# testutils comes from the mender-server submodule -- this repo no longer keeps a
-# fork of it. Both of these have to happen before the first testutils import
-# below: the path so it resolves at all, and the hostname because
-# testutils.api.client reads it into a module-level constant at import time.
-#
-# Traefik's routers match on Host as well as path, and the tests address the
-# ingress by container IP, so every request has to carry this. Upstream defaults
-# it to "traefik", which is the name its own deployments answer to.
-sys.path.insert(
-    0,
-    os.path.join(
-        os.path.dirname(os.path.abspath(__file__)), "mender_server", "backend", "tests"
-    ),
-)
-os.environ.setdefault("GATEWAY_HOSTNAME", "docker.mender.io")
-
 import filelock
 import pytest
 from filelock import FileLock
-from testutils.infra.container_manager.base import BaseContainerManagerNamespace
-from testutils.infra.device import MenderDevice, MenderDeviceGroup
+from mender_testkit.compose import compose_dir
+from mender_testkit.testutils.infra.container_manager.base import (
+    BaseContainerManagerNamespace,
+)
+from mender_testkit.testutils.infra.device import MenderDevice, MenderDeviceGroup
 
 from . import log
 from .tests.mendertesting import MenderTesting
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 RELEASE_TOOL = os.path.join(THIS_DIR, "..", "extra", "release_tool.py")
 
+# mender-server's compose files now come from the mender-testkit package rather
+# than a submodule. They are written to the path the submodule used to occupy, so
+# that the include: directives in tests/compose/*.yml -- and the project_directory
+# they set, which is what makes the relative bind mounts resolve -- keep working
+# unchanged. Gitignored; regenerated on every run.
+compose_dir(dest=os.path.join(THIS_DIR, "mender_server"))
+
 logging.getLogger("requests").setLevel(logging.CRITICAL)
 logging.getLogger("paramiko").setLevel(logging.CRITICAL)
 logging.getLogger("urllib3").setLevel(logging.CRITICAL)
@@ -62,7 +56,7 @@
 
 machine_name = None
 
-collect_ignore = ["mender_server"]
+collect_ignore = ["mender_server"]  # the materialised compose tree, not test code
 
 
 def pytest_addoption(parser):