Junit functional upload

client/pyproject.toml

@@ -27,6 +27,7 @@ dependencies = [
     "couchbase==4.3.4",
     "cryptography==44.0.0",
     "deepdiff==8.6.1",
+    "junitparser==4.0.2",
     "netifaces==0.11.0",
     "opentelemetry-api==1.29.0",
     "opentelemetry-exporter-otlp-proto-common==1.29.0",

client/src/cbltest/greenboarduploader.py

@@ -9,11 +9,55 @@
 from couchbase.auth import PasswordAuthenticator
 from couchbase.cluster import Cluster
 from couchbase.options import ClusterOptions
+from junitparser import JUnitXml
+from pydantic import BaseModel, ConfigDict, Field
 
 from cbltest.api.syncgateway import CouchbaseVersion
 from cbltest.logging import cbl_info, cbl_warning
 
 
+def count_from_junit_xml(xml_path: Path) -> tuple[int, int] | None:
+    """Return ``(passed, failed)`` summed across every ``<testsuite>`` in
+    a JUnit XML file at ``xml_path``, or ``None`` if the file is missing or
+    can't be parsed.
+
+    ``passed = tests - failures - errors - skipped``; ``failed`` lumps
+    failures and errors together (the greenboard doc only carries a single
+    fail bucket). Used by the greenboard pytest fixture to derive upload
+    counts from pytest's ``--junitxml`` output instead of in-process
+    hook-driven counters.
+    """
+    xml = JUnitXml.fromfile(str(xml_path))
+
+    total_pass = 0
+    total_fail = 0
+    suites = list(xml)
+    for suite in suites:
+        tests = suite.tests or 0
+        failures = suite.failures or 0
+        errors = suite.errors or 0
+        skipped = suite.skipped or 0
+        total_pass += max(0, tests - failures - errors - skipped)
+        total_fail += failures + errors
+    return total_pass, total_fail
+
+
+class RunResult(BaseModel):
+    """
+    Store the information for a test run in greenboard
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    build: int  # build number of CBL build
+    version: str  # major.minor.patch version of CBL
+    sgw_version: str = Field(alias="sgwVersion")  # Sync Gateway version, optional
+    fail_count: int = Field(alias="failCount")  # number of failing tests
+    pass_count: int = Field(alias="passCount")  # number of passing tests
+    platform: str  # CBL platform
+    os: str  # Operating system for CBL
+
+
 class GreenboardUploader:
     """
     A class for uploading results to a specified greenboard server bucket.
@@ -63,6 +107,62 @@ def pytest_runtest_makereport(self, item: pytest.Item, call: pytest.CallInfo[Non
         elif report.failed:
             self.__fail_count += 1
 
+    def upload_from_junit_file(
+        self,
+        junit_output: Path,
+        platform: str,
+        os_name: str,
+        version: str,
+        sgw_version: CouchbaseVersion | None,
+    ) -> None:
+        """
+        Upload one greenboard doc whose pass/fail counts come from a JUnit
+        XML file produced by pytest.
+        Policy:
+        - If ``junit_output`` doesn't exist, fall back to the in-process
+          counter (``self.upload()`` without count overrides). This covers
+          the cold-start case where pytest never wrote an XML — e.g.
+          ``--junitxml`` was explicitly disabled by the caller.
+        - If the file exists but reports zero tests, skip the upload (the
+          session collected nothing worth recording).
+        - If it reports zero passes (a fully-red run), skip the upload per
+          the project's "don't post all-failed runs to greenboard" policy.
+        - Otherwise, upload with the JUnit-derived counts.
+
+        Parse errors propagate — a malformed JUnit XML is a real bug and
+        should fail the Jenkins job loudly.
+        """
+        if not junit_output.is_file():
+            # Pytest didn't write an XML for this session; use the in-process
+            # counter populated by pytest_runtest_makereport instead.
+            self.upload(platform, os_name, version, sgw_version)
+            return
+
+        counts = count_from_junit_xml(junit_output)
+        assert counts is not None, f"Failed to parse JUnit XML at {junit_output}"
+        junit_pass, junit_fail = counts
+        if junit_pass + junit_fail == 0:
+            cbl_info(
+                f"Greenboard: JUnit XML at {junit_output} reports zero tests; "
+                "skipping upload"
+            )
+            return
+        if junit_pass == 0:
+            cbl_info(
+                f"Greenboard: all tests failed (failCount={junit_fail}); "
+                "skipping upload per policy"
+            )
+            return
+
+        self.upload(
+            platform,
+            os_name,
+            version,
+            sgw_version,
+            pass_count=junit_pass,
+            fail_count=junit_fail,
+        )
+
     def has_sgw_marker(self) -> bool:
         """
         Returns True if any test in the session has @pytest.mark.sgw or @pytest.mark.upg_sgw marker
@@ -75,6 +175,9 @@ def upload(
         os_name: str,
         version: str,
         sgw_version: CouchbaseVersion | None,
+        *,
+        pass_count: int | None = None,
+        fail_count: int | None = None,
     ):
         """
         Uploads the results using the specified platform and version.  The reason that they
@@ -84,11 +187,21 @@ def upload(
         :param platform: The platform name (e.g. couchbase-lite-net) as specified by the test server
         :param version: The version string (e.g. 3.2.0-b0136, etc) as specified by the test server
         :param sgw_version: The parsed SGW CouchbaseVersion object, or None if unavailable
+        :param pass_count: Optional override for the pass count. When ``None``
+            (default), uses the in-process counter populated by
+            :py:meth:`pytest_runtest_makereport`. When provided (e.g. the
+            greenboard fixture derived counts from a JUnit XML), the supplied
+            value is used instead.
+        :param fail_count: Optional override for the fail count. Same
+            semantics as ``pass_count``.
         """
         if self.__overall_fail:
             cbl_warning("Overall result is failure, skipping upload...")
             return
 
+        resolved_pass = pass_count if pass_count is not None else self.__pass_count
+        resolved_fail = fail_count if fail_count is not None else self.__fail_count
+
         parsed_version = "0.0.0"
         parsed_build = 0
         sgw_version_str = "n/a"
@@ -122,15 +235,15 @@ def upload(
                     cbl_warning(f"Could not parse build number from '{version}'")
 
         self._upload_document(
-            {
-                "build": parsed_build,
-                "version": parsed_version,
-                "sgwVersion": sgw_version_str,
-                "failCount": self.__fail_count,
-                "passCount": self.__pass_count,
-                "platform": platform,
-                "os": os_name,
-            }
+            RunResult(
+                build=parsed_build,
+                version=parsed_version,
+                sgwVersion=sgw_version_str,
+                failCount=resolved_fail,
+                passCount=resolved_pass,
+                platform=platform,
+                os=os_name,
+            )
         )
 
     def record_upgrade_step(
@@ -288,7 +401,7 @@ def upload_upgrade_batch(self, results_file: str) -> None:
             # `build` fields preserve what was actually running at each step.
             target_build = 0
 
-        self._upload_document(
+        self._upsert(
             {
                 "build": target_build,
                 "version": target_version,
@@ -306,13 +419,17 @@ def upload_upgrade_batch(self, results_file: str) -> None:
             f"failedAt={failed_at}"
         )
 
-    def _upload_document(self, doc: dict) -> None:
-        """Upload a document to the greenboard bucket with common fields added."""
+    def _upload_document(self, test_run: RunResult) -> None:
+        self._upsert(test_run.model_dump(by_alias=True))
+
+    def _upsert(self, doc: dict) -> None:
+        """Add timestamp fields and write one document to the greenboard bucket."""
         now = datetime.now(timezone.utc)
         unix_timestamp = (
             now - datetime(1970, 1, 1, tzinfo=timezone.utc)
         ).total_seconds()
 
+        # Do not add to RunResult since this code will go away shortly
         doc["uploaded"] = unix_timestamp
         doc["date"] = now.strftime("%Y-%m-%d")
 

client/src/cbltest/plugins/greenboard_fixture.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 import pytest_asyncio
@@ -96,10 +97,14 @@ async def greenboard(cblpytest: CBLPyTest, pytestconfig: pytest.Config):
                 if "systemName" in test_server_info.device:
                     os_name = test_server_info.device["systemName"]
             if len(cblpytest.sync_gateways) > 0:
-                sgw_version = await cblpytest.sync_gateways[0].get_version()
-            uploader.upload(test_platform, os_name, library_version, sgw_version)
-    except Exception as e:
-        cbl_warning(f"Failed to upload results to Greenboard: {e}")
+                try:
+                    sgw_version = await cblpytest.sync_gateways[0].get_version()
+                except Exception as e:
+                    cbl_warning(f"Could not fetch SGW version for greenboard doc: {e}")
+            xmlpath = pytestconfig.option.xmlpath
+            uploader.upload_from_junit_file(
+                Path(xmlpath), test_platform, os_name, library_version, sgw_version
+            )
     finally:
         pytestconfig.pluginmanager.unregister(uploader)
 
@@ -120,3 +125,21 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         "(e.g. '3.3.0,4.0.1,4.1.0'). First is the baseline, rest are upgrade "
         "targets. Triggers sgw-upgrade platform upload.",
     )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    """Default ``--junitxml=junit_result.xml`` so the greenboard fixture's
+    session-finish step can read pass/fail counts from the XML.
+
+    Doing this in code (instead of via ``addopts`` in ``client/pyproject.toml``)
+    is necessary because pytest's rootdir discovery walks up from the cwd and
+    typically picks the repo-root ``pyproject.toml`` for production runs from
+    ``tests/QE`` or ``tests/dev_e2e`` — never reaching ``client/pyproject.toml``.
+    The plugin's entry-point registration guarantees this hook fires on every
+    pytest invocation that imports ``cbltest``.
+
+    Users can override with ``--junitxml=<path>`` on the CLI; pytest's
+    last-wins behavior leaves the explicit flag in charge.
+    """
+    if not getattr(config.option, "xmlpath", None):
+        config.option.xmlpath = "junit_result.xml"