Added delta-sync history based tests

client/src/cbltest/api/couchbaseserver.py

@@ -357,12 +357,16 @@ def restore_bucket(
         dataset_name: str,
         *,
         repo_name: str | None = None,
+        reset_expired_ttl: bool = False,
     ) -> None:
         """
         Restores a bucket from a backup source
 
         :param name: The name of the bucket to restore
         :param backup_source: The path to the backup source
+        :param reset_expired_ttl: When True, restore already-expired documents
+            with no expiry (``--replace-ttl expired --replace-ttl-with 0``) so
+            they are not purged on access.
         """
         with self.__tracer.start_as_current_span(
             "restore_bucket",
@@ -396,24 +400,29 @@ def restore_bucket(
                         f"Backup zip '{data_filepath}' is invalid: {e}"
                     ) from e
 
-                subprocess.run(
-                    [
-                        cbbackupmgr_path,
-                        "restore",
-                        "-a",
-                        str(extract_path / dataset_name),
-                        "-c",
-                        self.__hostname,
-                        "-r",
-                        repo_name or dataset_name,
-                        "-u",
-                        self.__username,
-                        "-p",
-                        self.__password,
-                        "--auto-create-buckets",
-                    ],
-                    check=True,
-                )
+                restore_args = [
+                    cbbackupmgr_path,
+                    "restore",
+                    "-a",
+                    str(extract_path / dataset_name),
+                    "-c",
+                    self.__hostname,
+                    "-r",
+                    repo_name or dataset_name,
+                    "-u",
+                    self.__username,
+                    "-p",
+                    self.__password,
+                    "--auto-create-buckets",
+                ]
+                if reset_expired_ttl:
+                    restore_args += [
+                        "--replace-ttl",
+                        "expired",
+                        "--replace-ttl-with",
+                        "0",
+                    ]
+                subprocess.run(restore_args, check=True)
 
     def indexes_count(self, bucket: str) -> int:
         """

client/src/cbltest/api/syncgateway.py

@@ -703,6 +703,24 @@ async def bytes_transferred(self, dataset_name: str) -> tuple[int, int]:
         doc_writes_bytes = db_stats["doc_writes_bytes_blip"]
         return doc_reads_bytes, doc_writes_bytes
 
+    async def get_delta_sync_stats(self, dataset_name: str) -> dict:
+        """
+        Gets the ``delta_sync`` counters for a database from ``GET /_expvar``.
+        Returns an empty dict if the section is absent.
+
+        :param dataset_name: The name of the SGW database to inspect.
+        """
+        resp_data = await self._send_request("get", "/_expvar")
+        assert isinstance(resp_data, dict)
+        expvars = cast(dict, resp_data)
+
+        try:
+            db_section = expvars["syncgateway"]["per_db"][dataset_name]
+        except KeyError:
+            return {}
+        delta = db_section.get("delta_sync")
+        return delta if isinstance(delta, dict) else {}
+
     async def _put_database(
         self, db_name: str, payload: PutDatabasePayload, retry_count: int = 0
     ) -> None:
@@ -763,8 +781,8 @@ async def _delete_database(self, db_name: str, retry_count: int = 0) -> None:
                     current_span.add_event("SGW returned 500, retry")
                     await asyncio.sleep(2)
                     await self._delete_database(db_name, retry_count + 1)
-                elif e.code == 403:
-                    pass
+                elif e.code == 403 or e.code == 404:
+                    pass  # Database doesn't exist anyway.
                 else:
                     raise
 

pyproject.toml

@@ -33,10 +33,11 @@ lint = [
 members = [ "client" ]
 
 [tool.ty.environment]
-root = [".", "client/src"]
+root = [".", "client/src", "tests"]
 
 [tool.pytest.ini_options]
 asyncio_default_fixture_loop_scope = "session"
+pythonpath = ["tests"]
 filterwarnings = [
   "ignore:Class property max_ttl is deprecated.*:couchbase.logic.supportability.CouchbaseDeprecationWarning",
 ]

spec/tests/QE/test_replication_upgrade_delta_sync.md

@@ -0,0 +1,128 @@
+# Test Cases
+
+These tests cover delta-sync PULL replication across a simulated 3.x → 4.x SGW
+upgrade. The pre-upgrade state is materialised by restoring the prebuilt
+`upgrade` CBS backup (revtree-only docs, no HLV xattrs) and resetting the CBL
+local DB from the matching `upgrade` cblite2 file; both binaries run 4.x
+throughout. Delta sync is enabled on the SGW `upgrade` database.
+
+The bucket is restored with expired old-revision backup bodies (`_sync:rev:*`)
+re-enabled (their captured TTL has long lapsed), so SGW can compute a delta
+against a legacy ancestor revision instead of falling back to a full body.
+
+Both tests assert that SGW actually sends the revision **as a delta** (via the
+per-rev `deltas_sent` expvar counter), and that the document converges on the
+client. The SGW delta-sync `history`-field defect is a wire-level issue that is
+not observable from end-state and is verified separately; asserting on it from
+the test is tracked as future work.
+
+## #1 test_delta_sync_history_pull_post_upgrade_sgw_mutation
+
+### Description
+
+PULL replication of a doc whose new revision is created on 4.x SGW (so SGW
+holds both a revtree leaf and an HLV) by a client that holds the **direct
+revtree parent**. Uses doc `nonconflict_3`; the new revision is created in-test
+by mutating the doc on 4.x SGW, which adds an HLV in parallel with the new
+revtree leaf.
+
+```
++-------------------+-------------------------------+-------------------------------+
+|                   |             CBL               |              SGW              |
+|                   +---------------+---------------+---------------+---------------+
+|                   |   Rev Tree    |      HLV      |   Rev Tree    |      HLV      |
++-------------------+---------------+---------------+---------------+---------------+
+| After restore     |     2-abc     |     none      |     2-abc     |     none      |
+| After SGW mutate  |     2-abc     |     none      | 3-xxx, 2-abc  |   [N@SGW]     |
+| Expected post-PULL|     none      |   [N@SGW]     | 3-xxx, 2-abc  |   [N@SGW]     |
++-------------------+---------------+---------------+---------------+---------------+
+```
+
+### Steps
+
+1. Delete Sync Gateway 'upgrade' database if exists.
+2. Restore Couchbase Server Bucket using `upgrade` dataset, re-enabling expired
+   old-revision backup bodies.
+3. Wait 2s to ensure SG picks up the restored database.
+4. Reset local database, and load `upgrade` dataset.
+5. Create SG 'upgrade' database with delta_sync enabled and import from bucket.
+   On 412 (already exists), force-recreate by `delete_database` + `put_database`.
+6. Verify delta_sync is actually enabled on SGW 'upgrade' database.
+7. Create user `user1` with full access to `_default._default`.
+8. Mutate `nonconflict_3` on 4.x SGW to create a new revtree leaf + HLV.
+9. Start a replicator:
+   * endpoint: `/upgrade`
+   * collections: `_default._default`
+   * type: pull
+   * document_ids: `['nonconflict_3']`
+   * continuous: False
+   * credentials: user1/pass
+10. Wait until the replicator is stopped.
+11. Check that the doc is replicated correctly.
+12. Validate revid and HLV of local and remote doc:
+    * Pre: local has revid + no HLV; SGW has revid + canonical (non-RTE) HLV.
+    * Post: local has no revid (4.x CBL is HLV-only); local HLV equals SGW HLV.
+13. Confirm SGW sent the revision as a delta (`deltas_sent` incremented).
+
+### Expected Outcome
+
+CBL ingests the delta and ends up HLV-only with an HLV matching SGW; the
+`deltas_sent` counter confirms a delta (not a full body) was sent.
+
+---
+
+## #2 test_delta_sync_history_pull_pre_upgrade_sgw_two_revs
+
+### Description
+
+PULL replication of a **legacy, pre-upgrade** second revision. Uses doc
+`nonconflict_2`: after restore the client is at rev 1 and SGW is at rev 2, both
+revtree-only with **no HLV** (rev 2 was created pre-upgrade on 3.x, so it never
+got an HLV). There is **no in-test mutation**. SGW sends the existing rev 2 as a
+**revID-identified (legacy) delta** computed against the client's rev 1 — which
+requires rev 1's old-revision backup body, made available by the TTL rescue at
+restore. This is the distinguishing case from #1: the delta is of a revtree-only
+rev with no HLV, not a 4.x HLV-bearing rev.
+
+```
++-------------------+-------------------------------+-------------------------------+
+|                   |             CBL               |              SGW              |
+|                   +---------------+---------------+---------------+---------------+
+|                   |   Rev Tree    |      HLV      |   Rev Tree    |      HLV      |
++-------------------+---------------+---------------+---------------+---------------+
+| After restore     |     1-abc     |     none      | 2-def, 1-abc  |     none      |
+| Expected post-PULL| 2-def, 1-abc  |     none      | 2-def, 1-abc  |     none      |
++-------------------+---------------+---------------+---------------+---------------+
+```
+
+### Steps
+
+1. Delete Sync Gateway 'upgrade' database if exists.
+2. Restore Couchbase Server Bucket using `upgrade` dataset, re-enabling expired
+   old-revision backup bodies.
+3. Wait 2s to ensure SG picks up the restored database.
+4. Reset local database, and load `upgrade` dataset.
+5. Create SG 'upgrade' database with delta_sync enabled and import from bucket.
+   On 412 (already exists), force-recreate by `delete_database` + `put_database`.
+6. Verify delta_sync is actually enabled on SGW 'upgrade' database.
+7. Create user `user1` with full access to `_default._default`.
+8. Start a replicator:
+   * endpoint: `/upgrade`
+   * collections: `_default._default`
+   * type: pull
+   * document_ids: `['nonconflict_2']`
+   * continuous: False
+   * credentials: user1/pass
+9. Wait until the replicator is stopped.
+10. Check that the doc is replicated correctly.
+11. Validate revid and HLV of local and remote doc:
+    * Pre: both sides have revid and no HLV; local revid < SGW revid.
+    * Post: local and SGW share the same revid; neither has an HLV (the legacy
+      rev carries no HLV and PULL doesn't touch SGW).
+12. Confirm SGW sent the revision as a delta (`deltas_sent` incremented).
+
+### Expected Outcome
+
+SGW sends rev 2 as a revID-identified legacy delta (no HLV) computed against the
+client's rev 1; CBL applies it and ends up revtree-only at rev 2; the
+`deltas_sent` counter confirms a delta (not a full body) was sent.