tests: Update tests to be compatible with current staging (#1738)

* tests: Drop signing tests that use a localhost TSA

* We have some localhost TSA tests already that test TSA client specifically
* The removed signing tests are mostly redundant as the rekor v2 tests
  already use require the staging TSA to work
* The removed tests are fragile since they require a hard coded TrustConfig
  but still use most of the staging infra

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* Fix cross-version-verify test

* Replace 3.6.6 with 3.6.7 as the latest release in that branch
* Start testing on both prod and staging
* Skip 3.5.6, 4.0.0 and 4.1.0 on staging: They now fail because of
  #1656

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* cross-version-verify: Use bundle name from env var

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

---------

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

Author: jku

.github/workflows/cross-version-verify.yaml

@@ -42,8 +42,9 @@ jobs:
       - name: Sign 
         run: |
           touch artifact
-          python -m sigstore --staging sign --bundle artifact-rekor2.sigstore.json --identity-token $(cat oidc-token.txt) --rekor-version=2 artifact
-          python -m sigstore --staging sign --bundle artifact-rekor1.sigstore.json --identity-token $(cat oidc-token.txt) --rekor-version=1 artifact
+          python -m sigstore --staging sign --bundle artifact-staging-rekor2.sigstore.json --identity-token $(cat oidc-token.txt) --rekor-version=2 artifact
+          python -m sigstore --staging sign --bundle artifact-staging-rekor1.sigstore.json --identity-token $(cat oidc-token.txt) --rekor-version=1 artifact
+          python -m sigstore sign --bundle artifact-prod-rekor1.sigstore.json --identity-token $(cat oidc-token.txt) --rekor-version=1 artifact
       - name: upload signature bundle
         uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
@@ -52,13 +53,24 @@ jobs:
           if-no-files-found: error
           retention-days: 1
   verify:
-    name: Verify with ${{ matrix.version }}
+    name: Verify with ${{ matrix.version }} on ${{ matrix.env }}
     needs: [sign]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false # Don't cancel other jobs if one fails
       matrix:
-        version: [3.5.6, 3.6.6, 4.0.0, 4.1.0, 4.2.0]
+        # hand crafted list of old versions we care about
+        version: [3.5.6, 3.6.7, 4.0.0, 4.1.0, 4.2.0]
+        env: [staging, prod]
+        exclude:
+          # exclude staging for versions with https://github.com/sigstore/sigstore-python/issues/1656
+          - env: staging
+            version: 3.5.6
+          - env: staging
+            version: 4.0.0
+          - env: staging
+            version: 4.1.0
+
     steps:
       - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
@@ -69,15 +81,22 @@ jobs:
           name: bundle
       - run: touch artifact
       - name: Verify (Rekor v2)
-        if: startsWith(matrix.version, '3.') != true
+        # Rekor v2 is currently only available on staging, and only supported on sigstore-python 4.x
+        if: startsWith(matrix.version, '3.') != true && matrix.env == 'staging'
+        env:
+          ENV_OPT: ${{ matrix.env == 'staging' && '--staging' || '' }}
+          BUNDLE: artifact-${{matrix.env}}-rekor2.sigstore.json
         run: |
-          python -m sigstore --staging verify github --verbose \
+          python -m sigstore $ENV_OPT verify github --verbose \
             --cert-identity "https://github.com/sigstore-conformance/extremely-dangerous-public-oidc-beacon/.github/workflows/extremely-dangerous-oidc-beacon.yml@refs/heads/main" \
-            --bundle artifact-rekor2.sigstore.json \
+            --bundle $BUNDLE \
             artifact
       - name: Verify (Rekor v1)
+        env:
+          ENV_OPT: ${{ matrix.env == 'staging' && '--staging' || '' }}
+          BUNDLE: artifact-${{matrix.env}}-rekor1.sigstore.json
         run: |
-          python -m sigstore --staging verify github --verbose \
+          python -m sigstore $ENV_OPT verify github --verbose \
             --cert-identity "https://github.com/sigstore-conformance/extremely-dangerous-public-oidc-beacon/.github/workflows/extremely-dangerous-oidc-beacon.yml@refs/heads/main" \
-            --bundle artifact-rekor1.sigstore.json \
+            --bundle $BUNDLE \
             artifact

test/unit/test_sign.py

@@ -12,19 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import hashlib
-import logging
 import secrets
 
 import pretend
 import pytest
 from sigstore_models.common.v1 import HashAlgorithm
 
 import sigstore.oidc
-from sigstore._internal.timestamp import TimestampAuthorityClient
 from sigstore.dsse import StatementBuilder, Subject
 from sigstore.errors import VerificationError
 from sigstore.hashes import Hashed
-from sigstore.models import ClientTrustConfig
 from sigstore.sign import SigningContext
 from sigstore.verify.policy import UnsafeNoOp
 
@@ -179,79 +176,3 @@ def test_sign_dsse(staging):
         bundle = signer.sign_dsse(stmt)
         # Ensures that all of our inner types serialize as expected.
         bundle.to_json()
-
-
-@pytest.mark.staging
-@pytest.mark.ambient_oidc
-@pytest.mark.timestamp_authority
-class TestSignWithTSA:
-    @pytest.fixture
-    def sig_ctx(self, asset, tsa_url) -> SigningContext:
-        trust_config = ClientTrustConfig.from_json(
-            asset("tsa/trust_config.json").read_text()
-        )
-
-        trust_config._inner.signing_config.tsa_urls[0].url = tsa_url
-
-        return SigningContext.from_trust_config(trust_config)
-
-    @pytest.fixture
-    def identity(self, staging):
-        _, _, identity = staging
-        return identity
-
-    @pytest.fixture
-    def hashed(self) -> Hashed:
-        input_ = secrets.token_bytes(32)
-        return Hashed(
-            digest=hashlib.sha256(input_).digest(), algorithm=HashAlgorithm.SHA2_256
-        )
-
-    def test_sign_artifact(self, sig_ctx, identity, hashed):
-        with sig_ctx.signer(identity) as signer:
-            bundle = signer.sign_artifact(hashed)
-
-        assert bundle.to_json()
-        assert (
-            bundle.verification_material.timestamp_verification_data.rfc3161_timestamps
-        )
-
-    def test_sign_dsse(self, sig_ctx, identity):
-        stmt = (
-            StatementBuilder()
-            .subjects(
-                [
-                    Subject(
-                        name="null", digest={"sha256": hashlib.sha256(b"").hexdigest()}
-                    )
-                ]
-            )
-            .predicate_type("https://cosign.sigstore.dev/attestation/v1")
-            .predicate(
-                {
-                    "Data": "",
-                    "Timestamp": "2023-12-07T00:37:58Z",
-                }
-            )
-        ).build()
-
-        with sig_ctx.signer(identity) as signer:
-            bundle = signer.sign_dsse(stmt)
-
-        assert bundle.to_json()
-        assert (
-            bundle.verification_material.timestamp_verification_data.rfc3161_timestamps
-        )
-
-    def test_with_timestamp_error(self, sig_ctx, identity, hashed, caplog):
-        # Simulate here an TSA that returns an invalid Timestamp
-        sig_ctx._tsa_clients.append(TimestampAuthorityClient("invalid-url"))
-
-        with caplog.at_level(logging.WARNING, logger="sigstore.sign"):
-            with sig_ctx.signer(identity) as signer:
-                bundle = signer.sign_artifact(hashed)
-
-        assert caplog.records[0].message.startswith("Unable to use invalid-url")
-        assert (
-            bundle.verification_material.timestamp_verification_data.rfc3161_timestamps
-        )