feat(delivery): support maximum backoff duration

.github/workflows/kind-e2e.yaml

@@ -3,34 +3,69 @@ name: KinD e2e tests
 on:
   push:
     branches: [ 'main', 'release-*' ]
+    paths:
+      - '**/*.go'
+      - 'go.mod'
+      - 'go.sum'
+      - 'config/**'
+      - 'test/**'
+      - 'hack/**'
+      - 'third_party/**'
+      - '.github/workflows/kind-e2e.yaml'
   pull_request:
     branches: [ 'main', 'release-*' ]
+    paths:
+      - '**/*.go'
+      - 'go.mod'
+      - 'go.sum'
+      - 'config/**'
+      - 'test/**'
+      - 'hack/**'
+      - 'third_party/**'
+      - '.github/workflows/kind-e2e.yaml'
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
 
 jobs:
 
-  ko-resolve:
-    name: e2e tests
+  e2e-tests:
+    name: e2e tests (${{ matrix.k8s-version }}, ${{ matrix.eventing-version }}, ${{ matrix.test-suite }})
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     strategy:
       fail-fast: false # Keep running if one leg fails.
       matrix:
         k8s-version:
-        - v1.34.3
+        - v1.34.x
+        - v1.35.x
 
         eventing-version:
-        - knative-v1.21.0
-
-        # Map between K8s and KinD versions.
-        # This is attempting to make it a bit clearer what's being tested.
-        # See: https://github.com/kubernetes-sigs/kind/releases/tag/v0.29.0
-        include:
-        - k8s-version: v1.34.3
-          kind-version: v0.31.0
-          kind-image-sha: sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48
+        # Exercise compatibility with a supported Eventing installation.
+        - knative-v1.23.0
+        # Build the Eventing revision selected by go.mod.
+        - go.mod
+
+        test-suite:
+        - ./test/e2e
+        - ./test/experimental
+
+        exclude:
+        # Experimental tests exercise the API selected by go.mod.
+        - eventing-version: knative-v1.23.0
+          test-suite: ./test/experimental
+
     env:
       KO_DOCKER_REPO: kind.local
       SYSTEM_NAMESPACE: knative-eventing
-      KIND_CLUSTER_NAME: kind
 
     steps:
     - name: Defaults
@@ -43,75 +78,61 @@ jobs:
       uses: knative/actions/setup-go@main
 
     - name: Install ko
-      uses: ko-build/setup-ko@v0.7
+      uses: ko-build/setup-ko@v0.10
 
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
 
-    - name: Install KinD
-      env:
-        KIND_VERSION: ${{ matrix.kind-version }}
+    - name: Resolve Knative Eventing dependency
+      if: matrix.eventing-version == 'go.mod'
+      id: eventing-dependency
       run: |
-        set -x
+        set -euo pipefail
+
+        eventing_version="$(go list -m -f '{{.Version}}' knative.dev/eventing)"
+        eventing_ref="$(
+          GOPROXY=direct go mod download -json "knative.dev/eventing@${eventing_version}" |
+            jq -er '.Origin.Hash'
+        )"
+        echo "ref=${eventing_ref}" >> "$GITHUB_OUTPUT"
+
+    - name: Check out Knative Eventing source
+      if: matrix.eventing-version == 'go.mod'
+      uses: actions/checkout@v7
+      with:
+        repository: knative/eventing
+        ref: ${{ steps.eventing-dependency.outputs.ref }}
+        path: .eventing-source
 
-        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${{ matrix.kind-version }}/kind-$(uname)-amd64
-        chmod +x ./kind
-        sudo mv kind /usr/local/bin
-
-    - name: Create KinD Cluster
-      run: |
-        set -x
+    - name: Install KinD
+      uses: chainguard-dev/actions/setup-kind@c69a264ec2a5934c3186c618f368fc1c86f16cff # main
+      with:
+        k8s-version: ${{ matrix.k8s-version }}
+        kind-worker-count: 1
+        cluster-suffix: cluster.local
 
-        # KinD configuration.
-        cat > kind.yaml <<EOF
-        apiVersion: kind.x-k8s.io/v1alpha4
-        kind: Cluster
-
-        # This is needed in order to support projected volumes with service account tokens.
-        # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
-        kubeadmConfigPatches:
-          - |
-            apiVersion: kubeadm.k8s.io/v1beta2
-            kind: ClusterConfiguration
-            metadata:
-              name: config
-            apiServer:
-              extraArgs:
-                "service-account-issuer": "kubernetes.default.svc"
-                "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
-        nodes:
-        - role: control-plane
-          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
-        - role: worker
-          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
-
-        EOF
-
-        # Create a cluster!
-        kind create cluster --config kind.yaml
-
-    - name: Install nats
+    - name: Install NATS
       run: |
         set -x
 
-        kubectl create namespace nats
-        kubectl apply -n nats-io -f ./config/mtbroker/natsjsm.yaml
+        kubectl apply -f ./config/mtbroker/natsjsm.yaml
         # Create a ConfigMap that we use to instruct Broker to create nats channels.
-        kubectl create namespace knative-eventing
-        kubectl apply -n knative-eventing -f ./config/mtbroker/config-br-default-channel-jsm.yaml
-        kubectl apply -n knative-eventing -f ./config/mtbroker/config-nats.yaml
-        
+        kubectl create namespace ${SYSTEM_NAMESPACE}
+        kubectl apply -n ${SYSTEM_NAMESPACE} -f ./config/mtbroker/config-br-default-channel-jsm.yaml
+        kubectl apply -n ${SYSTEM_NAMESPACE} -f ./config/mtbroker/config-nats.yaml
+
         set +x
         source ./vendor/knative.dev/hack/infra-library.sh
         wait_until_pods_running nats-io
-        
+
         set -x
         kubectl get pods -n nats-io
         kubectl get svc nats -n nats-io
         kubectl get pods -n kube-system -l k8s-app=kube-dns
         kubectl logs -n kube-system -l k8s-app=kube-dns
 
-    - name: Install Knative Eventing
+    - name: Install released Knative Eventing
+      if: matrix.eventing-version != 'go.mod'
       run: |
         set -x
 
@@ -120,73 +141,93 @@ jobs:
         kubectl apply --filename https://github.com/knative/eventing/releases/download/${{ matrix.eventing-version }}/eventing-core.yaml
         kubectl apply --filename https://github.com/knative/eventing/releases/download/${{ matrix.eventing-version }}/mt-channel-broker.yaml
 
-    - name: Install
+    - name: Build and install Knative Eventing source
+      if: matrix.eventing-version == 'go.mod'
+      working-directory: .eventing-source
+      run: |
+        set -x
+
+        ko apply --platform=linux/amd64 -R -f ./config/core/
+        ko apply --platform=linux/amd64 -f ./config/brokers/mt-channel-broker/
+
+    - name: Apply experimental features config
+      if: matrix.test-suite == './test/experimental'
+      run: |
+        kubectl apply -f ./test/experimental/config
+
+    - name: Install NATS Eventing components
       run: |
         set -x
         # TODO: this should use the release script and then apply the newly created release yaml in the future.
 
-        # Install the nats channel
-        ko apply -f ./config/webhook
-        ko apply -f ./config/jetstream
-        ko apply -f ./config/broker
+        ko apply --platform=linux/amd64 -f ./config/webhook
+        ko apply --platform=linux/amd64 -f ./config/jetstream
+        ko apply --platform=linux/amd64 -f ./config/broker
 
-    - name: Wait for Ready
+    - name: Wait for things to be up
       run: |
         set -e
         source ./vendor/knative.dev/hack/infra-library.sh
         wait_until_pods_running ${SYSTEM_NAMESPACE}
-        
+
         # For debugging.
         kubectl get pods --all-namespaces
 
     - name: Run e2e Tests
       run: |
-        set -x
-
         # Run the tests tagged as e2e on the KinD cluster.
-        go test -v -race -count=1 -timeout=15m -tags=e2e ./test/e2e/...
+        go test -race -count=1 -parallel=12 -timeout=90m -tags=e2e \
+          ${{ matrix.test-suite }}
 
-    - name: Gather Failure Data
+    - name: Collect system diagnostics
       if: ${{ failure() }}
       run: |
-        set -x
-
-        echo "===================== Brokers =============================="
+        echo '::group:: brokers'
         kubectl get broker --all-namespaces=true -oyaml
+        echo '::endgroup::'
 
-        echo "===================== Channels ============================="
+        echo '::group:: channels'
         kubectl get channel --all-namespaces=true -oyaml
+        echo '::endgroup::'
 
-        echo "===================== Triggers ============================="
+        echo '::group:: triggers'
         kubectl get trigger --all-namespaces=true -oyaml
+        echo '::endgroup::'
 
-        echo "===================== K8s Events ==========================="
-        kubectl get events --all-namespaces=true -oyaml
+        echo '::group:: all pods'
+        kubectl get pods --all-namespaces=true
+        echo '::endgroup::'
+
+        for namespace in ${SYSTEM_NAMESPACE} nats-io; do
+          echo "::group:: describe ${namespace} pods"
+          kubectl -n "${namespace}" describe pods || true
+          echo '::endgroup::'
+
+          for pod in $(kubectl -n "${namespace}" get pods -oname); do
+            echo "::group:: ${namespace} ${pod} logs"
+            kubectl -n "${namespace}" logs "${pod}" --all-containers || true
+            echo '::endgroup::'
 
-        echo "===================== Pod Logs ============================="
-        namespace=knative-eventing
-        for pod in $(kubectl get pod -n $namespace | awk '{print $1}'); do
-          for container in $(kubectl get pod "${pod}" -n $namespace -ojsonpath='{.spec.containers[*].name}'); do
-            echo "Namespace, Pod, Container: ${namespace}, ${pod}, ${container}"
-            kubectl logs -n $namespace "${pod}" -c "${container}" || true
-            echo "----------------------------------------------------------"
-            echo "Namespace, Pod, Container (Previous instance): ${namespace}, ${pod}, ${container}"
-            kubectl logs -p -n $namespace "${pod}" -c "${container}" || true
-            echo "============================================================"
+            echo "::group:: ${namespace} ${pod} previous logs"
+            kubectl -n "${namespace}" logs "${pod}" --all-containers --previous || true
+            echo '::endgroup::'
           done
         done
 
+        echo '::group:: all events'
+        kubectl get events --all-namespaces=true -oyaml
+        echo '::endgroup::'
+
     - name: Post failure notice to Slack
       # Note: using env.SLACK_WEBHOOK here because secrets are not allowed in the if block.
       if: ${{ env.SLACK_WEBHOOK != '' && failure() && github.event_name != 'pull_request' }}
-      uses: rtCamp/action-slack-notify@v2.1.0
+      uses: rtCamp/action-slack-notify@v2.4.0
       env:
-        SLACK_ICON: http://github.com/knative.png?size=48
+        SLACK_ICON: https://github.com/knative.png?size=48
         SLACK_USERNAME: github-actions
         SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         SLACK_CHANNEL: 'eventing-delivery'
         MSG_MINIMAL: 'true'
-        SLACK_TITLE: Periodic e2e for Nats on kind on (${{ matrix.k8s-version }}, ${{ matrix.eventing-version }}) failed.
+        SLACK_TITLE: Periodic e2e for NATS on KinD on (${{ matrix.k8s-version }}, ${{ matrix.eventing-version }}, ${{ matrix.test-suite }}) failed.
         SLACK_MESSAGE: |
           For detailed logs: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-

config/jetstream/302-jsm-channel.yaml

@@ -106,6 +106,9 @@ spec:
                     backoffDelay:
                       description: 'BackoffDelay is the delay before retrying. More information on Duration format: - https://www.iso.org/iso-8601-date-and-time-format.html - https://en.wikipedia.org/wiki/ISO_8601  For linear policy, backoff delay is backoffDelay*<numberOfRetries>. For exponential policy, backoff delay is backoffDelay*2^<numberOfRetries>.'
                       type: string
+                    backoffMax:
+                      description: 'BackoffMax is the maximum delay between normal delivery attempts. It caps the delay calculated from BackoffDelay and BackoffPolicy, but does not cap delays requested by a Retry-After response header. The value must be greater than zero. Cluster operators must enable the delivery-backoff-max feature before users can set this experimental field. More information on Duration format: - https://www.iso.org/iso-8601-date-and-time-format.html - https://en.wikipedia.org/wiki/ISO_8601'
+                      type: string
                     backoffPolicy:
                       description: BackoffPolicy is the retry backoff policy (linear, exponential).
                       type: string
@@ -740,6 +743,9 @@ spec:
                           backoffDelay:
                             description: 'BackoffDelay is the delay before retrying. More information on Duration format: - https://www.iso.org/iso-8601-date-and-time-format.html - https://en.wikipedia.org/wiki/ISO_8601  For linear policy, backoff delay is backoffDelay*<numberOfRetries>. For exponential policy, backoff delay is backoffDelay*2^<numberOfRetries>.'
                             type: string
+                          backoffMax:
+                            description: 'BackoffMax is the maximum delay between normal delivery attempts. It caps the delay calculated from BackoffDelay and BackoffPolicy, but does not cap delays requested by a Retry-After response header. The value must be greater than zero. Cluster operators must enable the delivery-backoff-max feature before users can set this experimental field. More information on Duration format: - https://www.iso.org/iso-8601-date-and-time-format.html - https://en.wikipedia.org/wiki/ISO_8601'
+                            type: string
                           backoffPolicy:
                             description: BackoffPolicy is the retry backoff policy (linear, exponential).
                             type: string

docs/broker.md

@@ -154,9 +154,42 @@ spec:
         apiVersion: v1
         kind: Service
         name: dead-letter-service
-    retry: 3
+    retry: 6
     backoffPolicy: exponential
     backoffDelay: PT1S
+    backoffMax: PT4S
+    retryAfterMax: PT6S
+```
+
+### Delivery retry limits
+
+The Trigger owner uses these fields to keep retry delays bounded when
+`my-service` is unavailable. With the example above, the normal exponential
+delays calculated by the Broker filter are `1s`, `2s`, `4s`, `4s`, `4s`, and
+`4s`.
+
+| Field | Delay it limits | When it applies | Example result |
+|-------|-----------------|-----------------|----------------|
+| `backoffMax` | The delay calculated from `backoffDelay` and `backoffPolicy` | Every normal retry | `PT4S` stops the exponential sequence at 4 seconds |
+| `retryAfterMax` | A delay requested by a subscriber's `Retry-After` response header | HTTP 429 and 503 responses | `PT6S` reduces `Retry-After: 10` to 6 seconds |
+
+For a 429 or 503 response, the Broker uses the larger of the normal backoff and
+the capped `Retry-After` value. These fields limit the delay requested from
+JetStream; scheduling and processing load can cause the next delivery to occur
+later.
+
+Cluster operators must enable both experimental fields in the Knative Eventing
+`config-features` ConfigMap before Trigger or Broker owners use this example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-features
+  namespace: knative-eventing
+data:
+  delivery-backoff-max: enabled
+  delivery-retryafter: enabled
 ```
 
 ## Configuration