Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/reconcile-sweeps-live-previews.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
---

Corrects the nightly preview reconcile before pull request previews reach a release; nothing an
operator has deployed changes.
40 changes: 19 additions & 21 deletions .github/workflows/reconcile-previews.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,28 +26,26 @@ jobs:
outputs:
previews: ${{ steps.candidates.outputs.previews }}
steps:
- name: Find the preview environments GitHub still records
id: candidates
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
candidates="${RUNNER_TEMP}/preview-candidates.txt"
gh api --paginate "repos/${GITHUB_REPOSITORY}/deployments?task=deploy%3Apreview&per_page=100" \
--jq '.[] | .environment | select(test("^preview/pr-[1-9][0-9]*$")) | sub("^preview/pr-"; "")' \
| sort -nu > "${candidates}"
- name: Load the trusted preview controller
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
persist-credentials: false
sparse-checkout: |
.github/actions/setup-bun
package.json
scripts

count=$(grep -c . "${candidates}" || true)
# Truncate rather than abort: a backlog past the bound is exactly when this job is needed,
# so refusing to run would make the next night's backlog larger still.
if [ "${count}" -gt 100 ]; then
echo "::warning::Found ${count} preview candidates; reconciling the oldest 100 this run."
head -100 "${candidates}" > "${candidates}.capped"
mv "${candidates}.capped" "${candidates}"
fi
previews=$(jq -Rsc 'split("\n") | map(select(length > 0) | tonumber)' "${candidates}")
echo "previews=${previews}" >> "${GITHUB_OUTPUT}"
echo "Found ${count} preview candidates."
- name: Set up the repository's Bun version
uses: ./.github/actions/setup-bun

- name: Find the previews that still hold a slot
id: candidates
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const controller = await import(`${process.env.GITHUB_WORKSPACE}/scripts/preview-controller.ts`);
await controller.inventory({ github, context, core });

cleanup:
name: "Preview / Reconcile PR ${{ matrix.pr }}"
Expand Down
19 changes: 19 additions & 0 deletions scripts/preview-controller.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, it } from "node:test";

import {
assess,
inventory,
TEARDOWN_REQUESTED_DESCRIPTION,
create,
finalize,
Expand Down Expand Up @@ -290,6 +291,24 @@ void describe("preview host capacity", () => {
assert.equal(core.outputs.get("eligible"), "true");
});

void it("sweeps only the previews that still hold a slot", async () => {
const core = makeCore();
await inventory({
github: makeGitHub({
deployments: occupants(3),
// pr-102's teardown was recorded, so re-sending its close event would ask Coolify to
// remove a stack that is already gone — and would spend the sweep's bound doing it.
statuses: {
102: [{ description: TEARDOWN_REQUESTED_DESCRIPTION, state: "inactive" }],
},
}),
context: makeContext(),
core,
});

assert.equal(core.outputs.get("previews"), JSON.stringify([100, 101]));
});

void it("still counts a preview whose cleanup was never verified", async () => {
const core = makeCore();
await resolve({
Expand Down
26 changes: 26 additions & 0 deletions scripts/preview-controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ const TRUSTED_ASSOCIATIONS = new Set(["COLLABORATOR", "MEMBER", "OWNER"]);
// GitHub's comparison endpoint reports at most this many files and gives no truncation flag.
const COMPARE_FILE_LIMIT = 300;
const DEFAULT_MAX_ACTIVE = 3;

/** One night's sweep. Reached only if teardown has been failing, which is when a bound matters. */
const RECONCILE_LIMIT = 100;
const LIVE_STATES = new Set(["in_progress", "pending", "queued", "success"]);
const TEARDOWN_REQUESTED_DESCRIPTION =
"Preview teardown requested; awaiting Coolify reconciliation.";
Expand Down Expand Up @@ -415,6 +418,28 @@ const assess = async ({ github, context, core }: ControllerInput): Promise<void>
core.setOutput("base_ref", context.payload.repository.default_branch);
};

/**
* Candidates for the nightly sweep: the environments admission still believes are occupied. A
* preview whose teardown was already recorded holds nothing, so re-sending its close event would
* ask Coolify to remove a stack that is gone — every night, for every preview ever deployed. Worse,
* the sweep is bounded, and a list that only grows would fill that bound with previews already dealt
* with, leaving a genuinely leaked one unreached.
*/
const inventory = async ({ github, context, core }: ControllerInput): Promise<void> => {
const { owner, repo } = context.repo;
const occupied = await occupiedEnvironments(github, owner, repo);
const numbers = occupied
.map((environment) => Number(environment.slice("preview/pr-".length)))
.filter((number) => Number.isSafeInteger(number) && number > 0)
.toSorted((left, right) => left - right);
core.setOutput("previews", JSON.stringify(numbers.slice(0, RECONCILE_LIMIT)));
if (numbers.length > RECONCILE_LIMIT) {
core.notice(
`Found ${numbers.length} previews still holding a slot; reconciling the oldest ${RECONCILE_LIMIT}.`,
);
}
};

const retire = async ({ github, context }: ControllerInput): Promise<void> => {
const { owner, repo } = context.repo;
await retireDeployments(github, owner, repo, requiredEnv(process.env, "ENVIRONMENT"));
Expand All @@ -425,6 +450,7 @@ export {
PREVIEW_LABEL,
assess,
create,
inventory,
finalize,
inactivate,
recheck,
Expand Down
Loading