Skip to content

Commit 8c42294

Browse files
committed
Remove results.tar.gz for out-of-repo archives
Signed-off-by: Andrew Stein <steinlink@gmail.com>
1 parent 5801d42 commit 8c42294

7 files changed

Lines changed: 158 additions & 63 deletions

File tree

.github/workflows/build.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -646,9 +646,12 @@ jobs:
646646
path: .
647647

648648
- name: Run Tests
649-
run: pnpm run test
649+
run: pnpm run test -- --fetch-snapshots
650650
env:
651651
PACKAGE: "server,client,viewer,viewer-datagrid,viewer-charts,viewer-openlayers,workspace,react"
652+
PSP_SNAPSHOT_REPO: ${{ vars.PSP_SNAPSHOT_REPO }}
653+
PSP_SNAPSHOT_TOKEN: ${{ secrets.PSP_SNAPSHOT_TOKEN }}
654+
PSP_SNAPSHOT_REF: ${{ github.head_ref || github.ref_name }}
652655
# PSP_USE_CCACHE: 1
653656

654657
# ,--,--' . .-,--. . .

pnpm-lock.yaml

Lines changed: 0 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

tools/scripts/test_js.mjs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,25 @@ function playwright(pkg, is_jlab) {
5656
console.log(`-- Running ${pkg_name}Playwright test suite`);
5757
const args = process.argv
5858
.slice(2)
59-
.filter((x) => x !== "--ci" && x !== "--jupyter");
59+
.filter(
60+
(x) =>
61+
x !== "--ci" && x !== "--jupyter" && x !== "--fetch-snapshots",
62+
);
6063

6164
const env = { ...process.env, TZ: "UTC" };
6265
if (is_jlab) {
6366
env.PSP_JUPYTERLAB_TESTS = "1";
6467
env.__JUPYTERLAB_PORT__ = "6538";
6568
}
6669

70+
if (getarg("--fetch-snapshots")) {
71+
env.PSP_FETCH_SNAPSHOTS = "1";
72+
}
73+
74+
if (getarg("--update-snapshots")) {
75+
env.PSP_UPDATE_SNAPSHOTS = "1";
76+
}
77+
6778
if (IS_CI) {
6879
env.CI = "1";
6980
}

tools/test/package.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
"dependencies": {
2323
"react": "catalog:",
2424
"react-dom": "catalog:",
25-
"tar": "catalog:",
2625
"tsx": "catalog:",
2726
"prettier": "catalog:",
2827
"superstore-arrow": "catalog:",

tools/test/src/js/global_startup.ts

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,10 @@
1010
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
1111
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
1212

13-
import * as tar from "tar";
14-
import fs from "node:fs";
15-
import path from "node:path";
16-
import url from "node:url";
17-
18-
const __filename = url.fileURLToPath(import.meta.url);
19-
const __dirname = path.dirname(__filename);
13+
import { fetchSnapshots } from "./snapshot-sync.js";
2014

2115
export default async function run() {
22-
const RESULTS_PATH = path.join(__dirname, "../../results.tar.gz");
23-
const cwd = path.join(__dirname, "..", "..");
24-
if (fs.existsSync(RESULTS_PATH)) {
25-
console.log("Using results.tar.gz");
26-
await tar.extract({ file: RESULTS_PATH, gzip: true, cwd });
16+
if (process.env.PSP_FETCH_SNAPSHOTS) {
17+
await fetchSnapshots();
2718
}
2819
}

tools/test/src/js/global_teardown.ts

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -10,52 +10,10 @@
1010
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
1111
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
1212

13-
import * as tar from "tar";
14-
import fs from "fs";
15-
import path from "path";
16-
import url from "node:url";
17-
18-
import "zx/globals";
19-
20-
const __filename = url.fileURLToPath(import.meta.url);
21-
const __dirname = path.dirname(__filename);
22-
const RESULTS_PATH = path.join(__dirname, "../../results.tar.gz");
13+
import { writebackSnapshots } from "./snapshot-sync.js";
2314

2415
export default async function run() {
25-
if (fs.existsSync(RESULTS_PATH)) {
26-
console.log("\nReplacing results.tar.gz");
27-
} else {
28-
console.log("\nCreating results.tar.gz");
16+
if (process.env.PSP_UPDATE_SNAPSHOTS) {
17+
await writebackSnapshots();
2918
}
30-
31-
const cwd = path.join(__dirname, "..", "..");
32-
await new Promise((x) =>
33-
tar.create(
34-
{
35-
cwd,
36-
gzip: true,
37-
file: RESULTS_PATH,
38-
sync: false,
39-
portable: true,
40-
noMtime: true,
41-
strip: 2,
42-
filter: (path, stat) => {
43-
stat.mtime = null;
44-
stat.atime = null;
45-
stat.ctime = null;
46-
// stat.birthtime = null;
47-
return !path.endsWith(".DS_Store");
48-
},
49-
},
50-
[
51-
...glob.sync("dist/snapshots/**/*.txt", { cwd }),
52-
...glob.sync("dist/snapshots/**/*.html", { cwd }),
53-
// Image baselines for visual-regression specs. Without
54-
// these, CI has no Playwright PNG comparison target and
55-
// every `toHaveScreenshot` call fails.
56-
...glob.sync("dist/snapshots/**/*.png", { cwd }),
57-
],
58-
x,
59-
),
60-
);
6119
}

tools/test/src/js/snapshot-sync.ts

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
2+
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
3+
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
4+
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
5+
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
6+
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
7+
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
8+
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
9+
// ┃ This file is part of the Perspective library, distributed under the terms ┃
10+
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
11+
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
12+
13+
import { execFileSync } from "node:child_process";
14+
import fs from "node:fs";
15+
import path from "node:path";
16+
import url from "node:url";
17+
18+
const __filename = url.fileURLToPath(import.meta.url);
19+
const __dirname = path.dirname(__filename);
20+
21+
const TEST_ROOT = path.resolve(__dirname, "..", "..");
22+
const CACHE_DIR = path.join(TEST_ROOT, "dist", "git_snapshots");
23+
const DEST_DIR = path.join(TEST_ROOT, "dist", "snapshots");
24+
const DEFAULT_REF = "master";
25+
26+
function git(args: string[], cwd: string) {
27+
execFileSync("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
28+
}
29+
30+
function remoteHasRef(remoteUrl: string, ref: string): boolean {
31+
try {
32+
const out = execFileSync(
33+
"git",
34+
["ls-remote", "--heads", remoteUrl, ref],
35+
{ stdio: ["ignore", "pipe", "pipe"] },
36+
)
37+
.toString()
38+
.trim();
39+
return out.length > 0;
40+
} catch {
41+
return false;
42+
}
43+
}
44+
45+
function buildRemoteUrl(repo: string, token: string | undefined): string {
46+
if (token) {
47+
return `https://x-access-token:${token}@github.com/${repo}.git`;
48+
}
49+
return `git@github.com:${repo}.git`;
50+
}
51+
52+
function mirrorSnapshots(srcRoot: string, dest: string) {
53+
const srcSnapshots = path.join(srcRoot);
54+
if (!fs.existsSync(srcSnapshots)) {
55+
throw new Error(
56+
`Snapshot clone at ${srcRoot} does not contain dist/snapshots/`,
57+
);
58+
}
59+
60+
fs.rmSync(dest, { recursive: true, force: true });
61+
fs.mkdirSync(path.dirname(dest), { recursive: true });
62+
fs.cpSync(srcSnapshots, dest, { recursive: true });
63+
}
64+
65+
export async function fetchSnapshots(): Promise<void> {
66+
const repo = process.env.PSP_SNAPSHOT_REPO;
67+
if (!repo) {
68+
throw new Error(
69+
"PSP_SNAPSHOT_REPO is required when fetching snapshots (e.g. 'perspective-dev/perspective-snapshots').",
70+
);
71+
}
72+
73+
const token =
74+
process.env.PSP_SNAPSHOT_TOKEN || process.env.GITHUB_TOKEN || undefined;
75+
const requestedRef = process.env.PSP_SNAPSHOT_REF || DEFAULT_REF;
76+
const remoteUrl = buildRemoteUrl(repo, token);
77+
78+
let ref = requestedRef;
79+
if (
80+
requestedRef !== DEFAULT_REF &&
81+
!remoteHasRef(remoteUrl, requestedRef)
82+
) {
83+
console.log(
84+
`Snapshot branch '${requestedRef}' not found on ${repo}; falling back to '${DEFAULT_REF}'.`,
85+
);
86+
ref = DEFAULT_REF;
87+
}
88+
89+
const cacheGitDir = path.join(CACHE_DIR, ".git");
90+
if (fs.existsSync(cacheGitDir)) {
91+
try {
92+
git(["remote", "set-url", "origin", remoteUrl], CACHE_DIR);
93+
git(["fetch", "--depth", "1", "origin", ref], CACHE_DIR);
94+
git(["checkout", "-B", ref, "FETCH_HEAD"], CACHE_DIR);
95+
git(["reset", "--hard", "FETCH_HEAD"], CACHE_DIR);
96+
git(["clean", "-fdx"], CACHE_DIR);
97+
} catch {
98+
fs.rmSync(CACHE_DIR, { recursive: true, force: true });
99+
}
100+
}
101+
102+
if (!fs.existsSync(cacheGitDir)) {
103+
fs.mkdirSync(path.dirname(CACHE_DIR), { recursive: true });
104+
execFileSync(
105+
"git",
106+
[
107+
"clone",
108+
"--depth",
109+
"1",
110+
"--filter=blob:none",
111+
"--branch",
112+
ref,
113+
remoteUrl,
114+
CACHE_DIR,
115+
],
116+
{ stdio: ["ignore", "pipe", "pipe"] },
117+
);
118+
}
119+
120+
console.log(`Fetched snapshots from ${repo}@${ref}`);
121+
mirrorSnapshots(CACHE_DIR, DEST_DIR);
122+
}
123+
124+
export async function writebackSnapshots(): Promise<void> {
125+
if (!fs.existsSync(path.join(CACHE_DIR, ".git"))) {
126+
console.log(
127+
`No snapshot clone at ${CACHE_DIR}; skipping writeback. Run with --fetch-snapshots first to populate the cache.`,
128+
);
129+
return;
130+
}
131+
if (!fs.existsSync(DEST_DIR)) {
132+
return;
133+
}
134+
fs.cpSync(DEST_DIR, CACHE_DIR, { recursive: true, force: true });
135+
console.log(`Copied updated snapshots into ${CACHE_DIR}`);
136+
}

0 commit comments

Comments
 (0)