Skip to content

Commit 025451c

Browse files
csasarakcsasarak
authored
Revert "Replace plan-hash cache keys with file-hash and tag rotation … (#1675)
* Revert "Replace plan-hash cache keys with file-hash and tag rotation (#1666)" This reverts commit f190d48. * Remove old echo output from script.
1 parent d4f6b4d commit 025451c

4 files changed

Lines changed: 120 additions & 42 deletions

File tree

‎.github/workflows/bench.yml‎

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,18 @@ jobs:
77
runs-on: ubuntu-latest
88
container: fossa/haskell-static-alpine:ghc-9.8.4
99

10-
env:
11-
GHC_VERSION: '9.8.4'
12-
1310
steps:
1411
- uses: dtolnay/rust-toolchain@stable
1512

1613
- uses: actions/checkout@v6
1714
with:
1815
lfs: true
19-
fetch-tags: true
20-
21-
- name: Get latest release tag
22-
id: latest-tag
23-
run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
2416

2517
# Adding the "git config ..." line ensures git doesn't fail during our build.
26-
- name: Configure Git
27-
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
18+
- name: Configure Git and Collect Config
19+
run: |
20+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
21+
echo "parent_commit=$(git rev-parse HEAD^)" >> $GITHUB_ENV
2822
2923
3024
# adduser cannot add users to group: https://unix.stackexchange.com/a/397733
@@ -53,24 +47,46 @@ jobs:
5347
5448
- uses: Swatinem/rust-cache@v2
5549

56-
# Benchmarks build at a different optimization level, so they use
57-
# their own cache prefix to avoid collisions with the main build.
58-
- uses: actions/cache@v4
50+
# Compute cache key and save to a temporary file.
51+
#
52+
# We compute the cache key based on the solved install plan instead of just
53+
# hashing the `.cabal` file, since there are many kinds of changes that will
54+
# cause `.cabal` to change (e.g. adding new source modules).
55+
- name: Compute cache key
56+
id: compute-cache-key
57+
run: |
58+
cabal --project-file=cabal.project.ci.linux update
59+
cabal --project-file=cabal.project.ci.linux build --dry-run
60+
cat dist-newstyle/cache/plan.json | jq '."install-plan"[]."id"' | sort > /tmp/cabal-cache-key
61+
echo "Install plan:"
62+
cat /tmp/cabal-cache-key
63+
export CABAL_CACHE_KEY=$(sha256sum /tmp/cabal-cache-key | awk '{print $1}')
64+
echo "Cabal cache key: $CABAL_CACHE_KEY"
65+
echo "cabal-cache-key=$CABAL_CACHE_KEY" >> $GITHUB_OUTPUT
66+
67+
# Benchmarks builds at a different optimization level than everything else.
68+
# I think this means it needs its own cache, it can't rely on the others.
69+
# Otherwise, the benchmarks need to be built at the same optimization level.
70+
- uses: actions/cache@v3
5971
name: Cache cabal store
6072
with:
6173
path: ${{ steps.setup-haskell.outputs.cabal-store || '~/.local/state/cabal' }}
62-
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-cabal-cache-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
74+
key: ${{ runner.os }}-benchmarks-cabal-cache-${{ steps.compute-cache-key.outputs.cabal-cache-key }}
6375
restore-keys: |
64-
${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-cabal-cache-
76+
${{ runner.os }}-benchmarks-cabal-cache-
77+
${{ runner.os }}-benchmarks-
78+
${{ runner.os }}-
6579
66-
- uses: actions/cache@v4
80+
- uses: actions/cache@v3
6781
name: Cache dist-newstyle
6882
with:
6983
path: ${{ github.workspace }}/dist-newstyle
70-
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
84+
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ github.sha }}
7185
restore-keys: |
72-
${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-
73-
${{ runner.os }}-${{ env.GHC_VERSION }}-benchmarks-dist-newstyle-
86+
${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ env.parent_commit }}
87+
${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-
88+
${{ runner.os }}-${{ env.GHC_VERSION }}-
89+
${{ runner.os }}-
7490
7591
- name: Build Rust dependencies
7692
run: cargo build --release

‎.github/workflows/build-all.yml‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,6 @@ jobs:
5858
with:
5959
lfs: true
6060
fetch-depth: 2
61-
fetch-tags: true
62-
63-
- name: Get latest release tag
64-
id: latest-tag
65-
run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
6661

6762
- name: Install MacOS binary dependencies
6863
if: ${{ contains(matrix.os, 'macos') }}
@@ -122,29 +117,50 @@ jobs:
122117
- name: Ensure git ownership check does not lead to compiler error
123118
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
124119

125-
- name: Check git status
120+
- name: Check git status and collect information
126121
# https://github.com/actions/checkout/issues/760#issuecomment-1099519159
127-
run: git status --porcelain
122+
run: |
123+
git status --porcelain
124+
echo "parent_commit=$(git rev-parse HEAD^)" >> $GITHUB_ENV
125+
126+
# Compute cache key and save to a temporary file.
127+
#
128+
# We compute the cache key based on the solved install plan instead of just
129+
# hashing the `.cabal` file, since there are many kinds of changes that will
130+
# cause `.cabal` to change (e.g. adding new source modules).
131+
- name: Compute cache key (not ARM Linux)
132+
if: ${{ matrix.os != 'LinuxARM' }}
133+
id: compute-cache-key
134+
run: ./.github/workflows/scripts/compute_cache_key.sh ${{ runner.os }} ${{ matrix.project-file }}
135+
136+
- name: Compute Cache Key (Linux Arm)
137+
if: ${{ matrix.os == 'LinuxARM' }}
138+
id: compute-cache-key-arm
139+
uses: docker://fossa/haskell-static-alpine:ghc-9.8.4
140+
with:
141+
args: ./.github/workflows/scripts/compute_cache_key.sh ${{ runner.os }} ${{ matrix.project-file }}
128142

129143
# The home directory inside a github container run during a step is different than one run outside of it.
130144
# This is why there is special logic for 'LinuxARM'.
131-
# Its builds run inside a container but are cached by an action outside of it.
145+
# Its builds run inside a container but are cached by an action outside of it..
132146
- uses: actions/cache@v4
133147
name: Cache cabal store
134148
with:
135149
path: ${{ steps.setup-haskell.outputs.cabal-store || ( matrix.os == 'LinuxARM' && format('{0}/_github_home/.local/state/cabal', runner.temp) || '~/.local/state/cabal') }}
136-
key: ${{ matrix.os-name }}-${{ matrix.ghc }}-cabal-cache-${{ hashFiles('spectrometer.cabal', matrix.project-file, 'cabal.project.common') }}
150+
key: ${{ matrix.os-name }}-${{ matrix.ghc }}-cabal-cache-${{ steps.compute-cache-key.outputs.cabal-cache-key || steps.compute-cache-key-arm.outputs.cabal-cache-key }}
137151
restore-keys: |
138152
${{ matrix.os-name }}-${{ matrix.ghc }}-cabal-cache-
153+
${{ matrix.os-name }}-${{ matrix.ghc }}-
139154
140155
- uses: actions/cache@v4
141156
name: Cache dist-newstyle
142157
with:
143158
path: ${{ github.workspace }}/dist-newstyle
144-
key: ${{ matrix.os-name }}-${{ matrix.ghc }}-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-${{ hashFiles('spectrometer.cabal', matrix.project-file, 'cabal.project.common') }}
159+
key: ${{ matrix.os-name }}-${{ matrix.ghc }}-dist-newstyle-${{ github.sha }}
145160
restore-keys: |
146-
${{ matrix.os-name }}-${{ matrix.ghc }}-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-
161+
${{ matrix.os-name }}-${{ matrix.ghc }}-dist-newstyle-${{ env.parent_commit }}
147162
${{ matrix.os-name }}-${{ matrix.ghc }}-dist-newstyle-
163+
${{ matrix.os-name }}-${{ matrix.ghc }}-
148164
149165
- name: Update vendored binaries
150166
run: |

‎.github/workflows/integrations-test.yml‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,6 @@ jobs:
2525
with:
2626
lfs: true
2727
fetch-depth: 2
28-
fetch-tags: true
29-
30-
- name: Get latest release tag
31-
id: latest-tag
32-
run: echo "tag=$(git describe --tags --abbrev=0 2>/dev/null || echo none)" >> $GITHUB_OUTPUT
3328

3429
- name: Ensures git ownership check does not lead to compile error (we run git during compile for version tagging, etc.)
3530
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
@@ -60,26 +55,45 @@ jobs:
6055
6156
- uses: Swatinem/rust-cache@v2
6257

63-
# Cache cabal store and dist-newstyle keyed on spectrometer.cabal + project file.
64-
# cabal build handles staleness internally — if a transitive dep changed,
65-
# it recompiles only what's needed. This avoids the 8-min cabal update +
66-
# dry-run solver step that the plan-hash approach requires.
58+
# Compute cache key and save to a temporary file.
59+
#
60+
# We compute the cache key based on the solved install plan instead of just
61+
# hashing the `.cabal` file, since there are many kinds of changes that will
62+
# cause `.cabal` to change (e.g. adding new source modules).
63+
- name: Compute cache key
64+
id: compute-cache-key
65+
run: |
66+
cabal --project-file=cabal.project.ci.linux update
67+
cabal --project-file=cabal.project.ci.linux build --dry-run
68+
cat dist-newstyle/cache/plan.json | jq '."install-plan"[]."id"' | sort > /tmp/cabal-cache-key
69+
echo "Install plan:"
70+
cat /tmp/cabal-cache-key
71+
export CABAL_CACHE_KEY=$(sha256sum /tmp/cabal-cache-key | awk '{print $1}')
72+
echo "Cabal cache key: $CABAL_CACHE_KEY"
73+
echo "cabal-cache-key=$CABAL_CACHE_KEY" >> $GITHUB_OUTPUT
74+
75+
echo "parent_commit=$(git rev-parse HEAD^)" >> $GITHUB_ENV
76+
6777
- uses: actions/cache@v4
6878
name: Cache cabal store
6979
with:
7080
path: ${{ steps.setup-haskell.outputs.cabal-store || '~/.local/state/cabal' }}
71-
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
81+
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-${{ steps.compute-cache-key.outputs.cabal-cache-key }}
7282
restore-keys: |
7383
${{ runner.os }}-${{ env.GHC_VERSION }}-cabal-cache-
84+
${{ runner.os }}-${{ env.GHC_VERSION }}-
85+
${{ runner.os }}-
7486
7587
- uses: actions/cache@v4
7688
name: Cache dist-newstyle
7789
with:
7890
path: ${{ github.workspace }}/dist-newstyle
79-
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-${{ hashFiles('spectrometer.cabal', 'cabal.project.ci.linux', 'cabal.project.common') }}
91+
key: ${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ github.sha }}
8092
restore-keys: |
81-
${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ steps.latest-tag.outputs.tag }}-
93+
${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-${{ env.parent_commit }}
8294
${{ runner.os }}-${{ env.GHC_VERSION }}-dist-newstyle-
95+
${{ runner.os }}-${{ env.GHC_VERSION }}-
96+
${{ runner.os }}-
8397
8498
- name: Build Rust dependencies
8599
run: cargo build --release

‎.github/workflows/scripts/compute_cache_key.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env sh
2+
3+
set -e
4+
set -x
5+
6+
if [ $# -lt 2 ] ; then
7+
echo "Usage: ./compute_cache_key <RUNNER OS> <PROJECT FILE>"
8+
exit 1
9+
fi
10+
11+
RUNNER_OS=$1
12+
PROJECT_FILE=$2
13+
14+
cabal --project-file="$PROJECT_FILE" update
15+
cabal --project-file="$PROJECT_FILE" build --dry-run
16+
jq '."install-plan"[]."id"' < dist-newstyle/cache/plan.json | sort > /tmp/cabal-cache-key
17+
echo "Install plan:"
18+
cat /tmp/cabal-cache-key
19+
20+
if [ "$RUNNER_OS" = "macOS" ]; then
21+
PLAN_SUM=$(shasum -a256 /tmp/cabal-cache-key)
22+
else
23+
PLAN_SUM=$(sha256sum /tmp/cabal-cache-key)
24+
fi
25+
26+
CABAL_CACHE_KEY="$(echo "$PLAN_SUM" | awk '{print $1}')"
27+
export CABAL_CACHE_KEY
28+
echo "Cabal cache key: $CABAL_CACHE_KEY"
29+
echo "cabal-cache-key=$CABAL_CACHE_KEY" >> "$GITHUB_OUTPUT"
30+
31+
# Cleanup. Restoring this cache seems to fail if the directory already exists.
32+
rm -rf dist-newstyle

0 commit comments

Comments
 (0)