Skip to content

Commit c6c22eb

Browse files
s3rj1ks3rj1k
committed
Add s3 artifact upload via rclone 
1 parent 5405734 commit c6c22eb

2 files changed

Lines changed: 234 additions & 2 deletions

File tree

‎.github/actions/rclone-s3-sync/action.yml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Rclone S3 Sync
2+
description: Sync files between local and S3 or between S3 locations using rclone
3+
4+
inputs:
5+
SRC:
6+
required: true
7+
description: "Source path (local directory or S3 path in bucket)"
8+
DST:
9+
required: true
10+
description: "Destination path (local directory or S3 path in bucket)"
11+
SYNC_TYPE:
12+
required: false
13+
default: "local-to-remote"
14+
description: "Sync type: local-to-remote, remote-to-local, or remote-to-remote"
15+
S3_BUCKET:
16+
required: true
17+
description: "S3 bucket name"
18+
S3_PROVIDER:
19+
required: false
20+
default: "aws"
21+
description: "S3 provider type"
22+
S3_REGION:
23+
required: false
24+
default: "us-east-2"
25+
description: "AWS S3 region"
26+
S3_LOCATION_CONSTRAINT:
27+
required: false
28+
default: "us-east-2"
29+
description: "AWS S3 location constraint"
30+
S3_STORAGE_CLASS:
31+
required: false
32+
default: ""
33+
description: "AWS S3 storage class"
34+
35+
runs:
36+
using: "composite"
37+
steps:
38+
- name: Detect architecture
39+
id: arch
40+
shell: bash
41+
run: |
42+
case "$(uname -m)" in
43+
x86_64) echo "arch=amd64" >> $GITHUB_OUTPUT ;;
44+
aarch64|arm64) echo "arch=arm64" >> $GITHUB_OUTPUT ;;
45+
armv7l) echo "arch=arm-v7" >> $GITHUB_OUTPUT ;;
46+
armv6l) echo "arch=arm-v6" >> $GITHUB_OUTPUT ;;
47+
arm*) echo "arch=arm" >> $GITHUB_OUTPUT ;;
48+
*) echo "Unsupported architecture: $(uname -m)"; exit 1 ;;
49+
esac
50+
51+
- name: Install dependencies
52+
shell: bash
53+
run: sudo apt-get update && sudo apt-get install -y curl unzip
54+
55+
- name: Download rclone
56+
shell: bash
57+
run: curl -O https://downloads.rclone.org/rclone-current-linux-${{ steps.arch.outputs.arch }}.zip
58+
59+
- name: Extract rclone
60+
shell: bash
61+
run: unzip rclone-current-linux-${{ steps.arch.outputs.arch }}.zip
62+
63+
- name: Install rclone binary
64+
shell: bash
65+
run: |
66+
cd rclone-*-linux-${{ steps.arch.outputs.arch }}
67+
sudo cp rclone /usr/bin/
68+
sudo chown root:root /usr/bin/rclone
69+
sudo chmod 755 /usr/bin/rclone
70+
71+
- name: Verify rclone installation
72+
shell: bash
73+
run: rclone version
74+
75+
- name: Cleanup installation files
76+
shell: bash
77+
run: rm -rf rclone-*-linux-${{ steps.arch.outputs.arch }}*
78+
79+
- name: Create rclone config directory
80+
if: ${{ inputs.S3_PROVIDER == 'aws' }}
81+
shell: bash
82+
run: mkdir -p ~/.config/rclone
83+
84+
- name: Create base S3 configuration
85+
if: ${{ inputs.S3_PROVIDER == 'aws' }}
86+
shell: bash
87+
run: |
88+
cat > ~/.config/rclone/rclone.conf << EOF
89+
[s3]
90+
type = s3
91+
provider = AWS
92+
env_auth = true
93+
region = ${{ inputs.S3_REGION }}
94+
no_check_bucket = true
95+
EOF
96+
97+
- name: Add location constraint to config
98+
if: ${{ inputs.S3_PROVIDER == 'aws' && inputs.S3_LOCATION_CONSTRAINT != '' }}
99+
shell: bash
100+
run: echo "location_constraint = ${{ inputs.S3_LOCATION_CONSTRAINT }}" >> ~/.config/rclone/rclone.conf
101+
102+
- name: Add storage class to config
103+
if: ${{ inputs.S3_PROVIDER == 'aws' && inputs.S3_STORAGE_CLASS != '' }}
104+
shell: bash
105+
run: echo "storage_class = ${{ inputs.S3_STORAGE_CLASS }}" >> ~/.config/rclone/rclone.conf
106+
107+
- name: Verify rclone configuration
108+
if: ${{ inputs.S3_PROVIDER == 'aws' }}
109+
shell: bash
110+
run: rclone config show s3
111+
112+
- name: Sanitize paths
113+
id: sanitized
114+
shell: bash
115+
run: |
116+
SRC_CLEAN=$(python3 -c "import os; path=os.path.normpath('${{ inputs.SRC }}'); print(path.lstrip('/'))")
117+
DST_CLEAN=$(python3 -c "import os; path=os.path.normpath('${{ inputs.DST }}'); print(path.lstrip('/'))")
118+
119+
echo "src_clean=$SRC_CLEAN" >> $GITHUB_OUTPUT
120+
echo "dst_clean=$DST_CLEAN" >> $GITHUB_OUTPUT
121+
122+
- name: Determine sync paths
123+
id: paths
124+
shell: bash
125+
run: |
126+
case "${{ inputs.SYNC_TYPE }}" in
127+
"local-to-remote")
128+
echo "src=${{ steps.sanitized.outputs.src_clean }}" >> $GITHUB_OUTPUT
129+
echo "dst=s3:${{ inputs.S3_BUCKET }}/${{ steps.sanitized.outputs.dst_clean }}" >> $GITHUB_OUTPUT
130+
echo "Syncing from local directory to S3"
131+
;;
132+
"remote-to-local")
133+
echo "src=s3:${{ inputs.S3_BUCKET }}/${{ steps.sanitized.outputs.src_clean }}" >> $GITHUB_OUTPUT
134+
echo "dst=${{ steps.sanitized.outputs.dst_clean }}" >> $GITHUB_OUTPUT
135+
echo "Syncing from S3 to local directory"
136+
;;
137+
"remote-to-remote")
138+
echo "src=s3:${{ inputs.S3_BUCKET }}/${{ steps.sanitized.outputs.src_clean }}" >> $GITHUB_OUTPUT
139+
echo "dst=s3:${{ inputs.S3_BUCKET }}/${{ steps.sanitized.outputs.dst_clean }}" >> $GITHUB_OUTPUT
140+
echo "Syncing between S3 locations"
141+
;;
142+
*)
143+
echo "Error: Invalid SYNC_TYPE '${{ inputs.SYNC_TYPE }}'. Must be local-to-remote, remote-to-local, or remote-to-remote"
144+
exit 1
145+
;;
146+
esac
147+
148+
- name: Perform rclone sync
149+
shell: bash
150+
env:
151+
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
152+
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
153+
run: |
154+
echo "Syncing from ${{ steps.paths.outputs.src }} to ${{ steps.paths.outputs.dst }}"
155+
rclone sync "${{ steps.paths.outputs.src }}/" "${{ steps.paths.outputs.dst }}/"
156+
157+
- name: Run deduplication on S3
158+
if: ${{ inputs.SYNC_TYPE == 'local-to-remote' || inputs.SYNC_TYPE == 'remote-to-remote' }}
159+
shell: bash
160+
env:
161+
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
162+
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
163+
run: |
164+
echo "Running deduplication on S3 destination by hash, keeping newest files"
165+
rclone dedupe --by-hash --dedupe-mode newest "${{ steps.paths.outputs.dst }}/"
166+
167+
- name: List destination contents
168+
shell: bash
169+
env:
170+
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
171+
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
172+
run: |
173+
echo "Listing destination contents:"
174+
rclone ls "${{ steps.paths.outputs.dst }}/"

‎.github/workflows/cicd-docker-build-and-distribute.yml‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,36 @@ on:
8181
type: string
8282
default: "build"
8383
description: Directory to place produced artifacts into (will be inside CODE_WORKING_DIRECTORY)
84+
PUSH_TO_S3:
85+
required: false
86+
type: boolean
87+
default: false
88+
description: Push artifacts into S3 bucket
89+
S3_BUCKET:
90+
required: false
91+
type: string
92+
default: ""
93+
description: S3 bucket name
94+
S3_PROVIDER:
95+
required: false
96+
type: string
97+
default: "AWS"
98+
description: S3 provider type
99+
S3_REGION:
100+
required: false
101+
type: string
102+
default: "us-east-2"
103+
description: AWS S3 region
104+
S3_LOCATION_CONSTRAINT:
105+
required: false
106+
type: string
107+
default: "us-east-2"
108+
description: AWS S3 location constraint
109+
S3_STORAGE_CLASS:
110+
required: false
111+
type: string
112+
default: ""
113+
description: AWS S3 storage class
84114

85115
secrets:
86116
HOSTNAME:
@@ -97,6 +127,10 @@ on:
97127
required: false
98128
DEPLOYMENT_TOKEN:
99129
required: false
130+
AWS_ACCESS_KEY_ID:
131+
required: false
132+
AWS_SECRET_ACCESS_KEY:
133+
required: false
100134
# GH_BOT_DEPLOY_TOKEN is DEPRECATED
101135
GH_BOT_DEPLOY_TOKEN:
102136
required: false
@@ -133,6 +167,7 @@ jobs:
133167
path: actions
134168
sparse-checkout: |
135169
.github/actions/teleport/action.yml
170+
.github/actions/rclone-s3-sync/action.yml
136171
.github/actions/docker-build-artifacts/action.yml
137172
sparse-checkout-cone-mode: false
138173

@@ -160,8 +195,31 @@ jobs:
160195
if-no-files-found: warn
161196
retention-days: ${{ inputs.LOG_RETENTION_DAYS }}
162197

198+
- name: Create hash file
199+
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS && inputs.PUSH_TO_S3 }}
200+
shell: bash
201+
working-directory: ${{ inputs.CODE_WORKING_DIRECTORY }}
202+
run: |
203+
echo "${{ github.sha }}" > ${{ inputs.ARTIFACTS_DIR }}/hash.txt
204+
205+
- name: Upload artifacts to S3
206+
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS && inputs.PUSH_TO_S3 }}
207+
uses: ./actions/.github/actions/rclone-s3-sync
208+
env:
209+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
210+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
211+
with:
212+
SRC: ${{ inputs.CODE_WORKING_DIRECTORY }}/${{ inputs.ARTIFACTS_DIR }}
213+
DST: ${{ inputs.META_FILE_PATH_PREFIX }}
214+
S3_BUCKET: ${{ inputs.S3_BUCKET }}
215+
S3_PROVIDER: ${{ inputs.S3_PROVIDER }}
216+
S3_REGION: ${{ inputs.S3_REGION }}
217+
S3_LOCATION_CONSTRAINT: ${{ inputs.S3_LOCATION_CONSTRAINT }}
218+
S3_STORAGE_CLASS: ${{ inputs.S3_STORAGE_CLASS }}
219+
SYNC_TYPE: local-to-remote
220+
163221
- name: Compress build artifacts
164-
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }}
222+
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS && !inputs.PUSH_TO_S3 }}
165223
shell: bash
166224
working-directory: ${{ inputs.CODE_WORKING_DIRECTORY }}
167225
env:
@@ -170,7 +228,7 @@ jobs:
170228
tar -czvf ${{ inputs.TARGET_ARTIFACT_NAME }}.tar.gz -C "${ARTIFACTS_DIR}" $(ls -1 "${ARTIFACTS_DIR}")
171229
172230
- name: Copy/Exec artifacts/commands to/on remote host
173-
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS }}
231+
if: ${{ inputs.UPLOAD_BUILD_ARTIFACTS && !inputs.PUSH_TO_S3 }}
174232
uses: ./actions/.github/actions/teleport
175233
with:
176234
EXEC_COMMANDS_PRE: "mkdir -p ${{ inputs.META_FILE_PATH_PREFIX }}"

0 commit comments

Comments
 (0)