Skip to content

Commit cddd17c

Browse files
committed
CI: harden startup and test execution
1 parent 05a4239 commit cddd17c

6 files changed

Lines changed: 159 additions & 10 deletions

File tree

.github/workflows/ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ jobs:
133133
./deploy.sh -g -D /mnt/data/docker -W /mnt/data/workspace ${{ matrix.mode == 'multinode' && '-M' || '' }} -C
134134
135135
- name: Run ${{ matrix.mode }} dojo tests
136-
timeout-minutes: 15
136+
timeout-minutes: 20
137137
run: |
138138
./deploy.sh -g -D /mnt/data/docker -W /mnt/data/workspace ${{ matrix.mode == 'multinode' && '-M' || '' }} -C -N -t
139139

Dockerfile

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,15 @@ EOF
111111
WORKDIR /opt/pwn.college
112112
COPY . .
113113

114-
RUN find /opt/pwn.college/ctfd/patches -exec patch -d /opt/CTFd -p1 -N -i {} \;
114+
RUN <<EOF
115+
set -eu
116+
find /opt/pwn.college/ctfd/patches -type f -name '*.patch' -print > /tmp/ctfd-patches
117+
LC_ALL=C sort -o /tmp/ctfd-patches /tmp/ctfd-patches
118+
while IFS= read -r patch_file; do
119+
patch --batch --forward -d /opt/CTFd -p1 -i "$patch_file"
120+
done < /tmp/ctfd-patches
121+
rm /tmp/ctfd-patches
122+
EOF
115123

116124
RUN <<EOF
117125
find /opt/pwn.college/etc/systemd/system -type f -exec ln -s {} /etc/systemd/system/ \;

dojo_plugin/__init__.py

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from flask import Response, request, redirect, current_app
1111
from itsdangerous.exc import BadSignature
1212
from marshmallow_sqlalchemy import field_for
13+
from sqlalchemy import text
1314
from CTFd.models import db, Challenges, Users, Solves
1415
from CTFd.utils.user import get_current_user
1516
from CTFd.plugins import register_admin_plugin_menu_bar
@@ -148,8 +149,25 @@ def handle_authorization(default_handler):
148149
default_handler()
149150

150151

152+
DOJO_SCHEMA_LOCK_NAMESPACE = 1685026671
153+
DOJO_SCHEMA_LOCK_ID = 1886351983
154+
155+
156+
def create_dojo_tables():
157+
with db.engine.begin() as connection:
158+
if connection.dialect.name == "postgresql":
159+
connection.execute(
160+
text("SELECT pg_advisory_xact_lock(:namespace, :lock_id)"),
161+
{
162+
"namespace": DOJO_SCHEMA_LOCK_NAMESPACE,
163+
"lock_id": DOJO_SCHEMA_LOCK_ID,
164+
},
165+
)
166+
db.metadata.create_all(bind=connection)
167+
168+
151169
def load(app):
152-
db.create_all()
170+
create_dojo_tables()
153171

154172
init_query_timer()
155173

test/test_background_stats.py

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -242,12 +242,36 @@ def test_multiple_solves_update_stats(stats_test_dojo, stats_test_user):
242242
assert second_solves > first_solves, f"Expected solves to increase from {first_solves} to more, got {second_solves}"
243243

244244
def test_cold_start_initializes_cache(example_dojo):
245-
cache_key = f"stats:dojo:{example_dojo}"
246-
cached_data = redis_get(cache_key)
247-
248-
if cached_data is None:
245+
official_dojo_id = example_dojo.split("~", 1)[0]
246+
cache_key = f"stats:dojo:{official_dojo_id}"
247+
redis_delete(cache_key, f"{cache_key}:updated")
248+
restart_time = time.time()
249+
restart = dojo_run(
250+
"docker", "restart", "--time", "1", "stats-worker", check=False
251+
)
252+
assert restart.returncode == 0, restart.stderr
253+
deadline = time.time() + 30
254+
worker_logs = None
255+
worker_output = ""
256+
while time.time() < deadline:
257+
worker_logs = dojo_run(
258+
"docker", "logs", "stats-worker", "--since",
259+
str(restart_time), check=False,
260+
)
261+
worker_output = worker_logs.stdout + worker_logs.stderr
262+
if "Cold start complete" in worker_output:
263+
break
264+
time.sleep(0.2)
265+
assert worker_logs is not None
266+
assert "Cold start complete" in worker_output, worker_output
267+
if not wait_for_cache_timestamp_updated(
268+
cache_key, after_time=restart_time, timeout=30
269+
):
249270
result = dojo_run("docker", "logs", "stats-worker", "--tail", "100", check=False)
250-
pytest.fail(f"Cold start should have initialized cache for {example_dojo}. Worker logs:\n{result.stdout}")
271+
pytest.fail(f"Cold start should have initialized cache for {official_dojo_id}. Worker logs:\n{result.stdout}\n{result.stderr}")
272+
273+
cached_data = redis_get(cache_key)
274+
assert cached_data is not None
251275

252276
stats = json.loads(cached_data)
253277
assert 'solves' in stats

test/test_schema.py

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
from utils import dojo_run
2+
3+
4+
def test_dojo_schema_creation_serializes_first_start():
5+
result = dojo_run("dojo", "flask", input="""
6+
import threading
7+
import time
8+
import uuid
9+
10+
from flask import current_app
11+
from sqlalchemy import Column, Integer, Table, inspect, text
12+
13+
from CTFd.models import db
14+
from CTFd.plugins import dojo_plugin
15+
16+
app = current_app._get_current_object()
17+
table = Table(
18+
f"test_dojo_schema_{uuid.uuid4().hex}",
19+
db.metadata,
20+
Column("id", Integer, primary_key=True),
21+
)
22+
real_create_all = db.metadata.create_all
23+
create_calls = []
24+
create_calls_lock = threading.Lock()
25+
first_create_entered = threading.Event()
26+
release_first_create = threading.Event()
27+
errors = []
28+
29+
def tracked_create_all(*args, **kwargs):
30+
with create_calls_lock:
31+
create_calls.append(threading.get_ident())
32+
call_number = len(create_calls)
33+
if call_number == 1:
34+
first_create_entered.set()
35+
assert release_first_create.wait(10)
36+
return real_create_all(*args, **kwargs)
37+
38+
def create_tables():
39+
try:
40+
with app.app_context():
41+
dojo_plugin.create_dojo_tables()
42+
except BaseException as error:
43+
errors.append(error)
44+
45+
db.metadata.create_all = tracked_create_all
46+
first_thread = threading.Thread(target=create_tables)
47+
second_thread = threading.Thread(target=create_tables)
48+
try:
49+
first_thread.start()
50+
assert first_create_entered.wait(10)
51+
second_thread.start()
52+
deadline = time.time() + 10
53+
waiting_for_lock = False
54+
while time.time() < deadline:
55+
with db.engine.connect() as connection:
56+
waiting_for_lock = bool(connection.execute(
57+
text(
58+
"SELECT COUNT(*) FROM pg_locks "
59+
"WHERE locktype = 'advisory' AND NOT granted "
60+
"AND classid = :namespace AND objid = :lock_id "
61+
"AND objsubid = 2"
62+
),
63+
{
64+
"namespace": dojo_plugin.DOJO_SCHEMA_LOCK_NAMESPACE,
65+
"lock_id": dojo_plugin.DOJO_SCHEMA_LOCK_ID,
66+
},
67+
).scalar())
68+
if waiting_for_lock:
69+
break
70+
time.sleep(0.01)
71+
assert waiting_for_lock
72+
assert len(create_calls) == 1
73+
release_first_create.set()
74+
first_thread.join(10)
75+
second_thread.join(10)
76+
assert not first_thread.is_alive()
77+
assert not second_thread.is_alive()
78+
assert errors == []
79+
assert len(create_calls) == 2
80+
assert inspect(db.engine).has_table(table.name)
81+
finally:
82+
release_first_create.set()
83+
for thread in (first_thread, second_thread):
84+
if thread.ident is not None:
85+
thread.join(10)
86+
db.metadata.create_all = real_create_all
87+
table.drop(db.engine, checkfirst=True)
88+
db.metadata.remove(table)
89+
90+
print("DOJO_SCHEMA_CREATION_SERIALIZED")
91+
""", check=False)
92+
assert result.returncode == 0, result.stderr
93+
assert "DOJO_SCHEMA_CREATION_SERIALIZED" in result.stdout, (
94+
result.stdout + result.stderr
95+
)

test/utils.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,12 @@ def dojo_run(*args, **kwargs):
122122

123123

124124
def db_sql(sql):
125-
db_result = dojo_run("dojo", "db", "-qAt", input=sql)
126-
return db_result.stdout
125+
db_result = dojo_run(
126+
"dojo", "db", "-v", "ON_ERROR_STOP=1", "-qAt", input=sql,
127+
check=False,
128+
)
129+
assert db_result.returncode == 0, db_result.stderr
130+
return db_result.stdout
127131

128132

129133
def get_user_id(user_name):

0 commit comments

Comments
 (0)