Skip to content

Commit e8e8b5a

Browse files
Antony1060VisenP
Antony1060
and
VisenP
authored
Generators overhaul (#111)
* WIP * WIP * WIP * WIP * WIP * WIP * WIP * WIP * First working version * WIP * Migration changes * Testcase submission files * Collect error data from generation * Sample clusters * Random changes * Allow direct testcase input/output * Switch to stramQuery for all selects in the big migration --------- Co-authored-by: VisenP <visen@visen.dev> Co-authored-by: VisenP <visen.pavlica12@gmail.com>
1 parent 20f305a commit e8e8b5a

61 files changed

Lines changed: 3224 additions & 659 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/backend/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,13 @@
2828
"argon2": "^0.41.1",
2929
"axios": "^1.2.6",
3030
"bcrypt": "^5.1.0",
31+
"cassandra-driver": "^4.8.0",
3132
"chalk": "^4.1.2",
3233
"cors": "^2.8.5",
3334
"docx": "^8.0.1",
3435
"dotenv": "^16.0.3",
3536
"express": "5.0.0-beta.1",
37+
"express-fileupload": "^1.5.1",
3638
"express-rate-limit": "^6.7.0",
3739
"http-status-codes": "^2.2.0",
3840
"jsonwebtoken": "^9.0.0",
@@ -56,6 +58,7 @@
5658
"@types/bcrypt": "^5.0.0",
5759
"@types/cors": "^2.8.13",
5860
"@types/express": "^4.17.13",
61+
"@types/express-fileupload": "^1.5.1",
5962
"@types/express-serve-static-core": "^4.17.30",
6063
"@types/jsonwebtoken": "^8.5.9",
6164
"@types/node": "^18.15.11",

‎apps/backend/src/app.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import { NotificationsHandler } from "./routes/notifications/NotificationsHandle
3131
import { subscribeToEvaluatorResponseQueue } from "./lib/evaluation_rs";
3232
import { initAaiEdu } from "./lib/aaiedu";
3333
import { initS3 } from "./s3/S3";
34+
import fileUpload from "express-fileupload";
3435

3536
declare global {
3637
interface BigInt {
@@ -80,6 +81,8 @@ app.use(async (req, res, next) => {
8081
next();
8182
});
8283

84+
app.use(fileUpload());
85+
8386
app.use(json({ limit: "50mb" }), (req, _, next) => {
8487
// json from express@5 yields undefined for empty bodies,
8588
// this breaks validation, so we're falling back to an empty object
@@ -116,6 +119,11 @@ app.use((error: Error, _: Request, res: Response, next: NextFunction) => {
116119

117120
Promise.allSettled([
118121
Database.awaitConnection()
122+
.then(() =>
123+
initS3().catch((error) => {
124+
Logger.panic("S3 failed", error);
125+
})
126+
)
119127
.then(async () => {
120128
Logger.database("Successfully connected to database!");
121129
await initDatabase();
@@ -134,14 +142,12 @@ Promise.allSettled([
134142
.catch((error) => {
135143
Logger.panic("Redis failed", error);
136144
}),
137-
initS3().catch((error) => {
138-
Logger.panic("S3 failed", error);
139-
}),
140145
// for consistency
141146
initInflux(),
142147
initAaiEdu(),
143148
]).then(async () => {
144149
Logger.info("Ready");
150+
// kurac(Database);
145151

146152
app.listen(Globals.port, () => {
147153
Logger.info(`Listening on ${Globals.port} (Express ${expressPackageJson.version})`);

‎apps/backend/src/database/Database.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
EduUser,
77
ExamFinalSubmission,
88
ExamGradingScale,
9+
Generator,
910
MailPreference,
1011
ManagedUser,
1112
Organisation,
@@ -72,6 +73,9 @@ import { migration_fix_contest_members_table } from "./migrations/0046_fix_conte
7273
import { migration_add_managed_users } from "./migrations/0047_add_managed_users";
7374
import { migration_submission_compiler_output } from "./migrations/0048_submission_compiler_output";
7475
import { migration_contest_show_leaderboard } from "./migrations/0049_contest_show_leaderboard";
76+
import { migration_improve_generators } from "./migrations/0050_improve_generators";
77+
import { migration_add_sample_clusters } from "./migrations/0051_add_sample_clusters";
78+
import { migration_generator_id_index } from "./migrations/0052_generator_id_index";
7579

7680
export const Database = new ScylloClient<{
7781
users: User;
@@ -93,6 +97,7 @@ export const Database = new ScylloClient<{
9397
mail_preferences: MailPreference;
9498
edu_users: EduUser;
9599
managed_users: ManagedUser;
100+
generators: Generator;
96101
}>({
97102
client: {
98103
contactPoints: [Globals.dbHost + ":" + Globals.dbPort],
@@ -155,6 +160,9 @@ const migrations: Migration<any>[] = [
155160
migration_add_managed_users,
156161
migration_submission_compiler_output,
157162
migration_contest_show_leaderboard,
163+
migration_improve_generators,
164+
migration_add_sample_clusters,
165+
migration_generator_id_index,
158166
];
159167

160168
export const initDatabase = async () => {

‎apps/backend/src/database/migrations/0050_improve_generators.ts‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
import {
2+
ClusterSubmissionV2,
3+
ClusterV2,
4+
ClusterV3,
5+
GeneratorV1,
6+
ProblemV4,
7+
ProblemV5,
8+
TestcaseSubmissionV3,
9+
TestcaseV3,
10+
TestcaseV4,
11+
} from "@kontestis/models";
12+
import { Migration } from "scyllo";
13+
14+
import { Globals } from "../../globals";
15+
import { Logger } from "../../lib/logger";
16+
import { generateSnowflake } from "../../lib/snowflake";
17+
import { initS3, S3Client } from "../../s3/S3";
18+
import { readBucketStream } from "../../utils/stream";
19+
import { streamQuery } from "../streamQuery";
20+
21+
type MigrationType = {
22+
generators: GeneratorV1;
23+
testcases: TestcaseV3 & TestcaseV4;
24+
clusters: ClusterV2 & ClusterV3;
25+
problems: ProblemV5;
26+
testcase_submissions: TestcaseSubmissionV3;
27+
cluster_submissions: ClusterSubmissionV2;
28+
};
29+
30+
// eslint-disable-next-line sonarjs/cognitive-complexity
31+
export const migration_improve_generators: Migration<MigrationType> = async (database, log) => {
32+
await database.createTable(
33+
"generators",
34+
true,
35+
{
36+
id: {
37+
type: "bigint",
38+
},
39+
user_id: {
40+
type: "bigint",
41+
},
42+
contest_id: {
43+
type: "bigint",
44+
},
45+
organisation_id: {
46+
type: "bigint",
47+
},
48+
problem_id: {
49+
type: "bigint",
50+
},
51+
code: {
52+
type: "text",
53+
},
54+
name: {
55+
type: "text",
56+
},
57+
language: {
58+
type: "text",
59+
},
60+
},
61+
"id"
62+
);
63+
64+
await database.createIndex("generators", "generators_by_user_id", "user_id");
65+
await database.createIndex("generators", "generators_by_problem_id", "problem_id");
66+
await database.createIndex("generators", "generators_by_contest_id", "contest_id");
67+
await database.createIndex("generators", "generators_by_organisation_id", "organisation_id");
68+
69+
await database.raw("ALTER TABLE testcases ADD input_type text");
70+
await database.raw("ALTER TABLE testcases ADD output_type text");
71+
await database.raw("ALTER TABLE testcases ADD status text");
72+
await database.raw("ALTER TABLE testcases ADD input_file text");
73+
await database.raw("ALTER TABLE testcases ADD output_file text");
74+
await database.raw("ALTER TABLE testcases ADD error text");
75+
await database.raw("ALTER TABLE testcases ADD generator_input text");
76+
await database.raw("ALTER TABLE testcases ADD generator_id bigint");
77+
78+
await database.raw("ALTER TABLE clusters ADD order_number bigint");
79+
await database.raw("ALTER TABLE clusters ADD status text");
80+
await database.raw("ALTER TABLE clusters ADD error text");
81+
82+
await database.raw("ALTER TABLE testcase_submissions ADD input_file text");
83+
await database.raw("ALTER TABLE testcase_submissions ADD output_file text");
84+
await database.raw("ALTER TABLE testcase_submissions ADD submission_output_file text");
85+
86+
const testcases = await streamQuery<TestcaseV3>(database.client, "SELECT * FROM testcases");
87+
88+
const clusters = await streamQuery<ClusterV2>(database.client, "SELECT * FROM clusters");
89+
90+
const problems = await streamQuery<Pick<ProblemV4, "id" | "title">>(
91+
database.client,
92+
"SELECT id, title FROM problems"
93+
);
94+
95+
const clustersById: Record<string, ClusterV2> = {};
96+
97+
for (const cluster of clusters) {
98+
clustersById[cluster.id.toString()] = cluster;
99+
}
100+
101+
const problemsById: Record<string, Pick<ProblemV5, "title" | "id">> = {};
102+
103+
for (const problem of problems) {
104+
problemsById[problem.id.toString()] = problem;
105+
}
106+
107+
await initS3();
108+
109+
for (const testcase of testcases) {
110+
Logger.info("Migrating testcase " + testcase.id);
111+
const cluster = clustersById[testcase.cluster_id.toString()];
112+
113+
if (!cluster) {
114+
Logger.error("Cluster not found for testcase " + testcase.id);
115+
continue;
116+
}
117+
118+
const problem = problemsById[cluster.problem_id.toString()];
119+
120+
if (!problem) {
121+
Logger.error("Problem not found for testcase " + testcase.id);
122+
continue;
123+
}
124+
125+
const inputFilePath = `${problem.title}-${problem.id}/${
126+
testcase.id
127+
}-${generateSnowflake()}.in`;
128+
129+
await S3Client.putObject(Globals.s3.buckets.testcases, inputFilePath, testcase.input);
130+
131+
await database.update(
132+
"testcases",
133+
{
134+
input_type: "manual",
135+
output_type: "auto",
136+
status: "not-ready",
137+
input_file: inputFilePath,
138+
},
139+
{ id: testcase.id }
140+
);
141+
142+
Logger.info("Migrated testcase " + testcase.id);
143+
}
144+
145+
const clustersByProblemId: Record<string, ClusterV2[]> = {};
146+
147+
for (const cluster of clusters) {
148+
if (!clustersByProblemId[cluster.problem_id.toString()]) {
149+
clustersByProblemId[cluster.problem_id.toString()] = [];
150+
}
151+
152+
clustersByProblemId[cluster.problem_id.toString()].push(cluster);
153+
}
154+
155+
for (const clusters of Object.values(clustersByProblemId)) {
156+
clusters.sort((a, b) => Number(a.id - b.id));
157+
158+
for (const [index, cluster] of clusters.entries()) {
159+
await database.update("clusters", { order_number: BigInt(index) }, { id: cluster.id });
160+
}
161+
}
162+
163+
for (const cluster of clusters) {
164+
// Migrate to new generator format
165+
if (!cluster.generator) {
166+
await database.update("clusters", { status: "not-ready" }, { id: cluster.id });
167+
continue;
168+
}
169+
170+
if (!cluster.generator_language || !cluster.generator_code) {
171+
Logger.error("Generator language or code not found for cluster " + cluster.id);
172+
await database.update(
173+
"clusters",
174+
{
175+
status: "generator-error",
176+
error: "Invalid generator state, no language and/or code (migration)",
177+
},
178+
{ id: cluster.id }
179+
);
180+
continue;
181+
}
182+
183+
const generator: GeneratorV1 = {
184+
id: generateSnowflake(),
185+
problem_id: cluster.problem_id,
186+
name: `${problemsById[cluster.problem_id.toString()].title}-${cluster.id}`,
187+
code: cluster.generator_code ?? "",
188+
language: cluster.generator_language ?? "python",
189+
};
190+
191+
await database.insertInto("generators", generator);
192+
193+
await database.update("clusters", { status: "not-ready" }, { id: cluster.id });
194+
195+
const batch = database.batch();
196+
197+
for (let index = 0; index < 10; index++) {
198+
const testcase: TestcaseV4 = {
199+
id: generateSnowflake(),
200+
generator_id: generator.id,
201+
status: "not-ready",
202+
cluster_id: cluster.id,
203+
output_type: "auto",
204+
input_type: "generator",
205+
generator_input: index.toString(),
206+
};
207+
208+
batch.insertInto("testcases", testcase);
209+
}
210+
await batch.execute();
211+
}
212+
213+
const testcaseSubmissions = await streamQuery<TestcaseSubmissionV3>(
214+
database.client,
215+
"SELECT * FROM testcase_submissions"
216+
);
217+
218+
const clusterSubmissionsById: Record<string, ClusterSubmissionV2> = {};
219+
const clusterSubmissions = await streamQuery<ClusterSubmissionV2>(
220+
database.client,
221+
"SELECT * FROM cluster_submissions"
222+
);
223+
224+
for (const clusterSubmission of clusterSubmissions) {
225+
clusterSubmissionsById[clusterSubmission.id.toString()] = clusterSubmission;
226+
}
227+
228+
for (const testcaseSubmission of testcaseSubmissions) {
229+
const clusterSubmission =
230+
clusterSubmissionsById[testcaseSubmission.cluster_submission_id.toString()];
231+
232+
if (!clusterSubmission) {
233+
Logger.error(
234+
"Cluster submission not found for testcase submission " + testcaseSubmission.id
235+
);
236+
continue;
237+
}
238+
239+
if (clusterSubmission.submission_id < 0n) {
240+
continue;
241+
}
242+
243+
const prefix = `${clusterSubmission.submission_id}/${clusterSubmission.cluster_id}/`;
244+
245+
const files = await readBucketStream(
246+
S3Client.listObjects(Globals.s3.buckets.submission_meta, prefix, true)
247+
).catch((error) => {
248+
Logger.error(
249+
`Failed to read bucket stream for testcase submission: ${testcaseSubmission.id}`,
250+
error
251+
);
252+
});
253+
254+
if (!files) continue;
255+
256+
const fileNames = new Set<string>(files.map((it) => it.name).filter(Boolean) as string[]);
257+
258+
if (fileNames.size === 0) continue;
259+
260+
await database.update(
261+
"testcase_submissions",
262+
{
263+
input_file: fileNames.has(`${testcaseSubmission.testcase_id}.in`)
264+
? `${prefix}${testcaseSubmission.testcase_id}.in`
265+
: undefined,
266+
output_file: fileNames.has(`${testcaseSubmission.testcase_id}.out`)
267+
? `${prefix}${testcaseSubmission.testcase_id}.out`
268+
: undefined,
269+
submission_output_file: fileNames.has(`${testcaseSubmission.testcase_id}.sout`)
270+
? `${prefix}${testcaseSubmission.testcase_id}.sout`
271+
: undefined,
272+
},
273+
{ id: testcaseSubmission.id }
274+
);
275+
}
276+
277+
await database.raw("ALTER TABLE testcases DROP input");
278+
279+
await database.raw("ALTER TABLE clusters DROP generator");
280+
await database.raw("ALTER TABLE clusters DROP generator_language");
281+
await database.raw("ALTER TABLE clusters DROP generator_code");
282+
283+
log("Done");
284+
};

0 commit comments

Comments
 (0)