Skip to content

Commit ca64a31

Browse files
minju-22minju-22
authored
Merge pull request #186 from yuja201/fix/185-progress-calculation
[fix] correct progress calculation for worker-based data generation
2 parents ad3e2eb + 4d3f9a6 commit ca64a31

3 files changed

Lines changed: 59 additions & 67 deletions

File tree

‎src/main/services/data-generator/data-generator-service.ts‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ import { createLogger } from '../../utils/logger'
1515
const logger = createLogger('data-generator-service')
1616
const MAX_PARALLEL = Math.max(1, Math.floor(os.cpus().length / 2))
1717

18+
/**
19+
* 데이터 생성 전체 흐름을 관리하는 메인 함수
20+
* - Worker 실행
21+
* - 진행률 전달
22+
* - SQL ZIP 생성
23+
*/
1824
export async function runDataGenerator(
1925
payload: GenerateRequest,
2026
mainWindow: BrowserWindow
@@ -44,7 +50,10 @@ export async function runDataGenerator(
4450
const schema = await fetchSchema(projectId)
4551

4652
const ruleIds = new Set<number>()
53+
let totalRows = 0
54+
let progressRows = 0
4755
for (const table of tables) {
56+
totalRows += table.recordCnt
4857
for (const column of table.columns) {
4958
if (
5059
(column.dataSource === 'FAKER' || column.dataSource === 'AI') &&
@@ -56,6 +65,13 @@ export async function runDataGenerator(
5665
}
5766
}
5867

68+
if (totalRows === 0) {
69+
throw new Error('Total row count is 0. No data to generate.')
70+
}
71+
72+
let successRows = 0
73+
let failedRows = 0
74+
5975
const rules = Array.from(ruleIds)
6076
.map((id) => getRuleById(id))
6177
.filter((rule): rule is NonNullable<typeof rule> => Boolean(rule))
@@ -86,6 +102,9 @@ export async function runDataGenerator(
86102
const results: WorkerResult[] = []
87103
const cacheRoot = getFileCacheRoot()
88104

105+
/**
106+
* 다음 테이블 작업을 Worker로 실행
107+
*/
89108
const startNext = async (): Promise<void> => {
90109
try {
91110
if (queue.length === 0) return
@@ -133,6 +152,15 @@ export async function runDataGenerator(
133152
if (data.type) {
134153
if (data.type === 'worker-result') {
135154
results.push(data.result)
155+
} else if (data.type === 'row-progress') {
156+
progressRows += data.progress
157+
const progress = Math.floor((progressRows / totalRows) * 100)
158+
data.progress = progress
159+
mainWindow.webContents.send('data-generator:progress', data)
160+
} else if (data.type === 'table-complete') {
161+
successRows += data.successRows
162+
failedRows += data.failedRows
163+
mainWindow.webContents.send('data-generator:progress', data)
136164
} else {
137165
mainWindow.webContents.send('data-generator:progress', data)
138166
}
@@ -218,7 +246,10 @@ export async function runDataGenerator(
218246
mainWindow.webContents.send('data-generator:progress', {
219247
type: 'all-complete',
220248
successCount: successResults.length,
221-
failCount: failedResults.length
249+
failCount: failedResults.length,
250+
successRows: successRows,
251+
failedRows: failedRows,
252+
totalRows: totalRows
222253
})
223254

224255
const allErrors = failedResults.map((r) => `[${r.tableName}] ${r.error ?? 'Unknown error'}`)
@@ -233,6 +264,9 @@ export async function runDataGenerator(
233264
}
234265
}
235266

267+
/**
268+
* DB URL에서 host와 port 추출
269+
*/
236270
function parseDatabaseUrl(rawUrl: string, dbType: SupportedDBMS): { host: string; port: number } {
237271
const defaultPort = dbType === 'mysql' ? 3306 : 5432
238272

@@ -261,7 +295,9 @@ function parseDatabaseUrl(rawUrl: string, dbType: SupportedDBMS): { host: string
261295
}
262296
}
263297

264-
// 재시도 기반 파일 삭제
298+
/**
299+
* 파일 잠금 시 재시도하며 삭제
300+
*/
265301
async function deleteWithRetry(filePath: string, maxRetries = 3, delayMs = 500): Promise<void> {
266302
for (let attempt = 0; attempt <= maxRetries; attempt++) {
267303
try {

‎src/main/services/data-generator/worker-runner.ts‎

Lines changed: 19 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ const logger = createLogger('worker-runner')
2626
const INVALID = { __invalid: true } as const
2727
type CellValue = string | typeof INVALID
2828

29-
// 컬럼별 스트림 생성 함수
29+
/**
30+
* 컬럼 설정에 따라 데이터 생성 스트림을 반환
31+
* (FAKER, AI, FILE, FIXED, REFERENCE)
32+
*/
3033
function createColumnStream(
3134
col: {
3235
columnName: string
@@ -146,7 +149,9 @@ function createColumnStream(
146149
}
147150
}
148151

149-
// AI 컬럼과 non-AI 컬럼 분리
152+
/**
153+
* 컬럼 목록을 AI / non-AI 인덱스로 분리
154+
*/
150155
function separateColumnsByType(columns: { dataSource: DataSourceType }[]): {
151156
aiColumns: number[]
152157
nonAiColumns: number[]
@@ -280,6 +285,9 @@ type DirectContext = {
280285
close: () => Promise<void>
281286
}
282287

288+
/**
289+
* DIRECT_DB 모드에서 사용할 DB 연결 컨텍스트 생성
290+
*/
283291
async function createDirectContext(
284292
dbType: keyof typeof DBMS_MAP,
285293
connection: NonNullable<WorkerTask['connection']>
@@ -322,6 +330,12 @@ async function createDirectContext(
322330
}
323331
}
324332

333+
/**
334+
* 테이블 단위 데이터 생성 Worker
335+
* - 컬럼 스트림 생성
336+
* - 행 조립
337+
* - SQL 파일 생성 또는 DB 삽입
338+
*/
325339
async function runWorker(task: WorkerTask): Promise<WorkerResult> {
326340
const { table, dbType, mode, connection } = task
327341
const { tableName, recordCnt, columns } = table
@@ -347,7 +361,6 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
347361
}
348362

349363
logger.info(`[${tableName}] 시작: ${recordCnt.toLocaleString()}행, ${columns.length}컬럼`)
350-
const startTime = Date.now()
351364
let totalProcessed = 0
352365
const numChunks = Math.max(1, Math.ceil(recordCnt / CHUNK_SIZE))
353366

@@ -359,22 +372,16 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
359372
continue
360373
}
361374

362-
logger.info(`\n[${tableName}] 청크 ${chunkIdx + 1}/${numChunks} 처리 중 (${chunkSize}행)`)
363-
const chunkStartTime = Date.now()
364-
365375
const columnStreams = columns.map((col) => createColumnStream(col, task, chunkSize))
366376
const { aiColumns, nonAiColumns } = separateColumnsByType(columns)
367377
const chunkColumnValues: (string | typeof INVALID)[][] = new Array(columns.length)
368378

369379
// === Non-AI 컬럼 처리 (예외 처리 일관화) ===
370380
if (nonAiColumns.length > 0) {
371-
logger.info(`[${tableName}] Non-AI 컬럼 동시 처리:`)
372381
const nonAiResults = await Promise.all(
373382
nonAiColumns.map(async (colIdx) => {
374383
const col = columns[colIdx]
375384
const stream = columnStreams[colIdx]
376-
const colStart = Date.now()
377-
logger.info(` ▶ [${col.columnName}] 처리 (${col.dataSource})`)
378385

379386
const values: (string | typeof INVALID)[] = []
380387
for (let i = 0; i < chunkSize; i++) {
@@ -394,9 +401,6 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
394401
}
395402
}
396403

397-
const colDuration = ((Date.now() - colStart) / 1000).toFixed(2)
398-
logger.info(` ✓ [${col.columnName}] 완료 (${colDuration}초)`)
399-
400404
return { colIdx, values }
401405
})
402406
)
@@ -408,17 +412,13 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
408412

409413
// === AI 컬럼 처리 (동시 MAX_AI_CONCURRENT개, 예외 처리 통일) ===
410414
if (aiColumns.length > 0) {
411-
logger.info(`[${tableName}] AI 컬럼 처리 (동시 ${MAX_AI_CONCURRENT}개):`)
412-
413415
for (let i = 0; i < aiColumns.length; i += MAX_AI_CONCURRENT) {
414416
const batch = aiColumns.slice(i, i + MAX_AI_CONCURRENT)
415417

416418
const batchResults = await Promise.all(
417419
batch.map(async (colIdx) => {
418420
const col = columns[colIdx]
419421
const stream = columnStreams[colIdx]
420-
const colStart = Date.now()
421-
logger.info(` ▶ [${col.columnName}] 처리 (${col.dataSource})`)
422422

423423
const values: (string | typeof INVALID)[] = []
424424
for (let j = 0; j < chunkSize; j++) {
@@ -438,9 +438,6 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
438438
}
439439
}
440440

441-
const colDuration = ((Date.now() - colStart) / 1000).toFixed(2)
442-
logger.info(` ✓ [${col.columnName}] 완료 (${colDuration}초)`)
443-
444441
return { colIdx, values }
445442
})
446443
)
@@ -450,7 +447,6 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
450447
})
451448

452449
if (i + MAX_AI_CONCURRENT < aiColumns.length) {
453-
logger.info(` … 다음 AI 컬럼 대기 (1초)...`)
454450
await new Promise((res) => setTimeout(res, 1000))
455451
}
456452
}
@@ -477,13 +473,11 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
477473
`[행 변환 오류] ${tableName} ${totalProcessed + rowIdx + 1}행 변환 실패`
478474
)
479475
}
476+
} else if (!directMode) {
477+
totalProcessed++
480478
}
481479

482480
rows.push(`(${rowValues.join(', ')})`)
483-
484-
if (!directMode) {
485-
totalProcessed++
486-
}
487481
}
488482

489483
// === DIRECT_DB bulk insert + fallback ===
@@ -529,47 +523,18 @@ async function runWorker(task: WorkerTask): Promise<WorkerResult> {
529523
totalProcessed += successOnThisChunk // 성공한 row만 증가
530524
totalFailed += failedOnThisChunk // fallback에서 실패한 row
531525

532-
process.parentPort.postMessage({
533-
type: 'row-delta',
534-
tableName,
535-
success: successOnThisChunk,
536-
fail: failedOnThisChunk
537-
})
538-
539526
await fs.promises.appendFile(sqlPath, bulkSQL + '\n', 'utf8')
540527
}
541528

542-
const chunkDuration = ((Date.now() - chunkStartTime) / 1000).toFixed(2)
543-
logger.info(`\n[${tableName}] 청크 ${chunkIdx + 1} 완료 (${chunkDuration}초)`)
544-
545-
const progressPercent =
546-
chunkIdx + 1 === numChunks ? 100 : Math.floor((chunkEnd / recordCnt) * 100)
547-
548529
process.parentPort.postMessage({
549530
type: 'row-progress',
550531
tableName,
551-
progress: progressPercent
532+
progress: chunkSize
552533
})
553534

554535
await new Promise((res) => setTimeout(res, 100))
555-
556-
if (chunkEnd === recordCnt) {
557-
columns.forEach((col) => {
558-
process.parentPort.postMessage({
559-
type: 'column-progress',
560-
tableName,
561-
columnName: col.columnName,
562-
progress: 100
563-
})
564-
})
565-
}
566536
}
567537

568-
const totalDuration = ((Date.now() - startTime) / 1000).toFixed(2)
569-
logger.info(
570-
`\n[${tableName}] 전체 완료 (${totalDuration}초, ${totalProcessed.toLocaleString()}행)`
571-
)
572-
573538
if (directContext) {
574539
await directContext.commit()
575540
await directContext.close()

‎src/renderer/src/views/DummyInsertView.tsx‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -86,21 +86,12 @@ const DummyInsertView: React.FC = () => {
8686
setProgress(message.progress)
8787
}
8888

89-
if (message.type === 'row-delta') {
90-
setSuccessRows((prev) => prev + (message.successDelta ?? 0))
91-
setFailedRows((prev) => prev + (message.failDelta ?? 0))
92-
}
93-
9489
if (message.type === 'table-complete' && message.tableName) {
9590
const hasFail = (message.failedRows ?? 0) > 0
9691

97-
setTotalRows((prev) => prev + (message.totalRows ?? 0))
98-
setSuccessRows((prev) => prev + (message.successRows ?? 0))
99-
setFailedRows((prev) => prev + (message.failedRows ?? 0))
100-
10192
setTables((prev) =>
10293
prev.map((t) =>
103-
t.name === message.tableName ? { ...t, status: hasFail ? 'warning' : 'success' } : t
94+
t.name === message.tableName ? { ...t, status: hasFail ? 'failure' : 'success' } : t
10495
)
10596
)
10697
}
@@ -327,7 +318,7 @@ const DummyInsertView: React.FC = () => {
327318
</>
328319
) : (
329320
<>
330-
<p style={{ font: 'var(--preBold20)', marginBottom: 16 }}>더미데이터 삽입 완료</p>
321+
<p style={{ font: 'var(--preBold20)', marginBottom: 16 }}>더미데이터 생성 완료</p>
331322
<p style={{ marginBottom: 20 }}>
332323
전체 {totalCount.toLocaleString()}개 중 {insertedCount.toLocaleString()}개 완료
333324
</p>

0 commit comments

Comments
 (0)