Skip to content

Commit b5e811a

Browse files
committed
fix: chunk SQLite upsert in save_daily_data to avoid bind limit
Split the single multi-row INSERT into chunks of 50 records to stay within SQLite's bind-parameter limit (commonly 999). Without chunking, multi-year backfills could exceed the limit and raise OperationalError. Also clarifies docstring: return value is new inserts only, not updates.
1 parent 6b609ce commit b5e811a

1 file changed

Lines changed: 28 additions & 22 deletions

File tree

src/storage.py

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1423,14 +1423,15 @@ def save_daily_data(
14231423
策略:
14241424
- 按 `(code, date)` 做批量 UPSERT,已存在记录会覆盖更新
14251425
- 同一批次内若存在重复日期,以最后一条记录为准
1426+
- SQLite 分支按 chunk 写入以避免绑定参数上限
14261427
14271428
Args:
14281429
df: 包含日线数据的 DataFrame
14291430
code: 股票代码
14301431
data_source: 数据来源名称
14311432
14321433
Returns:
1433-
本次实际新增的记录数
1434+
本次实际新增的记录数(不含更新)
14341435
"""
14351436
if df is None or df.empty:
14361437
logger.warning(f"保存数据为空,跳过 {code}")
@@ -1467,28 +1468,33 @@ def save_daily_data(
14671468

14681469
def _write(session: Session) -> int:
14691470
if self._is_sqlite_engine:
1470-
stmt = sqlite_insert(StockDaily).values(records)
1471-
excluded = stmt.excluded
1472-
session.execute(
1473-
stmt.on_conflict_do_update(
1474-
index_elements=['code', 'date'],
1475-
set_={
1476-
'open': excluded.open,
1477-
'high': excluded.high,
1478-
'low': excluded.low,
1479-
'close': excluded.close,
1480-
'volume': excluded.volume,
1481-
'amount': excluded.amount,
1482-
'pct_chg': excluded.pct_chg,
1483-
'ma5': excluded.ma5,
1484-
'ma10': excluded.ma10,
1485-
'ma20': excluded.ma20,
1486-
'volume_ratio': excluded.volume_ratio,
1487-
'data_source': excluded.data_source,
1488-
'updated_at': excluded.updated_at,
1489-
},
1471+
# SQLite has a per-statement bind-parameter limit (commonly 999).
1472+
# Each record has ~15 columns, so chunk to stay well within bounds.
1473+
_SQLITE_CHUNK = 50
1474+
for i in range(0, len(records), _SQLITE_CHUNK):
1475+
chunk = records[i : i + _SQLITE_CHUNK]
1476+
stmt = sqlite_insert(StockDaily).values(chunk)
1477+
excluded = stmt.excluded
1478+
session.execute(
1479+
stmt.on_conflict_do_update(
1480+
index_elements=['code', 'date'],
1481+
set_={
1482+
'open': excluded.open,
1483+
'high': excluded.high,
1484+
'low': excluded.low,
1485+
'close': excluded.close,
1486+
'volume': excluded.volume,
1487+
'amount': excluded.amount,
1488+
'pct_chg': excluded.pct_chg,
1489+
'ma5': excluded.ma5,
1490+
'ma10': excluded.ma10,
1491+
'ma20': excluded.ma20,
1492+
'volume_ratio': excluded.volume_ratio,
1493+
'data_source': excluded.data_source,
1494+
'updated_at': excluded.updated_at,
1495+
},
1496+
)
14901497
)
1491-
)
14921498
else:
14931499
existing_rows = {
14941500
row.date: row

0 commit comments

Comments
 (0)