原始导入日志优化
This commit is contained in:
@@ -395,6 +395,7 @@ class OriginalDataService(BaseService[OriginalData]):
|
||||
is_exists = db.execute(check_query, {"nyid": nyid}).fetchone()[0]
|
||||
|
||||
if is_exists > 0:
|
||||
logger.warning(f"Data for NYID {nyid} already exists in {table_name}, skipping import")
|
||||
return 0
|
||||
|
||||
# ===== 性能优化:批量查询沉降数据 =====
|
||||
@@ -412,6 +413,7 @@ class OriginalDataService(BaseService[OriginalData]):
|
||||
# 将数据分组,每组1000条(MySQL默认支持)
|
||||
batch_size = 1000
|
||||
total_inserted = 0
|
||||
logger.info(f"Starting batch insert into {table_name}, total records: {len(data)}")
|
||||
for i in range(0, len(data), batch_size):
|
||||
batch_data = data[i:i + batch_size]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user