admin/railway_cloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

原始导入日志优化

Browse Source
This commit is contained in:
lhx
2025-11-18 10:08:49 +08:00
parent 0e782baed9
commit 61df55074a
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/services/original_data.py
View File

@@ -395,6 +395,7 @@ class OriginalDataService(BaseService[OriginalData]):
is_exists = db.execute(check_query, {"nyid": nyid}).fetchone()[0] is_exists = db.execute(check_query, {"nyid": nyid}).fetchone()[0]
if is_exists > 0: if is_exists > 0:
logger.warning(f"Data for NYID {nyid} already exists in {table_name}, skipping import")
return 0 return 0
# ===== 性能优化：批量查询沉降数据 ===== # ===== 性能优化：批量查询沉降数据 =====
@@ -412,6 +413,7 @@ class OriginalDataService(BaseService[OriginalData]):
# 将数据分组，每组1000条（MySQL默认支持） # 将数据分组，每组1000条（MySQL默认支持）
batch_size = 1000 batch_size = 1000
total_inserted = 0 total_inserted = 0
logger.info(f"Starting batch insert into {table_name}, total records: {len(data)}")
for i in range(0, len(data), batch_size): for i in range(0, len(data), batch_size):
batch_data = data[i:i + batch_size] batch_data = data[i:i + batch_size]