水准导入、沉降导入优化
This commit is contained in:
@@ -341,7 +341,8 @@ class SettlementDataService(BaseService[SettlementData]):
|
||||
|
||||
def batch_import_settlement_data(self, db: Session, data: List) -> Dict[str, Any]:
|
||||
"""
|
||||
批量导入沉降数据，
|
||||
批量导入沉降数据 - 性能优化版
|
||||
使用批量查询和批量操作，大幅提升导入速度
|
||||
1.根据观测点ID和期数ID判断是否重复，修复记录，跳过插入操作
|
||||
2.判断观测点数据是否存在，不存在则记录，跳过插入操作
|
||||
支持事务回滚，失败时重试一次
|
||||
@@ -354,6 +355,16 @@ class SettlementDataService(BaseService[SettlementData]):
|
||||
failed_count = 0
|
||||
failed_items = []
|
||||
|
||||
if total_count == 0:
|
||||
return {
|
||||
'success': False,
|
||||
'message': '导入数据不能为空',
|
||||
'total_count': 0,
|
||||
'success_count': 0,
|
||||
'failed_count': 0,
|
||||
'failed_items': []
|
||||
}
|
||||
|
||||
for attempt in range(2): # 最多重试1次
|
||||
try:
|
||||
db.begin()
|
||||
@@ -361,91 +372,131 @@ class SettlementDataService(BaseService[SettlementData]):
|
||||
failed_count = 0
|
||||
failed_items = []
|
||||
|
||||
# ===== 性能优化1：批量查询观测点数据（IN查询） =====
|
||||
point_id_list = list(set(item.get('point_id') for item in data if item.get('point_id')))
|
||||
logger.info(f"Checking {len(point_id_list)} unique point_ids in checkpoint data")
|
||||
checkpoints = db.query(Checkpoint).filter(Checkpoint.point_id.in_(point_id_list)).all()
|
||||
checkpoint_map = {c.point_id: c for c in checkpoints}
|
||||
missing_point_ids = set(point_id_list) - set(checkpoint_map.keys())
|
||||
|
||||
# 记录缺失的观测点
|
||||
for item_data in data:
|
||||
try:
|
||||
point_id = item_data.get('point_id')
|
||||
if point_id in missing_point_ids:
|
||||
failed_count += 1
|
||||
failed_items.append({
|
||||
'data': item_data,
|
||||
'error': '测点id不存在，跳过插入操作'
|
||||
})
|
||||
|
||||
# 判断观测点数据是否存在
|
||||
checkpoint = self._check_checkpoint_exists(db, item_data.get('point_id'))
|
||||
logger.info(f"Checkpoint {item_data.get('point_id')}: {checkpoint}")
|
||||
if not checkpoint:
|
||||
logger.error(f"Checkpoint {item_data.get('point_id')} not found")
|
||||
logger.error(f"Checkpoint {item_data} not found")
|
||||
failed_count += 1
|
||||
failed_items.append({
|
||||
'data': item_data,
|
||||
'error': '测点id不存在，跳过插入操作'
|
||||
})
|
||||
continue
|
||||
# 如果所有数据都失败，直接返回
|
||||
if failed_count == total_count:
|
||||
db.rollback()
|
||||
return {
|
||||
'success': False,
|
||||
'message': '所有观测点ID都不存在',
|
||||
'total_count': total_count,
|
||||
'success_count': 0,
|
||||
'failed_count': total_count,
|
||||
'failed_items': failed_items
|
||||
}
|
||||
|
||||
settlement = self.get_by_point_and_nyid(
|
||||
db,
|
||||
nyid=item_data.get('NYID'),
|
||||
point_id=item_data.get('point_id')
|
||||
)
|
||||
if settlement:
|
||||
# 跳过数据
|
||||
logger.info(f"Continue settlement data: {item_data.get('point_id')}-{item_data.get('NYID')}")
|
||||
logger.info(f"Existing settlement data: {settlement}")
|
||||
# ===== 性能优化2：批量查询现有沉降数据（IN查询） =====
|
||||
# 只查询有效的观测点数据
|
||||
valid_items = [item for item in data if item.get('point_id') not in missing_point_ids]
|
||||
if valid_items:
|
||||
# 使用组合键 (point_id, NYID) 查询现有数据
|
||||
existing_data = db.query(SettlementData).filter(
|
||||
SettlementData.point_id.in_(point_id_list)
|
||||
).all()
|
||||
|
||||
# 使用组合键创建查找表：key = f"{point_id}_{NYID}"
|
||||
existing_map = {
|
||||
f"{item.point_id}_{item.NYID}": item
|
||||
for item in existing_data
|
||||
}
|
||||
logger.info(f"Found {len(existing_data)} existing settlement records")
|
||||
|
||||
# ===== 性能优化3：批量处理插入和跳过 =====
|
||||
to_insert = []
|
||||
|
||||
for item_data in valid_items:
|
||||
point_id = item_data.get('point_id')
|
||||
nyid = str(item_data.get('NYID'))
|
||||
|
||||
# 构建组合键
|
||||
key = f"{point_id}_{nyid}"
|
||||
|
||||
if key in existing_map:
|
||||
# 数据已存在，跳过
|
||||
logger.info(f"Continue settlement data: {point_id}-{nyid}")
|
||||
failed_count += 1
|
||||
failed_items.append({
|
||||
'data': item_data,
|
||||
'error': '数据已存在，跳过插入操作'
|
||||
})
|
||||
continue
|
||||
# 更新操作
|
||||
# settlement.CVALUE = item_data.get('CVALUE')
|
||||
# settlement.MAVALUE = item_data.get('MAVALUE')
|
||||
# settlement.MTIME_W = item_data.get('MTIME_W')
|
||||
# settlement.PRELOADH = item_data.get('PRELOADH')
|
||||
# settlement.PSTATE = item_data.get('PSTATE')
|
||||
# settlement.REMARK = item_data.get('REMARK')
|
||||
# settlement.WORKINFO = item_data.get('WORKINFO')
|
||||
# settlement.createdate = item_data.get('createdate')
|
||||
# settlement.day = item_data.get('day')
|
||||
# settlement.day_jg = item_data.get('day_jg')
|
||||
# settlement.isgzjdxz = item_data.get('isgzjdxz')
|
||||
# settlement.mavalue_bc = item_data.get('mavalue_bc')
|
||||
# settlement.mavalue_lj = item_data.get('mavalue_lj')
|
||||
# settlement.sjName = item_data.get('sjName')
|
||||
# settlement.useflag = item_data.get('useflag')
|
||||
# settlement.workinfoname = item_data.get('workinfoname')
|
||||
# settlement.upd_remark = item_data.get('upd_remark')
|
||||
|
||||
else:
|
||||
# 新增操作
|
||||
settlement = SettlementData(
|
||||
point_id=item_data.get('point_id'),
|
||||
CVALUE=item_data.get('CVALUE'),
|
||||
MAVALUE=item_data.get('MAVALUE'),
|
||||
MTIME_W=item_data.get('MTIME_W'),
|
||||
NYID=item_data.get('NYID'),
|
||||
PRELOADH=item_data.get('PRELOADH'),
|
||||
PSTATE=item_data.get('PSTATE'),
|
||||
REMARK=item_data.get('REMARK'),
|
||||
WORKINFO=item_data.get('WORKINFO'),
|
||||
createdate=item_data.get('createdate'),
|
||||
day=item_data.get('day'),
|
||||
day_jg=item_data.get('day_jg'),
|
||||
isgzjdxz=item_data.get('isgzjdxz'),
|
||||
mavalue_bc=item_data.get('mavalue_bc'),
|
||||
mavalue_lj=item_data.get('mavalue_lj'),
|
||||
sjName=item_data.get('sjName'),
|
||||
useflag=item_data.get('useflag'),
|
||||
workinfoname=item_data.get('workinfoname'),
|
||||
upd_remark=item_data.get('upd_remark')
|
||||
)
|
||||
db.add(settlement)
|
||||
logger.info(f"Created settlement data: {item_data.get('point_id')}-{item_data.get('NYID')}")
|
||||
# 记录需要插入的数据
|
||||
to_insert.append(item_data)
|
||||
|
||||
success_count += 1
|
||||
except Exception as e:
|
||||
failed_count += 1
|
||||
failed_items.append({
|
||||
'data': item_data,
|
||||
'error': str(e)
|
||||
})
|
||||
logger.error(f"Failed to process settlement data {item_data.get('point_id')}-{item_data.get('NYID')}: {str(e)}")
|
||||
raise e
|
||||
# ===== 执行批量插入 =====
|
||||
if to_insert:
|
||||
logger.info(f"Inserting {len(to_insert)} new records")
|
||||
# 分批插入，每批500条（避免SQL过长）
|
||||
batch_size = 500
|
||||
for i in range(0, len(to_insert), batch_size):
|
||||
batch = to_insert[i:i + batch_size]
|
||||
try:
|
||||
settlement_data_list = [
|
||||
SettlementData(
|
||||
point_id=item.get('point_id'),
|
||||
CVALUE=item.get('CVALUE'),
|
||||
MAVALUE=item.get('MAVALUE'),
|
||||
MTIME_W=item.get('MTIME_W'),
|
||||
NYID=str(item.get('NYID')),
|
||||
PRELOADH=item.get('PRELOADH'),
|
||||
PSTATE=item.get('PSTATE'),
|
||||
REMARK=item.get('REMARK'),
|
||||
WORKINFO=item.get('WORKINFO'),
|
||||
createdate=item.get('createdate'),
|
||||
day=item.get('day'),
|
||||
day_jg=item.get('day_jg'),
|
||||
isgzjdxz=item.get('isgzjdxz'),
|
||||
mavalue_bc=item.get('mavalue_bc'),
|
||||
mavalue_lj=item.get('mavalue_lj'),
|
||||
sjName=item.get('sjName'),
|
||||
useflag=item.get('useflag'),
|
||||
workinfoname=item.get('workinfoname'),
|
||||
upd_remark=item.get('upd_remark')
|
||||
)
|
||||
for item in batch
|
||||
]
|
||||
db.add_all(settlement_data_list)
|
||||
success_count += len(batch)
|
||||
logger.info(f"Inserted batch {i//batch_size + 1}: {len(batch)} records")
|
||||
except Exception as e:
|
||||
failed_count += len(batch)
|
||||
failed_items.extend([
|
||||
{
|
||||
'data': item,
|
||||
'error': f'插入失败: {str(e)}'
|
||||
}
|
||||
for item in batch
|
||||
])
|
||||
logger.error(f"Failed to insert batch: {str(e)}")
|
||||
raise e
|
||||
|
||||
# 如果有失败记录，不提交事务
|
||||
if failed_items:
|
||||
db.rollback()
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'批量导入失败: {len(failed_items)}条记录处理失败',
|
||||
'total_count': total_count,
|
||||
'success_count': success_count,
|
||||
'failed_count': failed_count,
|
||||
'failed_items': failed_items
|
||||
}
|
||||
|
||||
db.commit()
|
||||
logger.info(f"Batch import settlement data completed. Success: {success_count}, Failed: {failed_count}")
|
||||
|
||||
Reference in New Issue
Block a user