admin/railway_cloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

批量导入优化

Browse Source
This commit is contained in:
lhx
2025-11-17 16:14:12 +08:00
parent 502ef50a87
commit 54ac4037d5
3 changed files with 283 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
app/services/section_data.py
View File

@@ -251,7 +251,9 @@ class SectionDataService(BaseService[SectionData]):
def batch_import_sections(self, db: Session, data: List) -> Dict[str, Any]:
"""
批量导入断面数据，根据断面id判断是否重复，重复数据改为更新操作
批量导入断面数据 - 性能优化版
使用批量查询和批量操作，大幅提升导入速度
根据断面ID判断是否重复，重复数据跳过，不进行更新操作
支持事务回滚，失败时重试一次
"""
import logging
@@ -262,6 +264,16 @@ class SectionDataService(BaseService[SectionData]):
failed_count = 0
failed_items = []
if total_count == 0:
return {
'success': False,
'message': '导入数据不能为空',
'total_count': 0,
'success_count': 0,
'failed_count': 0,
'failed_items': []
}
for attempt in range(2): # 最多重试1次
try:
db.begin()
@@ -269,56 +281,90 @@ class SectionDataService(BaseService[SectionData]):
failed_count = 0
failed_items = []
for item_data in data:
try:
section = self.get_by_section_id(db, item_data.get('section_id'))
if section:
# 更新操作
section.mileage = item_data.get('mileage')
section.work_site = item_data.get('work_site')
section.basic_types = item_data.get('basic_types')
section.height = item_data.get('height')
section.status = item_data.get('status')
section.number = item_data.get('number')
section.transition_paragraph = item_data.get('transition_paragraph')
section.design_fill_height = item_data.get('design_fill_height')
section.compression_layer_thickness = item_data.get('compression_layer_thickness')
section.treatment_depth = item_data.get('treatment_depth')
section.foundation_treatment_method = item_data.get('foundation_treatment_method')
section.rock_mass_classification = item_data.get('rock_mass_classification')
section.account_id = item_data.get('account_id')
logger.info(f"Updated section: {item_data.get('section_id')}")
else:
# 新增操作
from ..models.section_data import SectionData
section = SectionData(
section_id=item_data.get('section_id'),
mileage=item_data.get('mileage'),
work_site=item_data.get('work_site'),
basic_types=item_data.get('basic_types'),
height=item_data.get('height'),
status=item_data.get('status'),
number=item_data.get('number'),
transition_paragraph=item_data.get('transition_paragraph'),
design_fill_height=item_data.get('design_fill_height'),
compression_layer_thickness=item_data.get('compression_layer_thickness'),
treatment_depth=item_data.get('treatment_depth'),
foundation_treatment_method=item_data.get('foundation_treatment_method'),
rock_mass_classification=item_data.get('rock_mass_classification'),
account_id=item_data.get('account_id')
)
db.add(section)
logger.info(f"Created section: {item_data.get('section_id')}")
# ===== 性能优化1：批量查询现有断面数据（IN查询） =====
# 统一转换为字符串处理（数据库section_id字段是VARCHAR类型）
section_id_list = list(set(str(item.get('section_id')) for item in data if item.get('section_id')))
logger.info(f"Checking {len(section_id_list)} unique section_ids")
existing_sections = db.query(SectionData).filter(SectionData.section_id.in_(section_id_list)).all()
success_count += 1
except Exception as e:
# 使用section_id创建查找表
existing_map = {
section.section_id: section
for section in existing_sections
}
logger.info(f"Found {len(existing_sections)} existing sections")
# ===== 性能优化2：批量处理插入和跳过 =====
to_insert = []
for item_data in data:
section_id = str(item_data.get('section_id')) # 统一转换为字符串
if section_id in existing_map:
# 数据已存在，跳过
logger.info(f"Continue section data: {section_id}")
failed_count += 1
failed_items.append({
'data': item_data,
'error': str(e)
'error': '数据已存在，跳过插入操作'
})
logger.error(f"Failed to process section {item_data.get('section_id')}: {str(e)}")
raise e
else:
# 记录需要插入的数据
to_insert.append(item_data)
# ===== 执行批量插入 =====
if to_insert:
logger.info(f"Inserting {len(to_insert)} new records")
# 分批插入，每批500条（避免SQL过长）
batch_size = 500
for i in range(0, len(to_insert), batch_size):
batch = to_insert[i:i + batch_size]
try:
section_data_list = [
SectionData(
section_id=str(item.get('section_id')), # 统一转换为字符串
mileage=item.get('mileage'),
work_site=item.get('work_site'),
basic_types=item.get('basic_types'),
height=item.get('height'),
status=item.get('status'),
number=str(item.get('number')) if item.get('number') else None, # 统一转换为字符串
transition_paragraph=item.get('transition_paragraph'),
design_fill_height=item.get('design_fill_height'),
compression_layer_thickness=item.get('compression_layer_thickness'),
treatment_depth=item.get('treatment_depth'),
foundation_treatment_method=item.get('foundation_treatment_method'),
rock_mass_classification=item.get('rock_mass_classification'),
account_id=str(item.get('account_id')) if item.get('account_id') else None # 统一转换为字符串
)
for item in batch
]
db.add_all(section_data_list)
success_count += len(batch)
logger.info(f"Inserted batch {i//batch_size + 1}: {len(batch)} records")
except Exception as e:
failed_count += len(batch)
failed_items.extend([
{
'data': item,
'error': f'插入失败: {str(e)}'
}
for item in batch
])
logger.error(f"Failed to insert batch: {str(e)}")
raise e
# 如果有失败记录，不提交事务
if failed_items:
db.rollback()
return {
'success': False,
'message': f'批量导入失败: {len(failed_items)}条记录处理失败',
'total_count': total_count,
'success_count': success_count,
'failed_count': failed_count,
'failed_items': failed_items
}
db.commit()
logger.info(f"Batch import sections completed. Success: {success_count}, Failed: {failed_count}")