403 lines
18 KiB
Python
403 lines
18 KiB
Python
from sqlalchemy.orm import Session
|
||
from typing import List, Optional, Dict, Any
|
||
from ..models.section_data import SectionData
|
||
from .base import BaseService
|
||
from .checkpoint import CheckpointService
|
||
from .settlement_data import SettlementDataService
|
||
from .level_data import LevelDataService
|
||
from .original_data import OriginalDataService
|
||
from typing import Dict
|
||
|
||
class SectionDataService(BaseService[SectionData]):
|
||
def __init__(self):
|
||
super().__init__(SectionData)
|
||
self.checkpoint_service = CheckpointService()
|
||
self.settlement_service = SettlementDataService()
|
||
self.level_service = LevelDataService()
|
||
self.original_service = OriginalDataService()
|
||
|
||
def get_by_section_id(self, db: Session, section_id: str) -> Optional[SectionData]:
|
||
"""根据断面ID获取断面数据"""
|
||
sections = self.get_by_field(db, "section_id", section_id)
|
||
return sections[0] if sections else None
|
||
def get_by_account_id(self, db: Session, account_id: str) -> List[SectionData]:
|
||
"""根据账号ID获取断面数据"""
|
||
accounts = self.get_by_field(db, "account_id", account_id)
|
||
return accounts if accounts else []
|
||
|
||
def get_by_account_id_batch(self, db: Session, account_ids: List[str]) -> List[SectionData]:
|
||
"""批量根据账号ID列表获取断面数据（使用IN查询优化性能）"""
|
||
if not account_ids:
|
||
return []
|
||
return db.query(SectionData).filter(SectionData.account_id.in_(account_ids)).all()
|
||
|
||
def get_by_section_ids_batch(self, db: Session, section_ids: List[str]) -> List[SectionData]:
|
||
"""批量根据断面ID列表获取断面数据（使用IN查询优化性能）"""
|
||
if not section_ids:
|
||
return []
|
||
return db.query(SectionData).filter(SectionData.section_id.in_(section_ids)).all()
|
||
|
||
def get_by_number(self, db: Session, number: str) -> List[SectionData]:
|
||
"""根据桥梁墩(台)编号获取断面数据"""
|
||
return self.get_by_field(db, "number", number)
|
||
|
||
def search_section_data(self, db: Session,
|
||
id: Optional[int] = None,
|
||
section_id: Optional[str] = None,
|
||
mileage: Optional[str] = None,
|
||
work_site: Optional[str] = None,
|
||
number: Optional[str] = None,
|
||
status: Optional[str] = None,
|
||
basic_types: Optional[str] = None,
|
||
account_id: Optional[str] = None,
|
||
skip: int = 0,
|
||
limit: Optional[int] = None) -> List[SectionData]:
|
||
"""根据多个条件搜索断面数据"""
|
||
conditions = {}
|
||
if section_id is not None:
|
||
conditions["section_id"] = section_id
|
||
if work_site is not None:
|
||
conditions["work_site"] = work_site
|
||
if number is not None:
|
||
conditions["number"] = number
|
||
if status is not None:
|
||
conditions["status"] = status
|
||
if basic_types is not None:
|
||
conditions["basic_types"] = basic_types
|
||
if id is not None:
|
||
conditions['id'] = id
|
||
if mileage is not None:
|
||
conditions['mileage'] = mileage
|
||
if account_id is not None:
|
||
conditions['account_id'] = account_id
|
||
|
||
return self.search_by_conditions(db, conditions, skip, limit)
|
||
|
||
def search_sections_with_checkpoints(self, db: Session,
|
||
id: Optional[int] = None,
|
||
section_id: Optional[str] = None,
|
||
mileage: Optional[str] = None,
|
||
work_site: Optional[str] = None,
|
||
number: Optional[str] = None,
|
||
status: Optional[str] = None,
|
||
account_id: Optional[str] = None,
|
||
skip: int = 0,
|
||
limit: Optional[int] = None) -> Dict[str, Any]:
|
||
"""查询断面数据并返回带观测点的结果（支持分页）"""
|
||
# 构建查询条件
|
||
conditions = {}
|
||
if section_id is not None:
|
||
conditions["section_id"] = section_id
|
||
if work_site is not None:
|
||
conditions["work_site"] = work_site
|
||
if number is not None:
|
||
conditions["number"] = number
|
||
if status is not None:
|
||
conditions["status"] = status
|
||
if id is not None:
|
||
conditions['id'] = id
|
||
if mileage is not None:
|
||
conditions['mileage'] = mileage
|
||
if account_id is not None:
|
||
conditions['account_id'] = account_id
|
||
|
||
# 获取总数
|
||
total_count = self.search_by_conditions_count(db, conditions)
|
||
|
||
# 获取分页数据
|
||
sections = self.search_by_conditions(db, conditions, skip, limit)
|
||
|
||
result = []
|
||
for section in sections:
|
||
checkpoints = self.checkpoint_service.get_by_section_id(db, section.section_id)
|
||
|
||
section_dict = {
|
||
"id": section.id,
|
||
"section_id": section.section_id,
|
||
"mileage": section.mileage,
|
||
"work_site": section.work_site,
|
||
"basic_types": section.basic_types,
|
||
"height": section.height,
|
||
"status": section.status,
|
||
"number": section.number,
|
||
"transition_paragraph": section.transition_paragraph,
|
||
"design_fill_height": section.design_fill_height,
|
||
"compression_layer_thickness": section.compression_layer_thickness,
|
||
"treatment_depth": section.treatment_depth,
|
||
"foundation_treatment_method": section.foundation_treatment_method,
|
||
"rock_mass_classification": section.rock_mass_classification,
|
||
"account_id": section.account_id,
|
||
"checkpoints": [
|
||
{
|
||
"id": cp.id,
|
||
"point_id": cp.point_id,
|
||
"aname": cp.aname,
|
||
"burial_date": cp.burial_date,
|
||
"section_id": cp.section_id
|
||
} for cp in checkpoints
|
||
]
|
||
}
|
||
result.append(section_dict)
|
||
|
||
return {
|
||
"data": result,
|
||
"total": total_count,
|
||
"skip": skip,
|
||
"limit": limit
|
||
}
|
||
|
||
def get_section_with_checkpoints(self, db: Session, section_id: str) -> Dict[str, Any]:
|
||
"""获取断面数据及其关联的观测点"""
|
||
section = self.get_by_section_id(db, section_id)
|
||
if not section:
|
||
return {}
|
||
|
||
checkpoints = self.checkpoint_service.get_by_section_id(db, section_id)
|
||
|
||
return {
|
||
"section": section,
|
||
"checkpoints": checkpoints,
|
||
"checkpoint_count": len(checkpoints)
|
||
}
|
||
|
||
def get_section_with_all_data(self, db: Session, section_id: str) -> Dict[str, Any]:
|
||
"""获取断面数据及其所有关联数据（观测点、沉降数据、原始数据）"""
|
||
section = self.get_by_section_id(db, section_id)
|
||
if not section:
|
||
return {}
|
||
|
||
checkpoints = self.checkpoint_service.get_by_section_id(db, section_id)
|
||
|
||
all_settlement_data = []
|
||
all_original_data = []
|
||
all_level_data = []
|
||
|
||
for checkpoint in checkpoints:
|
||
point_id = checkpoint.point_id
|
||
|
||
settlement_data = self.settlement_service.get_by_point_id(db, point_id)
|
||
all_settlement_data.extend(settlement_data)
|
||
|
||
for settlement in settlement_data:
|
||
nyid = settlement.NYID
|
||
|
||
level_data = self.level_service.get_by_nyid(db, nyid)
|
||
all_level_data.extend(level_data)
|
||
|
||
original_data = self.original_service.get_by_nyid(db, nyid)
|
||
all_original_data.extend(original_data)
|
||
|
||
return {
|
||
"section": section,
|
||
"checkpoints": checkpoints,
|
||
"settlement_data": all_settlement_data,
|
||
"level_data": all_level_data,
|
||
"original_data": all_original_data,
|
||
"summary": {
|
||
"checkpoint_count": len(checkpoints),
|
||
"settlement_data_count": len(all_settlement_data),
|
||
"level_data_count": len(all_level_data),
|
||
"original_data_count": len(all_original_data)
|
||
}
|
||
}
|
||
|
||
def get_sections_by_checkpoint_point_id(self, db: Session, point_id: str) -> List[SectionData]:
|
||
"""根据观测点ID反向查找断面数据"""
|
||
checkpoint = self.checkpoint_service.get_by_point_id(db, point_id)
|
||
if checkpoint:
|
||
return [self.get_by_section_id(db, checkpoint.section_id)]
|
||
return []
|
||
|
||
def get_sections_by_settlement_nyid(self, db: Session, nyid: str) -> List[SectionData]:
|
||
"""根据期数ID反向查找相关的断面数据"""
|
||
settlement_data = self.settlement_service.get_by_nyid(db, nyid)
|
||
sections = []
|
||
|
||
for settlement in settlement_data:
|
||
point_id = settlement.point_id
|
||
checkpoint = self.checkpoint_service.get_by_point_id(db, point_id)
|
||
if checkpoint:
|
||
section = self.get_by_section_id(db, checkpoint.section_id)
|
||
if section and section not in sections:
|
||
sections.append(section)
|
||
|
||
return sections
|
||
|
||
def get_settlement_data_by_section(self, db: Session, section_id: str) -> List:
|
||
"""获取指定断面的所有沉降数据"""
|
||
checkpoints = self.checkpoint_service.get_by_section_id(db, section_id)
|
||
all_settlement_data = []
|
||
|
||
for checkpoint in checkpoints:
|
||
settlement_data = self.settlement_service.get_by_point_id(db, checkpoint.point_id)
|
||
all_settlement_data.extend(settlement_data)
|
||
|
||
return all_settlement_data
|
||
|
||
def get_original_data_by_section(self, db: Session, section_id: str) -> List:
|
||
"""获取指定断面的所有原始数据"""
|
||
settlement_data = self.get_settlement_data_by_section(db, section_id)
|
||
all_original_data = []
|
||
|
||
for settlement in settlement_data:
|
||
original_data = self.original_service.get_by_nyid(db, settlement.NYID)
|
||
all_original_data.extend(original_data)
|
||
|
||
return all_original_data
|
||
|
||
def get_level_data_by_section(self, db: Session, section_id: str) -> List:
|
||
"""获取指定断面的所有水准数据"""
|
||
settlement_data = self.get_settlement_data_by_section(db, section_id)
|
||
all_level_data = []
|
||
|
||
for settlement in settlement_data:
|
||
level_data = self.level_service.get_by_nyid(db, settlement.NYID)
|
||
all_level_data.extend(level_data)
|
||
|
||
return all_level_data
|
||
|
||
def batch_import_sections(self, db: Session, data: List) -> Dict[str, Any]:
|
||
"""
|
||
批量导入断面数据 - 性能优化版
|
||
使用批量查询和批量操作，大幅提升导入速度
|
||
根据断面ID判断是否重复，重复数据跳过，不进行更新操作
|
||
支持事务回滚，失败时重试一次
|
||
"""
|
||
import logging
|
||
logger = logging.getLogger(__name__)
|
||
|
||
total_count = len(data)
|
||
success_count = 0
|
||
failed_count = 0
|
||
failed_items = []
|
||
|
||
if total_count == 0:
|
||
return {
|
||
'success': False,
|
||
'message': '导入数据不能为空',
|
||
'total_count': 0,
|
||
'success_count': 0,
|
||
'failed_count': 0,
|
||
'failed_items': []
|
||
}
|
||
|
||
for attempt in range(2): # 最多重试1次
|
||
try:
|
||
db.begin()
|
||
success_count = 0
|
||
failed_count = 0
|
||
failed_items = []
|
||
|
||
# ===== 性能优化1：批量查询现有断面数据（IN查询） =====
|
||
# 统一转换为字符串处理（数据库section_id字段是VARCHAR类型）
|
||
section_id_list = list(set(str(item.get('section_id')) for item in data if item.get('section_id')))
|
||
logger.info(f"Checking {len(section_id_list)} unique section_ids")
|
||
existing_sections = db.query(SectionData).filter(SectionData.section_id.in_(section_id_list)).all()
|
||
|
||
# 使用section_id创建查找表
|
||
existing_map = {
|
||
section.section_id: section
|
||
for section in existing_sections
|
||
}
|
||
logger.info(f"Found {len(existing_sections)} existing sections")
|
||
|
||
# ===== 性能优化2：批量处理插入和跳过 =====
|
||
to_insert = []
|
||
|
||
for item_data in data:
|
||
section_id = str(item_data.get('section_id')) # 统一转换为字符串
|
||
|
||
if section_id in existing_map:
|
||
# 数据已存在，跳过
|
||
logger.info(f"Continue section data: {section_id}")
|
||
failed_count += 1
|
||
failed_items.append({
|
||
'data': item_data,
|
||
'error': '数据已存在，跳过插入操作'
|
||
})
|
||
else:
|
||
# 记录需要插入的数据
|
||
to_insert.append(item_data)
|
||
|
||
# ===== 执行批量插入 =====
|
||
if to_insert:
|
||
logger.info(f"Inserting {len(to_insert)} new records")
|
||
# 分批插入，每批500条（避免SQL过长）
|
||
batch_size = 500
|
||
for i in range(0, len(to_insert), batch_size):
|
||
batch = to_insert[i:i + batch_size]
|
||
try:
|
||
section_data_list = [
|
||
SectionData(
|
||
section_id=str(item.get('section_id')), # 统一转换为字符串
|
||
mileage=item.get('mileage'),
|
||
work_site=item.get('work_site'),
|
||
basic_types=item.get('basic_types'),
|
||
height=item.get('height'),
|
||
status=item.get('status'),
|
||
number=str(item.get('number')) if item.get('number') else None, # 统一转换为字符串
|
||
transition_paragraph=item.get('transition_paragraph'),
|
||
design_fill_height=item.get('design_fill_height'),
|
||
compression_layer_thickness=item.get('compression_layer_thickness'),
|
||
treatment_depth=item.get('treatment_depth'),
|
||
foundation_treatment_method=item.get('foundation_treatment_method'),
|
||
rock_mass_classification=item.get('rock_mass_classification'),
|
||
account_id=str(item.get('account_id')) if item.get('account_id') else None # 统一转换为字符串
|
||
)
|
||
for item in batch
|
||
]
|
||
db.add_all(section_data_list)
|
||
success_count += len(batch)
|
||
logger.info(f"Inserted batch {i//batch_size + 1}: {len(batch)} records")
|
||
except Exception as e:
|
||
failed_count += len(batch)
|
||
failed_items.extend([
|
||
{
|
||
'data': item,
|
||
'error': f'插入失败: {str(e)}'
|
||
}
|
||
for item in batch
|
||
])
|
||
logger.error(f"Failed to insert batch: {str(e)}")
|
||
raise e
|
||
|
||
# 如果有插入失败记录（不是跳过记录），不提交事务
|
||
# 跳过记录不应该影响事务，只插入失败的记录才需要回滚
|
||
insert_failed_items = [item for item in failed_items if '插入失败' in item.get('error', '')]
|
||
if insert_failed_items:
|
||
db.rollback()
|
||
return {
|
||
'success': False,
|
||
'message': f'批量导入失败: {len(insert_failed_items)}条记录插入失败',
|
||
'total_count': total_count,
|
||
'success_count': success_count,
|
||
'failed_count': failed_count,
|
||
'failed_items': failed_items
|
||
}
|
||
|
||
db.commit()
|
||
logger.info(f"Batch import sections completed. Success: {success_count}, Failed: {failed_count}")
|
||
break
|
||
|
||
except Exception as e:
|
||
db.rollback()
|
||
logger.warning(f"Batch import attempt {attempt + 1} failed: {str(e)}")
|
||
if attempt == 1: # 最后一次重试失败
|
||
logger.error("Batch import sections failed after retries")
|
||
return {
|
||
'success': False,
|
||
'message': f'批量导入失败: {str(e)}',
|
||
'total_count': total_count,
|
||
'success_count': 0,
|
||
'failed_count': total_count,
|
||
'failed_items': failed_items
|
||
}
|
||
|
||
return {
|
||
'success': True,
|
||
'message': '批量导入完成' if failed_count == 0 else f'部分导入失败',
|
||
'total_count': total_count,
|
||
'success_count': success_count,
|
||
'failed_count': failed_count,
|
||
'failed_items': failed_items
|
||
} |