admin/railway_cloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

水准数据模型一些修改

Browse Source
This commit is contained in:
lhx
2025-11-19 18:23:30 +08:00
parent e19ea35622
commit f6b9e649f4
3 changed files with 61 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/models/section_data.py
View File

@@ -5,12 +5,12 @@ class SectionData(Base):
__tablename__ = "section_data" __tablename__ = "section_data"
id = Column(Integer, primary_key=True, index=True, autoincrement=True) id = Column(Integer, primary_key=True, index=True, autoincrement=True)
mileage = Column(String(100), nullable=False, comment="断面里程") mileage = Column(String(100), nullable=True, comment="断面里程")
work_site = Column(String(100), nullable=False, comment="工点") work_site = Column(String(100), nullable=True, comment="工点")
basic_types = Column(String(100), comment="基础类型") basic_types = Column(String(100), comment="基础类型")
height = Column(String(100), comment="桥墩台高度") height = Column(String(100), comment="桥墩台高度")
status = Column(String(100), nullable=False, comment="断面状态") status = Column(String(100), nullable=True, comment="断面状态")
number = Column(String(100), nullable=False, comment="所在桥梁墩(台)编号", index=True) number = Column(String(100), nullable=True, comment="所在桥梁墩(台)编号", index=True)
transition_paragraph = Column(String(100), comment="过渡段") transition_paragraph = Column(String(100), comment="过渡段")
design_fill_height = Column(String(100), comment="设计填土高度") design_fill_height = Column(String(100), comment="设计填土高度")
compression_layer_thickness = Column(String(100), comment="压实层厚度") compression_layer_thickness = Column(String(100), comment="压实层厚度")

42
upload_app/insert_all.py
View File

@@ -10,6 +10,16 @@ from insert_data_online import (
batch_import_level_data, batch_import_level_data,
batch_import_original_data batch_import_original_data
) )
import logging
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
filename='tools.log',
filemode='a',
encoding='utf-8'
)
logger = logging.getLogger(__name__)
# ------------------------------ 核心配置 ------------------------------ # ------------------------------ 核心配置 ------------------------------
DATA_TYPE_MAPPING = { DATA_TYPE_MAPPING = {
@@ -286,22 +296,22 @@ def batch_import(data_list, data_type, settlement_nyids=None, progress=None):
print(f"\n=== [入库] {data_type} 第 {batch_num} 批（共{total}条，当前{batch_len}条）===") print(f"\n=== [入库] {data_type} 第 {batch_num} 批（共{total}条，当前{batch_len}条）===")
# 水准数据过滤：仅保留沉降已存在的NYID # 水准数据过滤：仅保留沉降已存在的NYID
# if data_type == "level" and settlement_nyids is not None: if data_type == "level" and settlement_nyids is not None:
# valid_batch = [ valid_batch = [
# item for item in batch_data item for item in batch_data
# if str(item.get("NYID", "")) in settlement_nyids if str(item.get("NYID", "")) in settlement_nyids
# ] ]
# invalid_count = batch_len - len(valid_batch) invalid_count = batch_len - len(valid_batch)
# if invalid_count > 0: if invalid_count > 0:
# print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）") print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）")
# batch_data = valid_batch batch_data = valid_batch
# batch_len = len(batch_data) batch_len = len(batch_data)
# if batch_len == 0: if batch_len == 0:
# print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过") print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过")
# # 标记为空批次已处理 # 标记为空批次已处理
# progress["processed_batches"][data_type].append(batch_num) progress["processed_batches"][data_type].append(batch_num)
# save_progress(progress) save_progress(progress)
# continue continue
# 重试机制 # 重试机制
retry_count = 0 retry_count = 0

41
upload_app/insert_data_online.py
View File

@@ -4,6 +4,18 @@ from datetime import datetime
import time import time
import random # 新增：用于随机选择User-Agent import random # 新增：用于随机选择User-Agent
import logging
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
filename='tools.log',
filemode='a',
encoding='utf-8'
)
logger = logging.getLogger(__name__)
# 全局常见PC端User-Agent列表（包含Chrome、Firefox、Edge等主流浏览器） # 全局常见PC端User-Agent列表（包含Chrome、Firefox、Edge等主流浏览器）
USER_AGENTS = [ USER_AGENTS = [
# Chrome # Chrome
@@ -35,7 +47,7 @@ def save_point_times(point_id, point_times):
# 批量导入断面数据 # 批量导入断面数据
def batch_import_sections(data_list): def batch_import_sections(data_list):
"""批量导入断面数据到指定API""" """批量导入断面数据到指定API"""
url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_sections" url = "http://localhost:8000/api/comprehensive_data/batch_import_sections"
# 数据格式校验 # 数据格式校验
for index, item in enumerate(data_list): for index, item in enumerate(data_list):
@@ -89,7 +101,7 @@ def batch_import_sections(data_list):
# 批量导入测点数据 # 批量导入测点数据
def batch_import_checkpoints(data_list): def batch_import_checkpoints(data_list):
"""批量导入检查点数据到指定API""" """批量导入检查点数据到指定API"""
url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_checkpoints" url = "http://localhost:8000/api/comprehensive_data/batch_import_checkpoints"
# 构建请求体 # 构建请求体
payload = json.dumps({"data": data_list}) payload = json.dumps({"data": data_list})
@@ -99,7 +111,7 @@ def batch_import_checkpoints(data_list):
'User-Agent': random.choice(USER_AGENTS), # 核心修改 'User-Agent': random.choice(USER_AGENTS), # 核心修改
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Accept': '*/*', 'Accept': '*/*',
'Host': 'www.yuxindazhineng.com:3002', # 'Host': 'www.yuxindazhineng.com:3002',
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
@@ -121,7 +133,7 @@ def batch_import_checkpoints(data_list):
# 导入沉降数据 # 导入沉降数据
def batch_import_settlement_data(settlement_data_list): def batch_import_settlement_data(settlement_data_list):
"""批量导入沉降数据到指定API接口""" """批量导入沉降数据到指定API接口"""
api_url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_settlement_data" api_url = "http://localhost:8000/api/comprehensive_data/batch_import_settlement_data"
request_payload = json.dumps({"data": settlement_data_list}) request_payload = json.dumps({"data": settlement_data_list})
@@ -130,7 +142,7 @@ def batch_import_settlement_data(settlement_data_list):
'User-Agent': random.choice(USER_AGENTS), # 核心修改 'User-Agent': random.choice(USER_AGENTS), # 核心修改
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Accept': '*/*', 'Accept': '*/*',
'Host': 'www.yuxindazhineng.com:3002', # 'Host': 'www.yuxindazhineng.com:3002',
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
@@ -157,7 +169,7 @@ def batch_import_settlement_data(settlement_data_list):
# 导入水准数据 # 导入水准数据
def batch_import_level_data(data_list): def batch_import_level_data(data_list):
"""批量导入层级数据到指定API""" """批量导入层级数据到指定API"""
url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_level_data" url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_level_data"
payload = json.dumps({"data": data_list}) payload = json.dumps({"data": data_list})
@@ -166,7 +178,7 @@ def batch_import_level_data(data_list):
'User-Agent': random.choice(USER_AGENTS), # 核心修改 'User-Agent': random.choice(USER_AGENTS), # 核心修改
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Accept': '*/*', 'Accept': '*/*',
'Host': 'www.yuxindazhineng.com:3002', # 'Host': 'www.yuxindazhineng.com:3002',
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
@@ -186,7 +198,8 @@ def batch_import_level_data(data_list):
# 插入原始数据 # 插入原始数据
def batch_import_original_data(data_list): def batch_import_original_data(data_list):
"""批量导入原始数据到指定API""" """批量导入原始数据到指定API"""
url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data" # url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data"
url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_original_data"
# 校验数据格式 # 校验数据格式
for i, item in enumerate(data_list): for i, item in enumerate(data_list):
@@ -213,14 +226,22 @@ def batch_import_original_data(data_list):
'User-Agent': random.choice(USER_AGENTS), # 核心修改 'User-Agent': random.choice(USER_AGENTS), # 核心修改
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Accept': '*/*', 'Accept': '*/*',
# 'Host': 'www.yuxindazhineng.com:3002', 'Host': 'www.yuxindazhineng.com:3002',
'Host': '127.0.0.1:8000', # 'Host': '127.0.0.1:8000',
'Connection': 'keep-alive' 'Connection': 'keep-alive'
} }
try: try:
start_time = time.time()
print(f'headers:{time.time()}')
logger.info(f'开始时间:{start_time}')
response = requests.post(url, headers=headers, data=payload, timeout=60) response = requests.post(url, headers=headers, data=payload, timeout=60)
response.raise_for_status() response.raise_for_status()
end_time = time.time()
print(f'结束时间:{end_time}秒')
logger.info(f'结束时间:{end_time}')
print(f'耗时:{end_time - start_time}秒')
logger.info(f'耗时:{end_time - start_time}秒')
return True, response.text return True, response.text
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
print(f'http_error:{e}{time.time()}') print(f'http_error:{e}{time.time()}')