From f6b9e649f439c4d231755040abffa7c28fb5832b Mon Sep 17 00:00:00 2001 From: lhx Date: Wed, 19 Nov 2025 18:23:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B0=B4=E5=87=86=E6=95=B0=E6=8D=AE=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E4=B8=80=E4=BA=9B=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/section_data.py | 8 +++--- upload_app/insert_all.py | 42 ++++++++++++++++++++------------ upload_app/insert_data_online.py | 41 +++++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/app/models/section_data.py b/app/models/section_data.py index 3c49473..3e09e29 100644 --- a/app/models/section_data.py +++ b/app/models/section_data.py @@ -5,12 +5,12 @@ class SectionData(Base): __tablename__ = "section_data" id = Column(Integer, primary_key=True, index=True, autoincrement=True) - mileage = Column(String(100), nullable=False, comment="断面里程") - work_site = Column(String(100), nullable=False, comment="工点") + mileage = Column(String(100), nullable=True, comment="断面里程") + work_site = Column(String(100), nullable=True, comment="工点") basic_types = Column(String(100), comment="基础类型") height = Column(String(100), comment="桥墩台高度") - status = Column(String(100), nullable=False, comment="断面状态") - number = Column(String(100), nullable=False, comment="所在桥梁墩(台)编号", index=True) + status = Column(String(100), nullable=True, comment="断面状态") + number = Column(String(100), nullable=True, comment="所在桥梁墩(台)编号", index=True) transition_paragraph = Column(String(100), comment="过渡段") design_fill_height = Column(String(100), comment="设计填土高度") compression_layer_thickness = Column(String(100), comment="压实层厚度") diff --git a/upload_app/insert_all.py b/upload_app/insert_all.py index f64dbef..64745a0 100644 --- a/upload_app/insert_all.py +++ b/upload_app/insert_all.py @@ -10,6 +10,16 @@ from insert_data_online import ( batch_import_level_data, batch_import_original_data ) +import logging +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + filename='tools.log', + filemode='a', + encoding='utf-8' +) +logger = logging.getLogger(__name__) # ------------------------------ 核心配置 ------------------------------ DATA_TYPE_MAPPING = { @@ -286,22 +296,22 @@ def batch_import(data_list, data_type, settlement_nyids=None, progress=None): print(f"\n=== [入库] {data_type} 第 {batch_num} 批（共{total}条，当前{batch_len}条）===") # 水准数据过滤：仅保留沉降已存在的NYID - # if data_type == "level" and settlement_nyids is not None: - # valid_batch = [ - # item for item in batch_data - # if str(item.get("NYID", "")) in settlement_nyids - # ] - # invalid_count = batch_len - len(valid_batch) - # if invalid_count > 0: - # print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）") - # batch_data = valid_batch - # batch_len = len(batch_data) - # if batch_len == 0: - # print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过") - # # 标记为空批次已处理 - # progress["processed_batches"][data_type].append(batch_num) - # save_progress(progress) - # continue + if data_type == "level" and settlement_nyids is not None: + valid_batch = [ + item for item in batch_data + if str(item.get("NYID", "")) in settlement_nyids + ] + invalid_count = batch_len - len(valid_batch) + if invalid_count > 0: + print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）") + batch_data = valid_batch + batch_len = len(batch_data) + if batch_len == 0: + print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过") + # 标记为空批次已处理 + progress["processed_batches"][data_type].append(batch_num) + save_progress(progress) + continue # 重试机制 retry_count = 0 diff --git a/upload_app/insert_data_online.py b/upload_app/insert_data_online.py index 44a903b..45de0ff 100644 --- a/upload_app/insert_data_online.py +++ b/upload_app/insert_data_online.py @@ -4,6 +4,18 @@ from datetime import datetime import time import random # 新增：用于随机选择User-Agent +import logging + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + filename='tools.log', + filemode='a', + encoding='utf-8' +) +logger = logging.getLogger(__name__) + # 全局常见PC端User-Agent列表（包含Chrome、Firefox、Edge等主流浏览器） USER_AGENTS = [ # Chrome @@ -35,7 +47,7 @@ def save_point_times(point_id, point_times): # 批量导入断面数据 def batch_import_sections(data_list): """批量导入断面数据到指定API""" - url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_sections" + url = "http://localhost:8000/api/comprehensive_data/batch_import_sections" # 数据格式校验 for index, item in enumerate(data_list): @@ -89,7 +101,7 @@ def batch_import_sections(data_list): # 批量导入测点数据 def batch_import_checkpoints(data_list): """批量导入检查点数据到指定API""" - url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_checkpoints" + url = "http://localhost:8000/api/comprehensive_data/batch_import_checkpoints" # 构建请求体 payload = json.dumps({"data": data_list}) @@ -99,7 +111,7 @@ def batch_import_checkpoints(data_list): 'User-Agent': random.choice(USER_AGENTS), # 核心修改 'Content-Type': 'application/json', 'Accept': '*/*', - 'Host': 'www.yuxindazhineng.com:3002', + # 'Host': 'www.yuxindazhineng.com:3002', 'Connection': 'keep-alive' } @@ -121,7 +133,7 @@ def batch_import_checkpoints(data_list): # 导入沉降数据 def batch_import_settlement_data(settlement_data_list): """批量导入沉降数据到指定API接口""" - api_url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_settlement_data" + api_url = "http://localhost:8000/api/comprehensive_data/batch_import_settlement_data" request_payload = json.dumps({"data": settlement_data_list}) @@ -130,7 +142,7 @@ def batch_import_settlement_data(settlement_data_list): 'User-Agent': random.choice(USER_AGENTS), # 核心修改 'Content-Type': 'application/json', 'Accept': '*/*', - 'Host': 'www.yuxindazhineng.com:3002', + # 'Host': 'www.yuxindazhineng.com:3002', 'Connection': 'keep-alive' } @@ -157,7 +169,7 @@ def batch_import_settlement_data(settlement_data_list): # 导入水准数据 def batch_import_level_data(data_list): """批量导入层级数据到指定API""" - url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_level_data" + url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_level_data" payload = json.dumps({"data": data_list}) @@ -166,7 +178,7 @@ def batch_import_level_data(data_list): 'User-Agent': random.choice(USER_AGENTS), # 核心修改 'Content-Type': 'application/json', 'Accept': '*/*', - 'Host': 'www.yuxindazhineng.com:3002', + # 'Host': 'www.yuxindazhineng.com:3002', 'Connection': 'keep-alive' } @@ -186,7 +198,8 @@ def batch_import_level_data(data_list): # 插入原始数据 def batch_import_original_data(data_list): """批量导入原始数据到指定API""" - url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data" + # url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data" + url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_original_data" # 校验数据格式 for i, item in enumerate(data_list): @@ -213,14 +226,22 @@ def batch_import_original_data(data_list): 'User-Agent': random.choice(USER_AGENTS), # 核心修改 'Content-Type': 'application/json', 'Accept': '*/*', - # 'Host': 'www.yuxindazhineng.com:3002', - 'Host': '127.0.0.1:8000', + 'Host': 'www.yuxindazhineng.com:3002', + # 'Host': '127.0.0.1:8000', 'Connection': 'keep-alive' } try: + start_time = time.time() + print(f'headers:{time.time()}') + logger.info(f'开始时间:{start_time}') response = requests.post(url, headers=headers, data=payload, timeout=60) response.raise_for_status() + end_time = time.time() + print(f'结束时间:{end_time}秒') + logger.info(f'结束时间:{end_time}') + print(f'耗时:{end_time - start_time}秒') + logger.info(f'耗时:{end_time - start_time}秒') return True, response.text except requests.exceptions.HTTPError as e: print(f'http_error:{e}{time.time()}')