水准数据模型一些修改

2025-11-19 18:23:30 +08:00
parent e19ea35622
commit f6b9e649f4
3 changed files with 61 additions and 30 deletions
--- a/app/models/section_data.py
+++ b/app/models/section_data.py
@@ -5,12 +5,12 @@ class SectionData(Base):
    __tablename__ = "section_data"

    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
-    mileage = Column(String(100), nullable=False, comment="断面里程")
-    work_site = Column(String(100), nullable=False, comment="工点")
+    mileage = Column(String(100), nullable=True, comment="断面里程")
+    work_site = Column(String(100), nullable=True, comment="工点")
    basic_types = Column(String(100), comment="基础类型")
    height = Column(String(100), comment="桥墩台高度")
-    status = Column(String(100), nullable=False, comment="断面状态")
-    number = Column(String(100), nullable=False, comment="所在桥梁墩(台)编号", index=True)
+    status = Column(String(100), nullable=True, comment="断面状态")
+    number = Column(String(100), nullable=True, comment="所在桥梁墩(台)编号", index=True)
    transition_paragraph = Column(String(100), comment="过渡段")
    design_fill_height = Column(String(100), comment="设计填土高度")
    compression_layer_thickness = Column(String(100), comment="压实层厚度")
--- a/upload_app/insert_all.py
+++ b/upload_app/insert_all.py
@@ -10,6 +10,16 @@ from insert_data_online import (
    batch_import_level_data,
    batch_import_original_data
 )
+import logging
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    filename='tools.log',
+    filemode='a',
+    encoding='utf-8'
+)
+logger = logging.getLogger(__name__)

 # ------------------------------ 核心配置 ------------------------------
 DATA_TYPE_MAPPING = {
@@ -286,22 +296,22 @@ def batch_import(data_list, data_type, settlement_nyids=None, progress=None):
        print(f"\n=== [入库] {data_type} 第 {batch_num} 批（共{total}条，当前{batch_len}条）===")

        # 水准数据过滤：仅保留沉降已存在的NYID
-        # if data_type == "level" and settlement_nyids is not None:
-        #     valid_batch = [
-        #         item for item in batch_data
-        #         if str(item.get("NYID", "")) in settlement_nyids
-        #     ]
-        #     invalid_count = batch_len - len(valid_batch)
-        #     if invalid_count > 0:
-        #         print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）")
-        #         batch_data = valid_batch
-        #         batch_len = len(batch_data)
-        #         if batch_len == 0:
-        #             print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过")
-        #             # 标记为空批次已处理
-        #             progress["processed_batches"][data_type].append(batch_num)
-        #             save_progress(progress)
-        #             continue
+        if data_type == "level" and settlement_nyids is not None:
+            valid_batch = [
+                item for item in batch_data
+                if str(item.get("NYID", "")) in settlement_nyids
+            ]
+            invalid_count = batch_len - len(valid_batch)
+            if invalid_count > 0:
+                print(f"[入库] 过滤 {invalid_count} 条无效水准数据（NYID不在沉降列表中）")
+                batch_data = valid_batch
+                batch_len = len(batch_data)
+                if batch_len == 0:
+                    print(f"[入库] 第 {batch_num} 批无有效数据 → 跳过")
+                    # 标记为空批次已处理
+                    progress["processed_batches"][data_type].append(batch_num)
+                    save_progress(progress)
+                    continue

        # 重试机制
        retry_count = 0
--- a/upload_app/insert_data_online.py
+++ b/upload_app/insert_data_online.py
@@ -4,6 +4,18 @@ from datetime import datetime
 import time
 import random  # 新增：用于随机选择User-Agent

+import logging
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    filename='tools.log',
+    filemode='a',
+    encoding='utf-8'
+)
+logger = logging.getLogger(__name__)
+
 # 全局常见PC端User-Agent列表（包含Chrome、Firefox、Edge等主流浏览器）
 USER_AGENTS = [
    # Chrome
@@ -35,7 +47,7 @@ def save_point_times(point_id, point_times):
 # 批量导入断面数据
 def batch_import_sections(data_list):
    """批量导入断面数据到指定API"""
-    url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_sections"
+    url = "http://localhost:8000/api/comprehensive_data/batch_import_sections"

    # 数据格式校验
    for index, item in enumerate(data_list):
@@ -89,7 +101,7 @@ def batch_import_sections(data_list):
 # 批量导入测点数据
 def batch_import_checkpoints(data_list):
    """批量导入检查点数据到指定API"""
-    url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_checkpoints"
+    url = "http://localhost:8000/api/comprehensive_data/batch_import_checkpoints"

    # 构建请求体
    payload = json.dumps({"data": data_list})
@@ -99,7 +111,7 @@ def batch_import_checkpoints(data_list):
        'User-Agent': random.choice(USER_AGENTS),  # 核心修改
        'Content-Type': 'application/json',
        'Accept': '*/*',
-        'Host': 'www.yuxindazhineng.com:3002',
+        # 'Host': 'www.yuxindazhineng.com:3002',
        'Connection': 'keep-alive'
    }

@@ -121,7 +133,7 @@ def batch_import_checkpoints(data_list):
 # 导入沉降数据
 def batch_import_settlement_data(settlement_data_list):
    """批量导入沉降数据到指定API接口"""
-    api_url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_settlement_data"
+    api_url = "http://localhost:8000/api/comprehensive_data/batch_import_settlement_data"

    request_payload = json.dumps({"data": settlement_data_list})

@@ -130,7 +142,7 @@ def batch_import_settlement_data(settlement_data_list):
        'User-Agent': random.choice(USER_AGENTS),  # 核心修改
        'Content-Type': 'application/json',
        'Accept': '*/*',
-        'Host': 'www.yuxindazhineng.com:3002',
+        # 'Host': 'www.yuxindazhineng.com:3002',
        'Connection': 'keep-alive'
    }

@@ -157,7 +169,7 @@ def batch_import_settlement_data(settlement_data_list):
 # 导入水准数据
 def batch_import_level_data(data_list):
    """批量导入层级数据到指定API"""
-    url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_level_data"
+    url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_level_data"

    payload = json.dumps({"data": data_list})

@@ -166,7 +178,7 @@ def batch_import_level_data(data_list):
        'User-Agent': random.choice(USER_AGENTS),  # 核心修改
        'Content-Type': 'application/json',
        'Accept': '*/*',
-        'Host': 'www.yuxindazhineng.com:3002',
+        # 'Host': 'www.yuxindazhineng.com:3002',
        'Connection': 'keep-alive'
    }

@@ -186,7 +198,8 @@ def batch_import_level_data(data_list):
 # 插入原始数据
 def batch_import_original_data(data_list):
    """批量导入原始数据到指定API"""
-    url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data"
+    # url = "http://www.yuxindazhineng.com:3002/api/comprehensive_data/batch_import_original_data"
+    url = "http://127.0.0.1:8000/api/comprehensive_data/batch_import_original_data"

    # 校验数据格式
    for i, item in enumerate(data_list):
@@ -213,14 +226,22 @@ def batch_import_original_data(data_list):
        'User-Agent': random.choice(USER_AGENTS),  # 核心修改
        'Content-Type': 'application/json',
        'Accept': '*/*',
-        # 'Host': 'www.yuxindazhineng.com:3002',
-        'Host': '127.0.0.1:8000',
+        'Host': 'www.yuxindazhineng.com:3002',
+        # 'Host': '127.0.0.1:8000',
        'Connection': 'keep-alive'
    }

    try:
+        start_time = time.time()
+        print(f'headers:{time.time()}')
+        logger.info(f'开始时间:{start_time}')
        response = requests.post(url, headers=headers, data=payload, timeout=60)
        response.raise_for_status()
+        end_time = time.time()
+        print(f'结束时间:{end_time}秒')
+        logger.info(f'结束时间:{end_time}')
+        print(f'耗时:{end_time - start_time}秒')
+        logger.info(f'耗时:{end_time - start_time}秒')
        return True, response.text
    except requests.exceptions.HTTPError as e:
        print(f'http_error:{e}{time.time()}')