直接到数据库脚本

upload_app/direct_import/config.py

@@ -0,0 +1,102 @@
+"""
+批量导入脚本配置文件
+统一管理所有配置参数
+"""
+import os
+from pathlib import Path
+
+# 获取项目根目录
+PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
+
+# 数据文件配置
+DATA_ROOT = PROJECT_ROOT / "upload_app" / "data"  # parquet文件根目录
+DEFAULT_ACCOUNT_ID = 1  # 默认账号ID
+
+# 批次大小配置（根据数据库性能调整）
+BATCH_SIZE = {
+    "section": 500,     # 断面数据批次大小
+    "checkpoint": 500,  # 观测点批次大小
+    "settlement": 500,  # 沉降数据批次大小
+    "level": 500,       # 水准数据批次大小
+    "original": 1000    # 原始数据批次大小（支持更大批次）
+}
+
+# 断点续传配置
+RESUME_PROGRESS_FILE = PROJECT_ROOT / "data_import_progress.json"
+RESUME_ENABLE = False  # 是否开启断点续传
+
+# 数据类型映射（复用现有逻辑）
+DATA_TYPE_MAPPING = {
+    "section": (
+        "断面数据表",
+        "section_",
+        "sections",
+        ["account_id"]
+    ),
+    "checkpoint": (
+        "观测点数据表",
+        "point_",
+        "checkpoints",
+        []
+    ),
+    "settlement": (
+        "沉降数据表",
+        "settlement_",
+        "settlements",
+        []
+    ),
+    "level": (
+        "水准数据表",
+        "level_",
+        "levels",
+        []
+    ),
+    "original": (
+        "原始数据表",
+        "original_",
+        "originals",
+        []
+    )
+}
+
+# 数据依赖顺序（可忽略）
+DATA_TYPE_ORDER = [
+    # ("section", "断面数据"),
+    # ("checkpoint", "观测点数据"),
+    # ("settlement", "沉降数据"),
+    # ("level", "水准数据"),
+    ("original", "原始数据")
+]
+
+# 核心字段校验配置
+CRITICAL_FIELDS = {
+    "section": ["section_id", "account_id", "mileage", "work_site"],
+    "checkpoint": ["point_id", "section_id", "aname", "burial_date"],
+    "settlement": ["NYID", "point_id", "sjName"],
+    "level": ["NYID", "linecode", "wsphigh", "createDate"],
+    "original": ["NYID", "bfpcode", "mtime", "bfpvalue", "sort"]
+}
+
+# 数值型字段强制转换配置
+TYPE_CONVERSION = {
+    "section": {
+        "section_id": int,
+        "account_id": int
+    },
+    "checkpoint": {
+        "point_id": int
+    },
+    "settlement": {
+        "NYID": str  # 沉降NYID转为字符串
+    },
+    "original": {
+        "sort": int
+    }
+}
+
+# 日志配置
+LOG_LEVEL = "INFO"
+LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# 文件过滤配置
+MIN_FILE_SIZE = 1024  # 最小文件大小（字节），过滤空文件