103 lines
2.5 KiB
Python
103 lines
2.5 KiB
Python
"""
|
|
批量导入脚本配置文件
|
|
统一管理所有配置参数
|
|
"""
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# 获取项目根目录
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
|
|
|
|
# 数据文件配置
|
|
DATA_ROOT = PROJECT_ROOT / "upload_app" / "data" # parquet文件根目录
|
|
DEFAULT_ACCOUNT_ID = 1 # 默认账号ID
|
|
|
|
# 批次大小配置（根据数据库性能调整）
|
|
BATCH_SIZE = {
|
|
"section": 500, # 断面数据批次大小
|
|
"checkpoint": 500, # 观测点批次大小
|
|
"settlement": 500, # 沉降数据批次大小
|
|
"level": 500, # 水准数据批次大小
|
|
"original": 1000 # 原始数据批次大小（支持更大批次）
|
|
}
|
|
|
|
# 断点续传配置
|
|
RESUME_PROGRESS_FILE = PROJECT_ROOT / "data_import_progress.json"
|
|
RESUME_ENABLE = False # 是否开启断点续传
|
|
|
|
# 数据类型映射（复用现有逻辑）
|
|
DATA_TYPE_MAPPING = {
|
|
"section": (
|
|
"断面数据表",
|
|
"section_",
|
|
"sections",
|
|
["account_id"]
|
|
),
|
|
"checkpoint": (
|
|
"观测点数据表",
|
|
"point_",
|
|
"checkpoints",
|
|
[]
|
|
),
|
|
"settlement": (
|
|
"沉降数据表",
|
|
"settlement_",
|
|
"settlements",
|
|
[]
|
|
),
|
|
"level": (
|
|
"水准数据表",
|
|
"level_",
|
|
"levels",
|
|
[]
|
|
),
|
|
"original": (
|
|
"原始数据表",
|
|
"original_",
|
|
"originals",
|
|
[]
|
|
)
|
|
}
|
|
|
|
# 数据依赖顺序（可忽略）
|
|
DATA_TYPE_ORDER = [
|
|
# ("section", "断面数据"),
|
|
# ("checkpoint", "观测点数据"),
|
|
# ("settlement", "沉降数据"),
|
|
# ("level", "水准数据"),
|
|
("original", "原始数据")
|
|
]
|
|
|
|
# 核心字段校验配置
|
|
CRITICAL_FIELDS = {
|
|
"section": ["section_id", "account_id", "mileage", "work_site"],
|
|
"checkpoint": ["point_id", "section_id", "aname", "burial_date"],
|
|
"settlement": ["NYID", "point_id", "sjName"],
|
|
"level": ["NYID", "linecode", "wsphigh", "createDate"],
|
|
"original": ["NYID", "bfpcode", "mtime", "bfpvalue", "sort"]
|
|
}
|
|
|
|
# 数值型字段强制转换配置
|
|
TYPE_CONVERSION = {
|
|
"section": {
|
|
"section_id": int,
|
|
"account_id": int
|
|
},
|
|
"checkpoint": {
|
|
"point_id": int
|
|
},
|
|
"settlement": {
|
|
"NYID": str # 沉降NYID转为字符串
|
|
},
|
|
"original": {
|
|
"sort": int
|
|
}
|
|
}
|
|
|
|
# 日志配置
|
|
LOG_LEVEL = "INFO"
|
|
LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
|
|
# 文件过滤配置
|
|
MIN_FILE_SIZE = 1024 # 最小文件大小（字节），过滤空文件
|