diff --git a/.env.example b/.env.example index 106beb5..b2dea34 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ # ============================================================================= -# 生产环境模板（aisite.yuxindazhineng.com） -# 用法：cp .env.example .env 后填入 API Key，再 ./restart.sh -# 密钥勿提交 Git；供应商说明见 ai-service/etc/llm.yaml +# 本地 / 测试环境模板 +# 用法：cp .env.example .env 后填入 API Key，再 ./start.sh 或 ./restart.sh +# 密钥勿提交 Git；生产见 .env.production # ============================================================================= # ---- LLM ---- @@ -9,23 +9,38 @@ LLM_PROVIDER=deepseek DEEPSEEK_API_KEY= MINIMAX_API_KEY= -# 视觉 / 截图还原度：生产关闭 +# 视觉（可选；默认读 llm.yaml 的 vision 段，也可用环境变量覆盖） VISION_PROVIDER=dashscope DASHSCOPE_API_KEY= DASHSCOPE_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 VISION_MODEL=qwen3.6-plus -FIDELITY_LOOP=0 +# AI_CONFIG_PATH=/app/etc/llm.yaml -# ---- 端口（绑 127.0.0.1，由宿主机 Nginx 对外 80/443）---- +# 有截图时：视觉摘录 → 蓝图 → 发布截真页 → 对比打分 +FIDELITY_LOOP=1 +FIDELITY_TARGET=95 +FIDELITY_MAX_ROUNDS=4 +FIDELITY_REAL_SCREEN=1 +PLATFORM_BASE=http://127.0.0.1:8180 +WEB_BASE=http://127.0.0.1:5173 +FIDELITY_USER=demo +FIDELITY_PASSWORD=demo123 +# FIDELITY_SHOT_DIR=E:/project/ai建站/test/_out + +# ---- 同机 Docker 端口（本地测试绑 127.0.0.1）---- AIJZ_WEB_PUBLISH=127.0.0.1:5173 AIJZ_GATEWAY_PUBLISH=127.0.0.1:8180 AIJZ_PLATFORM_PUBLISH=127.0.0.1:8888 AIJZ_AI_PUBLISH=127.0.0.1:8001 AIJZ_PG_PUBLISH=127.0.0.1:5432 +# 宿主机已有 Postgres 时改为：AIJZ_PG_PUBLISH=127.0.0.1:15432 +# 需要局域网直连时改为：AIJZ_WEB_PUBLISH=0.0.0.0:5173 -# ---- 对外域名 + 宿主机 Nginx 反代 ---- -AIJZ_ENABLE_HOST_NGINX=1 -AIJZ_DOMAIN=aisite.yuxindazhineng.com -AIJZ_PUBLIC_BASE_URL=https://aisite.yuxindazhineng.com -# 证书：nginx/aisite.yuxindazhineng.com.pem + .key -# 或 nginx/fullchain.pem + nginx/privkey.pem +# ---- 本地测试：不开宿主机域名 Nginx ---- +AIJZ_ENABLE_HOST_NGINX=0 +# AIJZ_DOMAIN= +AIJZ_PUBLIC_BASE_URL=http://127.0.0.1:8180 +# 本地想跳过 Docker、直接 native：AIJZ_USE_HOST=1 +# AIJZ_USE_HOST=1 +# Docker 半挂时最长空等秒数（进程在但引擎不通时脚本最多等 15s） +# AIJZ_DOCKER_WAIT_SEC=30 diff --git a/ai-service/generation_rules.py b/ai-service/generation_rules.py index 0738fa7..3083cea 100644 --- a/ai-service/generation_rules.py +++ b/ai-service/generation_rules.py @@ -85,6 +85,9 @@ SYSTEM_RULES = """ 其它： - 功能按键写入 pages[].layout.actions + action_labels，原文显示。 +- 业务表 apis.resources[].operations **默认**含 list/get/create/update/delete/**import**/**export**； + list 页 layout.actions **默认**含 create/edit/delete/export/**import**/refresh。 + 需求未提「导入」也要加；仅用户明确禁止导入/导出时才去掉。 - 数据：Excel/CSV 表头→字段；JSON list→表；并行数组先展平再关联。snake_case；高基数不做 enum。 - REST：GET/POST/PUT/DELETE；看板 widgets（kpi/line_chart/status_strip/table/bar/pie）按截图或需求取舍。 - 全量导入：校验失败只记错误，不提前中断。 diff --git a/blueprint/ai/prompt-contract.md b/blueprint/ai/prompt-contract.md index 32f993a..0d78856 100644 --- a/blueprint/ai/prompt-contract.md +++ b/blueprint/ai/prompt-contract.md @@ -59,7 +59,7 @@ Excel 解析侧建议先产出中间结构再喂给 LLM： 2. 一表对应 Excel 主 sheet；多 sheet 才多 entity（≤ 20） 3. 优先 enum：当某列 distinct 值 ≤ 20 且稳定 4. 页面至少包含一个 `list`；若用户提到统计再加 `dashboard` -5. API operations 默认 `list/get/create/update/delete`；提到导入导出再加 +5. API operations 默认 `list/get/create/update/delete/**import**/**export**`；列表页 `layout.actions` 默认含 `import`（及 `export`）。**不要**因需求未提「导入」而省略。仅当用户明确禁止导入/导出时才去掉对应 op/action 6. `security.visibility` 默认 `private`，`row_policies` 默认 `tenant_isolated` ## 与中台的分工 diff --git a/docs/同步表约定.md b/docs/同步表约定.md index b8d8090..1e3584c 100644 --- a/docs/同步表约定.md +++ b/docs/同步表约定.md @@ -35,6 +35,22 @@ CREATE TABLE IF NOT EXISTS orders ( 形态 B 下本机 SQLite 可能不可达：至少 **remote** 须能完成校验。 +### 线上库 A DSN（生产统一 Postgres） + +```text +postgres://user:pass@127.0.0.1:5432/app_online?sslmode=disable +``` + +联调临时可用 SQLite（不宜多 writer）： + +```text +file:E:/data/ajz_remote.db?_pragma=foreign_keys(1) +``` + +使用正斜杠；平台会规范化反斜杠并补 `busy_timeout`。 + +单服务器一条通道即可；多服务器为**每台线上库各建一条通道**，客户端选用对应 `channel_id`。 + ## 宇恒开通（opt-in） ```bash @@ -55,6 +71,9 @@ YXD_SYNC_DBSYNC_TABLES=orders,order_items 推送落到通道 **remote**（线上 A），幂等认客户端 version；平台不连用户本机 SQLite。 +**幂等**：同 version 再 push → `{ success:true, result:{ skipped:true, ... } }`；重复 HTTP 安全，线上不双行。 +**可重试**：remote IO 失败 → `503` + `retryable:true`（勿换 UUID）。 + 宇恒建议环境变量（由对方配置，不在智建仓改）： ```bash diff --git a/docs/数据同步-中间件.md b/docs/数据同步-中间件.md index 0026d15..c910feb 100644 --- a/docs/数据同步-中间件.md +++ b/docs/数据同步-中间件.md @@ -1,6 +1,7 @@ # 跨库数据同步中间件 -支持 **SQLite ↔ MySQL ↔ Postgres**。平台侧 worker 可轮询两端 outbox；**形态 B（本机 agent）** 下由终端 agent 经平台 **push** 写线上 A，平台**不直连用户本机 SQLite**。 +支持方言 **Postgres（生产线上库 A 推荐）** / MySQL / SQLite。 +**形态 B（本机 agent）**：终端 agent 经平台 **push** 写线上 A；平台**不直连用户本机 SQLite**。 冲突策略默认 **自动 LWW（源端覆盖）**；落败写入**平台超级管理员**覆盖日志。公司管理员**无冲突台**。 @@ -13,20 +14,29 @@ | LWW 覆盖审计 | **仅平台超级管理员**；公司 top → 403 | | 数据隔离 | 通道带 `tenant_id`；公司 A 看不到公司 B | -## 推荐场景（松离线 B→A） +## 推荐场景（松离线 B→A，A=Postgres） | 端 | 角色 | |----|------| | **B** | 本机正式库（UUID 主键）；开通且白名单表本地可见 | -| **A** | 线上库；agent 经平台 push 幂等写入 | +| **A** | 线上 **Postgres**（通道 `remote`）；agent 经平台 push 幂等写入 | | **Agent** | 读本机 outbox → `POST /api/v1/agent/sync/channels/:id/push` | 推荐配置： -1. 通道：`local` 描述本机表名单，`remote` = A 的 DSN；方向 **`local_to_remote`**；策略 **`lww_source`**。 -2. 表白名单须 UUID TEXT PK + FK 闭包（保存时校验）。 +1. 通道：`remote.driver=postgres` + DSN；方向 **`local_to_remote`**；策略 **`lww_source`**。 +2. 表白名单须 UUID TEXT/UUID PK + FK 闭包（保存时校验）。 3. 客户端显式 `local_dbsync`（见开通说明）；未开通用户零感。 -4. 怀疑漏数时点 **同步修复（对账）**（有最小间隔限流）；双向通道自动对账约 **15 分钟** 一次。 +4. 怀疑漏数时点 **同步修复（对账）**（有最小间隔限流）。 + +### 单服务器 / 多服务器（同一套 API） + +| 部署 | 配置 | +|------|------| +| 单服务器 | 一条通道指向该 Postgres；一个 `channel_id` | +| 多服务器 | **每库一条通道**（不同 remote DSN）；客户端按库选择 `channel_id` / Binding | + +协议相同，均为「经智建 push」，无需第二套接口。 如何保证**不漏、不多**： @@ -42,22 +52,104 @@ | 项 | 说明 | |----|------| -| 方言 | `sqlite` / `mysql` / `postgres` | +| 方言 | 生产 A：`postgres`；兼容 `mysql`；`sqlite` 仅联调临时 A（不宜多 writer） | | 方向 | 本地→线上（推荐）/ 线上→本地 / 双向 | | 冲突 | 默认 `lww_source`；`lww_target`；`queue` 仅调试（租户不可见） | -| Agent | `GET .../agent/sync/.../whitelist`；`POST .../push` | -| Binding | `GET/POST /api/v1/admin/sync/bindings` | -| 对账 | `POST .../reconcile`（默认最少间隔 300s） | +| Agent | `GET .../agent/sync/.../whitelist`；`POST .../push`（管理员/智能体「数据同步」，或登录用户 + 本人 Binding） | +| Binding | `GET/POST /api/v1/admin/sync/bindings`（同上鉴权；普通用户仅本人） | +| 对账 | `POST .../reconcile`（默认最少间隔 300s；仍需「数据同步」） | | 审计 | `GET /api/v1/platform/dbsync/lww-overrides`（超管） | -持久化：`data/dbsync/channels.json`、`lww_overrides.json`（及遗留 `conflicts.json`）。 +持久化：`data/dbsync/channels.json`、`bindings.json`、`lww_overrides.json`（及遗留 `conflicts.json`）。 + +## 用户自助同步（库级三态） + +终端用户可在客户端对名下库选择仅本地 / 仅线上 / 同步；选 **同步** 后用**登录 JWT**（无需单独「数据同步」管理员权）调 agent API。 + +| 角色 | 范围 | +|------|------| +| 「数据同步」管理员 / 智能体 | 租户内通道；push 不强制 Binding | +| 普通登录用户 | 仅本人 Binding：`channel_id` + `online_db_id`；push **须带** `online_db_id` | + +### 表白名单策略（Z4，已冻结） + +| 项 | 行为 | +|----|------| +| **权限源** | 登录 JWT + 本人 Binding 的 `online_db_id`（不是通道 `tables`） | +| **push** | 接受 Binding 库内**任意表**；表不存在时**自动建表**（TEXT 列 + PK） | +| **通道 tables** | 仅历史兼容 / whitelist 接口展示；**不拒收** | + +管理员不必再为业务表维护白名单。宇恒默认 `YXD_SYNC_ENFORCE_WHITELIST=0`。 + +### 下行 / 仅线上（Z5） + +形态 B：平台**不直连本机 B**，只读线上 A，把行返回给本机 agent 写入 B（建议 `WithApplying` 防回声）。 + +| mode | 用途 | +|------|------| +| `bootstrap` | 全量分页灌库（「仅线上」首启 / 本地空库） | +| `pks` | 只列 A 上主键，客户端 diff 后再 `rows` | +| `rows` | 按 `row_pks` 取行补齐 | + +```http +POST /api/v1/agent/sync/channels/{id}/pull +POST /api/v1/agent/sync/channels/{id}/bootstrap # 强制 mode=bootstrap +``` + +```json +{ + "mode": "bootstrap", + "table": "orders", + "after_pk": "", + "limit": 200, + "online_db_id": "用户自助必填" +} +``` + +循环直到 `result.has_more=false`。单页默认 200、上限 500。 + +## Agent push 幂等与重复投递 + +- **正确性**：同 `(table, row_pk, version)` 已落地 → `result.skipped=true` 且 HTTP 2xx；线上 A **不双行**。 +- **允许**客户端重复 push（例如多 agent / 双 drain）；去重是**客户端优化**（进程内 drain 锁），不是平台必做。 +- 通道 stats：`pushed_applied` / `pushed_skipped` 可观察重推噪声。 +- remote 暂不可达：HTTP **503** + `retryable: true`（agent 可稍后重试，勿换 UUID）。 + +- **Postgres A** 适合正常并发；若联调仍用 SQLite 作 A，请客户端**按库串行 drain**。 +- 平台对 remote 连接做**池化复用 + 通道启动预热**，避免每次 push 冷开 SQLite 触发超时。 + +## 请求日志怎么查 + +| 来源 | 位置 / 接口 | 内容 | +|------|-------------|------| +| Platform HTTP | `.runtime/logs/platform.err.log`（go-zero） | 路径、耗时、状态码；超时曾见 `slowcall(3000ms)` | +| Platform dbsync | 同文件，关键字 `dbsync push` / `dbsync pull` | channel、table、pk、user、outcome、dur、`req=`(X-Request-Id) | +| Gateway | `.runtime/logs/gateway.out.log` | sync 路径与 ≥500：`gateway POST /api/... status=... req=` | +| 审计 API | `GET /api/v1/audit/logs`（需「查看审计」） | `action` 含 `dbsync.push` / `dbsync.pull` / `*.error`，detail JSON 带 `req_id` | + +联调时用响应头 / 请求头 `X-Request-Id` 可在三处日志对齐同一笔请求。 + +## 线上 Postgres DSN 示例 + +```text +postgres://sync_user:pass@127.0.0.1:5432/app_online?sslmode=disable +``` + +联调临时 SQLite A（不推荐生产）： + +```text +file:E:/path/to/remote.db?_pragma=foreign_keys(1) +``` + +勿用反斜杠；平台会自动规范化并补 `busy_timeout`。 ## 控制台用法 -1. **数据同步** → 新建通道（UUID 表白名单）→ 测试 → 保存 → 启动 +1. **数据同步** → 新建通道（remote 选 **postgres**）→ 测试 → 保存 → 启动 2. 默认策略选 **源端覆盖**；方向首期用 **本地 → 线上** 3. 点 **同步修复** 做主键对账（勿连续狂点，有限流） 4. LWW 明细在 **平台工作台**（超管），不在公司同步页 +5. 多服务器：为每台库再新建一条通道，把对应 `channel_id` 发给客户端 开通与迁移：见 [数据同步-开通说明.md](./数据同步-开通说明.md)、[数据同步-迁移手册.md](./数据同步-迁移手册.md)。 @@ -76,13 +168,18 @@ | GET/POST | `/api/v1/admin/sync/bindings` | | GET/POST | `/api/v1/admin/sync/conflicts*` → **403**（已迁超管审计） | -### 本机 Agent +### 本机 Agent / 用户自助 + +鉴权：Bearer = 「数据同步」**或** 已登录人类 JWT（后者须先 `POST .../bindings` 且 push 带本人 `online_db_id`）。 | 方法 | 路径 | |------|------| | GET | `/api/v1/agent/sync/channels/{id}/whitelist` | | POST | `/api/v1/agent/sync/channels/{id}/push` | | POST | `/api/v1/agent/sync/channels/{id}/push/batch` | +| POST | `/api/v1/agent/sync/channels/{id}/pull` | +| POST | `/api/v1/agent/sync/channels/{id}/bootstrap` | +| GET/POST | `/api/v1/admin/sync/bindings`（用户自助亦可） | ### 平台超级管理员 diff --git a/docs/数据同步-开通说明.md b/docs/数据同步-开通说明.md index 9be867f..1e71dba 100644 --- a/docs/数据同步-开通说明.md +++ b/docs/数据同步-开通说明.md @@ -13,19 +13,70 @@ 智建控制台「数据同步」配的是**通道 + 表白名单 + 线上 DSN**；是否走 `local_dbsync` 由客户端环境变量决定，平台**不会**替全员切默认。 +## 线上库 A：统一 Postgres + +| 场景 | 推荐 | +|------|------| +| **生产 A** | **`postgres`**（经智建 agent push 写入；可与平台同机或独立实例） | +| 本机 B | 宇恒侧多为 SQLite（通道 `local` 仅描述表白名单，形态 B 下平台可不连 B） | +| 联调临时 A | 可用 sqlite 文件；**不宜**多 agent 并发写 | + +Postgres DSN 示例： + +```text +postgres://user:pass@127.0.0.1:5432/app_online?sslmode=disable +``` + +## 单服务器 / 多服务器（同一套 API） + +同步**一律经智建平台**（agent → whitelist/push），不因服务器数量改协议。 + +| 部署 | 怎么配 | +|------|--------| +| **单服务器** | 公司一条通道：`remote` 指向该 Postgres；宇恒配置一个 `YXD_SYNC_CHANNEL_ID` | +| **多服务器** | **每台线上库一条通道**（不同 remote DSN）；宇恒按 `local_database_id` / Binding 选用对应 `channel_id`（或多 agent 实例各绑一通道） | + +Binding：`POST /api/v1/admin/sync/bindings` 登记 `local_database_id → online_db_id`，并带上 `channel_id`。 +管理员 Token 或**登录用户 JWT**均可登记；普通用户只能登记/查看本人 Binding。 + +## 模块与本机库（一份表 · Z8） + +| 原则 | 说明 | +|------|------| +| **本机 B 管理面** | 模块业务表应能在宇恒本机库（如「AI建站智能体API」）里增删改查/导入，与普通表同一体验 | +| **上云** | 开通 sync 后经 outbox → agent push → 智能体绑定的线上库 A | +| **智能体绑库** | 控制台「用户管理」为智能体填 `channel_id` / `online_db_id` / `database_name`；「数据同步」可按智能体筛通道 | +| **发布落库** | 智能体 Token 新建发布时，若已设 `database_name`，优先 `database_per_app` 写入该库 | + +不要默认维持「模块只在平台 `/apps` schema、松离线又一套 SQLite」双轨；对账时本机有、线上无 = 尚未 push，不是串库。 + +## 用户自助（库级三态） + +客户端对每个库可选：仅本地 / 仅线上 / 同步。选同步后： + +1. 用户 JWT → `POST /api/v1/admin/sync/bindings`（带 `channel_id`） +2. 同 JWT → `GET .../agent/sync/channels/{id}/whitelist` +3. 同 JWT → `POST .../push`，Body 须含本人 `online_db_id` + +通道表白名单**非空**时只同步命中表；**空名单**时平台接受整库表名（UUID PK）。 +「仅线上 / 下行拉齐」：`POST .../agent/sync/channels/{id}/pull`（`mode=bootstrap|pks|rows`）或 `.../bootstrap`；平台返回 A 上行，客户端写入本机 B。 +**空表也要两侧建齐**：本机空表 → `POST .../schema/ensure`；线上空表 → `POST .../schema`（或 pull 的 `columns`）在本机 `CREATE IF NOT EXISTS`。仅靠 outbox 行 push **不会**带上空表。 +控制台「查看线上表」可 **删表**（`POST .../admin/sync/channels/{id}/drop-table`）：只删当前查看侧，**不同步** `DROP` 到另一侧；本机仍有同名表时下次 push/ensure 可能再建回来。 + ## 开通步骤（增值） -1. 公司管理员在「数据同步」建通道：`local`（本机 B 描述）+ `remote`（线上 A DSN），表白名单须 **UUID TEXT PK** + FK 闭包。 -2. 默认方向 **本地 → 线上**，冲突策略 **源端覆盖（lww_source）**。 -3. 客户端显式设 `YXD_SYNC_MODE=local_dbsync`，并配置通道 ID / token（见 `同步表约定.md`）。 -4. 装本机 sync agent 后变更才会上云；未装 agent：**本地可保存**，文案须为「需 agent 才上云」。 -5. Binding（可选）：登记 `local_database_id → online_db_id`，见 API `/api/v1/admin/sync/bindings`。 +1. 「数据同步」新建通道：`remote.driver=postgres` + 线上 DSN；表白名单须 **UUID TEXT/UUID PK** + FK 闭包。 +2. 方向 **本地 → 线上**，策略 **源端覆盖（lww_source）**。 +3. 客户端显式 `YXD_SYNC_MODE=local_dbsync`，配置 `YXD_ONLINE_API_BASE`、`YXD_SYNC_CHANNEL_ID`、token（见 `同步表约定.md`）。 +4. 装本机 sync agent 后才上云；未装：**本地可保存**，文案「需 agent 才上云」。 +5. 多库时登记 Binding，避免串库。 ## 谁能看什么 | 角色 | 可见 | |------|------| | 公司管理员 | 通道配置、对账（同步修复）、统计；**无**冲突台 / LWW 覆盖明细 | +| 普通登录用户 | 不可配通道；可自助 Binding + 用登录 JWT push 本人库 | | 平台超级管理员 | LWW 覆盖审计（平台工作台） | | 未开通终端用户 | **零同步文案**，无强制状态条 | diff --git a/gateway/etc/gateway.docker.yaml b/gateway/etc/gateway.docker.yaml index c55bdd5..1a4c37f 100644 --- a/gateway/etc/gateway.docker.yaml +++ b/gateway/etc/gateway.docker.yaml @@ -5,7 +5,7 @@ Port: 8180 PlatformURL: "http://platform:8888" AIURL: "http://ai:8001" RateLimitPerMin: 6000 -ProxyTimeoutSec: 60 +ProxyTimeoutSec: 180 CorsEnable: true JWTEnable: true diff --git a/gateway/etc/gateway.yaml b/gateway/etc/gateway.yaml index 6ec8885..048c326 100644 --- a/gateway/etc/gateway.yaml +++ b/gateway/etc/gateway.yaml @@ -5,7 +5,7 @@ Port: 8180 PlatformURL: "http://127.0.0.1:8888" AIURL: "http://127.0.0.1:8001" RateLimitPerMin: 6000 -ProxyTimeoutSec: 60 +ProxyTimeoutSec: 180 CorsEnable: true JWTEnable: true diff --git a/gateway/internal/config/config.go b/gateway/internal/config/config.go index 88af656..642b266 100644 --- a/gateway/internal/config/config.go +++ b/gateway/internal/config/config.go @@ -1,4 +1,4 @@ -package config +package config type Config struct { Name string `json:",optional"` @@ -7,7 +7,7 @@ type Config struct { PlatformURL string `json:",default=http://127.0.0.1:8888"` AIURL string `json:",default=http://127.0.0.1:8001"` RateLimitPerMin int `json:",default=240"` - ProxyTimeoutSec int `json:",default=60"` + ProxyTimeoutSec int `json:",default=120"` CorsEnable bool `json:",default=true"` // 与中台 Auth.AccessSecret 保持一致 JWTSecret string `json:",optional"` diff --git a/gateway/internal/proxy/director.go b/gateway/internal/proxy/director.go index a9be3ec..495da78 100644 --- a/gateway/internal/proxy/director.go +++ b/gateway/internal/proxy/director.go @@ -1,11 +1,15 @@ package proxy import ( + "context" + "errors" "io" + "log" "net" "net/http" "net/http/httputil" "net/url" + "strconv" "strings" "time" ) @@ -26,36 +30,102 @@ func New(platformURL, aiURL string, timeoutSec int) (*Director, error) { return nil, err } if timeoutSec <= 0 { - timeoutSec = 60 + timeoutSec = 180 } return &Director{Platform: p, AI: a, Timeout: time.Duration(timeoutSec) * time.Second}, nil } +// eofRetryTransport：空闲连接被 platform 关掉时首包 EOF，自动换新连接重试一次。 +type eofRetryTransport struct { + base http.RoundTripper +} + +func (t eofRetryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + base := t.base + if base == nil { + base = http.DefaultTransport + } + resp, err := base.RoundTrip(req) + if err == nil || req.Context().Err() != nil { + return resp, err + } + if !isTransientUpstreamEOF(err) { + return resp, err + } + // Body 可能已部分读；仅在尚未发出/可读失败时安全重试 GET/有 GetBody 的请求 + if req.Body != nil && req.GetBody == nil && req.ContentLength != 0 { + return resp, err + } + if req.GetBody != nil { + body, gerr := req.GetBody() + if gerr != nil { + return resp, err + } + req = req.Clone(req.Context()) + req.Body = body + } + return base.RoundTrip(req) +} + +func isTransientUpstreamEOF(err error) bool { + if err == nil { + return false + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "eof") || + strings.Contains(msg, "connection reset") || + strings.Contains(msg, "broken pipe") || + strings.Contains(msg, "server closed idle connection") +} + func (d *Director) Handler() http.HandlerFunc { - transport := &http.Transport{ + baseTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 10 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 8, + IdleConnTimeout: 30 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: d.Timeout, + // Windows 上 platform 重启后空闲连接易 EOF；关掉 keep-alive 比偶发 502 更稳 + DisableKeepAlives: true, } + transport := eofRetryTransport{base: baseTransport} return func(w http.ResponseWriter, r *http.Request) { target, path := d.route(r.URL.Path) + timeout := d.Timeout + // dbsync agent push/pull：Windows SQLite 冷打开可超过默认超时 + if isDBSyncHeavyPath(path) && timeout < 180*time.Second { + timeout = 180 * time.Second + } + proxy := httputil.NewSingleHostReverseProxy(target) proxy.Transport = transport proxy.FlushInterval = 100 * time.Millisecond proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, err error) { rw.Header().Set("Content-Type", "application/json") - rw.WriteHeader(http.StatusBadGateway) - _, _ = rw.Write([]byte(`{"code":502,"message":"upstream unavailable: ` + escape(err.Error()) + `"}`)) + msg := escape(err.Error()) + code := http.StatusBadGateway + // 客户端/网关超时 → 503，便于 agent 按 retryable 重试（与 platform 语义一致） + if errors.Is(err, context.DeadlineExceeded) || errors.Is(req.Context().Err(), context.DeadlineExceeded) || + strings.Contains(strings.ToLower(err.Error()), "deadline exceeded") || + strings.Contains(strings.ToLower(err.Error()), "context canceled") { + code = http.StatusServiceUnavailable + } + rw.WriteHeader(code) + _, _ = rw.Write([]byte( + `{"code":` + strconv.Itoa(code) + `,"message":"upstream unavailable: ` + msg + `","retryable":true}`, + )) } proxy.ModifyResponse = func(resp *http.Response) error { - // 网关已统一加 CORS；去掉上游重复头，避免浏览器报 Failed to fetch for _, h := range []string{ "Access-Control-Allow-Origin", "Access-Control-Allow-Credentials", @@ -70,7 +140,6 @@ func (d *Director) Handler() http.HandlerFunc { return nil } - // 重写路径 r.URL.Path = path r.Host = target.Host r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) @@ -78,15 +147,46 @@ func (d *Director) Handler() http.HandlerFunc { if r.Header.Get("X-Request-Id") == "" { r.Header.Set("X-Request-Id", newRequestID()) } + reqID := r.Header.Get("X-Request-Id") + started := time.Now() + rec := &statusRecorder{ResponseWriter: w, status: 200} - // 超时上下文 - ctx := r.Context() - proxy.ServeHTTP(w, r.WithContext(ctx)) + ctx, cancel := context.WithTimeout(r.Context(), timeout) + defer cancel() + proxy.ServeHTTP(rec, r.WithContext(ctx)) + if isDBSyncHeavyPath(path) || rec.status >= 500 { + log.Printf("gateway %s %s status=%d dur=%s req=%s", r.Method, path, rec.status, time.Since(started), reqID) + } } } +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (s *statusRecorder) WriteHeader(code int) { + s.status = code + s.ResponseWriter.WriteHeader(code) +} + +func isDBSyncHeavyPath(path string) bool { + if strings.Contains(path, "/agent/sync/") { + return true + } + if !strings.Contains(path, "/admin/sync/") { + return false + } + return strings.Contains(path, "/push") || + strings.HasSuffix(path, "/reconcile") || + strings.HasSuffix(path, "/ingest") || + strings.HasSuffix(path, "/prepare") || + strings.HasSuffix(path, "/start") || + strings.HasSuffix(path, "/pull") || + strings.HasSuffix(path, "/bootstrap") +} + func (d *Director) route(path string) (*url.URL, string) { - // AI：/ai/* → 上游去掉 /ai 前缀 if strings.HasPrefix(path, "/ai/") || path == "/ai" { next := strings.TrimPrefix(path, "/ai") if next == "" { @@ -94,14 +194,12 @@ func (d *Director) route(path string) (*url.URL, string) { } return d.AI, next } - // 生成蓝图：统一 /api/v1/apps/generate（兼容旧冒号路径，避免重复实现） if path == "/api/v1/apps/generate" || strings.HasPrefix(path, "/api/v1/apps/generate?") { return d.AI, path } if strings.HasPrefix(path, "/api/v1/apps:generate") || strings.Contains(path, "/apps%3Agenerate") { return d.AI, "/api/v1/apps/generate" } - // 其余 /api 走中台 return d.Platform, path } @@ -120,13 +218,11 @@ func newRequestID() string { // Health 聚合探测 func (d *Director) Health(w http.ResponseWriter, _ *http.Request) { - type st struct { - Name string `json:"name"` - OK bool `json:"ok"` - Detail string `json:"detail,omitempty"` - } client := &http.Client{Timeout: 2 * time.Second} - out := []st{ + out := []struct { + Name string `json:"name"` + OK bool `json:"ok"` + }{ probe(client, "platform", d.Platform.String()+"/api/v1/auth/login"), probe(client, "ai", d.AI.String()+"/health"), } @@ -158,29 +254,24 @@ func (d *Director) Health(w http.ResponseWriter, _ *http.Request) { } func probe(client *http.Client, name, rawURL string) struct { - Name string `json:"name"` - OK bool `json:"ok"` - Detail string `json:"detail,omitempty"` + Name string `json:"name"` + OK bool `json:"ok"` } { - // login 用 OPTIONS/GET 可能 405；用短超时 HEAD/GET health 风格 req, _ := http.NewRequest(http.MethodGet, rawURL, nil) if name == "platform" { - // 未登录会 401/405 都说明服务活着；连接失败才算挂 req, _ = http.NewRequest(http.MethodPost, rawURL, strings.NewReader(`{}`)) req.Header.Set("Content-Type", "application/json") } resp, err := client.Do(req) if err != nil { return struct { - Name string `json:"name"` - OK bool `json:"ok"` - Detail string `json:"detail,omitempty"` - }{Name: name, OK: false, Detail: err.Error()} + Name string `json:"name"` + OK bool `json:"ok"` + }{Name: name, OK: false} } defer resp.Body.Close() return struct { - Name string `json:"name"` - OK bool `json:"ok"` - Detail string `json:"detail,omitempty"` + Name string `json:"name"` + OK bool `json:"ok"` }{Name: name, OK: true} } diff --git a/platform/etc/platform.docker.yaml b/platform/etc/platform.docker.yaml index 2c61d60..4384757 100644 --- a/platform/etc/platform.docker.yaml +++ b/platform/etc/platform.docker.yaml @@ -1,6 +1,8 @@ Name: platform Host: 0.0.0.0 Port: 8888 +# go-zero 全局请求超时（毫秒）。dbsync push 冷开 SQLite 常 >3s，默认 3000 会直接 503。 +Timeout: 180000 DataSource: "postgres://platform:platform@postgres:5432/platform?sslmode=disable" DryRun: false diff --git a/platform/etc/platform.yaml b/platform/etc/platform.yaml index 704ecff..1f62962 100644 --- a/platform/etc/platform.yaml +++ b/platform/etc/platform.yaml @@ -1,6 +1,8 @@ Name: platform Host: 0.0.0.0 Port: 8888 +# go-zero 全局请求超时（毫秒）。dbsync push 冷开 SQLite 常 >3s，默认 3000 会直接 503。 +Timeout: 180000 DataSource: "postgres://platform:platform@127.0.0.1:5432/platform?sslmode=disable" DryRun: false diff --git a/platform/internal/agentstore/store.go b/platform/internal/agentstore/store.go index b4dd558..2d713ba 100644 --- a/platform/internal/agentstore/store.go +++ b/platform/internal/agentstore/store.go @@ -23,37 +23,47 @@ const ( ) type Account struct { - AgentID int64 `json:"agent_id"` - TenantID int64 `json:"tenant_id"` - Name string `json:"name"` - ClientID string `json:"client_id"` - HostKey string `json:"host_key,omitempty"` - RoleID int64 `json:"role_id,omitempty"` - RoleCode string `json:"role_code,omitempty"` - RoleName string `json:"role_name,omitempty"` - Status string `json:"status"` - Perms []string `json:"permissions"` - AppSlugs []string `json:"app_slugs"` - CreatedBy int64 `json:"created_by"` - CreatedAt time.Time `json:"created_at"` - LastTokenAt *time.Time `json:"last_token_at,omitempty"` + AgentID int64 `json:"agent_id"` + TenantID int64 `json:"tenant_id"` + Name string `json:"name"` + ClientID string `json:"client_id"` + HostKey string `json:"host_key,omitempty"` + RoleID int64 `json:"role_id,omitempty"` + RoleCode string `json:"role_code,omitempty"` + RoleName string `json:"role_name,omitempty"` + Status string `json:"status"` + Perms []string `json:"permissions"` + AppSlugs []string `json:"app_slugs"` + // Z8b：智能体绑定的同步通道 / 线上库（模块与松离线同一目标） + ChannelID string `json:"channel_id,omitempty"` + OnlineDBID string `json:"online_db_id,omitempty"` + DatabaseName string `json:"database_name,omitempty"` + CreatedBy int64 `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + LastTokenAt *time.Time `json:"last_token_at,omitempty"` } type CreateInput struct { - Name string - Perms []string - AppSlugs []string - Status string // 空则 active - HostKey string - RoleID int64 + Name string + Perms []string + AppSlugs []string + Status string // 空则 active + HostKey string + RoleID int64 + ChannelID string + OnlineDBID string + DatabaseName string } type UpdateInput struct { - Name *string - Status *string - Perms *[]string - AppSlugs *[]string - RoleID *int64 + Name *string + Status *string + Perms *[]string + AppSlugs *[]string + RoleID *int64 + ChannelID *string + OnlineDBID *string + DatabaseName *string } type Store interface { @@ -165,17 +175,20 @@ func (s *MemoryStore) createLocked(tenantID, createdBy int64, in CreateInput) (* s.seq++ a := &memAcc{ Account: Account{ - AgentID: s.seq, - TenantID: tenantID, - Name: strings.TrimSpace(in.Name), - ClientID: cid, - HostKey: strings.TrimSpace(in.HostKey), - RoleID: in.RoleID, - Status: st, - Perms: authx.NormalizePerms(uniq(in.Perms)), - AppSlugs: uniq(in.AppSlugs), - CreatedBy: createdBy, - CreatedAt: time.Now().UTC(), + AgentID: s.seq, + TenantID: tenantID, + Name: strings.TrimSpace(in.Name), + ClientID: cid, + HostKey: strings.TrimSpace(in.HostKey), + RoleID: in.RoleID, + Status: st, + Perms: authx.NormalizePerms(uniq(in.Perms)), + AppSlugs: uniq(in.AppSlugs), + ChannelID: strings.TrimSpace(in.ChannelID), + OnlineDBID: strings.TrimSpace(in.OnlineDBID), + DatabaseName: strings.TrimSpace(in.DatabaseName), + CreatedBy: createdBy, + CreatedAt: time.Now().UTC(), }, SecretHash: string(hash), } @@ -214,6 +227,15 @@ func (s *MemoryStore) Update(_ context.Context, tenantID, agentID int64, in Upda if in.RoleID != nil { a.RoleID = *in.RoleID } + if in.ChannelID != nil { + a.ChannelID = strings.TrimSpace(*in.ChannelID) + } + if in.OnlineDBID != nil { + a.OnlineDBID = strings.TrimSpace(*in.OnlineDBID) + } + if in.DatabaseName != nil { + a.DatabaseName = strings.TrimSpace(*in.DatabaseName) + } cp := cloneAcc(&a.Account) return &cp, nil } @@ -327,7 +349,9 @@ func NewPostgresStore(db *sql.DB) *PostgresStore { return &PostgresStore{DB: db} func (s *PostgresStore) List(ctx context.Context, tenantID int64) ([]Account, error) { rows, err := s.DB.QueryContext(ctx, ` -SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), status, created_by, created_at, last_token_at +SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), + COALESCE(channel_id,''), COALESCE(online_db_id,''), COALESCE(database_name,''), + status, created_by, created_at, last_token_at FROM platform_meta.agent_accounts WHERE tenant_id=$1 ORDER BY CASE status WHEN 'pending' THEN 0 WHEN 'active' THEN 1 ELSE 2 END, agent_id`, tenantID) if err != nil { @@ -338,7 +362,9 @@ ORDER BY CASE status WHEN 'pending' THEN 0 WHEN 'active' THEN 1 ELSE 2 END, agen for rows.Next() { var a Account var last sql.NullTime - if err := rows.Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, &a.Status, &a.CreatedBy, &a.CreatedAt, &last); err != nil { + if err := rows.Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, + &a.ChannelID, &a.OnlineDBID, &a.DatabaseName, + &a.Status, &a.CreatedBy, &a.CreatedAt, &last); err != nil { return nil, err } if last.Valid { @@ -359,9 +385,13 @@ func (s *PostgresStore) Get(ctx context.Context, tenantID, agentID int64) (*Acco var a Account var last sql.NullTime err := s.DB.QueryRowContext(ctx, ` -SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), status, created_by, created_at, last_token_at +SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), + COALESCE(channel_id,''), COALESCE(online_db_id,''), COALESCE(database_name,''), + status, created_by, created_at, last_token_at FROM platform_meta.agent_accounts WHERE agent_id=$1 AND tenant_id=$2`, agentID, tenantID, - ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, &a.Status, &a.CreatedBy, &a.CreatedAt, &last) + ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, + &a.ChannelID, &a.OnlineDBID, &a.DatabaseName, + &a.Status, &a.CreatedBy, &a.CreatedAt, &last) if errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("agent not found") } @@ -390,9 +420,13 @@ func (s *PostgresStore) Register(ctx context.Context, tenantID int64, name, host var a Account var last sql.NullTime err := s.DB.QueryRowContext(ctx, ` -SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), status, created_by, created_at, last_token_at +SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), + COALESCE(channel_id,''), COALESCE(online_db_id,''), COALESCE(database_name,''), + status, created_by, created_at, last_token_at FROM platform_meta.agent_accounts WHERE tenant_id=$1 AND host_key=$2`, tenantID, hostKey, - ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, &a.Status, &a.CreatedBy, &a.CreatedAt, &last) + ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, + &a.ChannelID, &a.OnlineDBID, &a.DatabaseName, + &a.Status, &a.CreatedBy, &a.CreatedAt, &last) if err == nil { if a.Status == StatusPending { secret, err := s.RotateSecret(ctx, tenantID, a.AgentID) @@ -439,11 +473,16 @@ func (s *PostgresStore) insert(ctx context.Context, tenantID, createdBy int64, i } var a Account err = s.DB.QueryRowContext(ctx, ` -INSERT INTO platform_meta.agent_accounts(tenant_id, name, client_id, client_secret_hash, status, created_by, host_key, role_id) -VALUES($1,$2,$3,$4,$5,$6,$7,NULLIF($8,0)) -RETURNING agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), status, created_by, created_at`, +INSERT INTO platform_meta.agent_accounts(tenant_id, name, client_id, client_secret_hash, status, created_by, host_key, role_id, channel_id, online_db_id, database_name) +VALUES($1,$2,$3,$4,$5,$6,$7,NULLIF($8,0),$9,$10,$11) +RETURNING agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), + COALESCE(channel_id,''), COALESCE(online_db_id,''), COALESCE(database_name,''), + status, created_by, created_at`, tenantID, name, cid, string(hash), st, createdBy, strings.TrimSpace(in.HostKey), in.RoleID, - ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, &a.Status, &a.CreatedBy, &a.CreatedAt) + strings.TrimSpace(in.ChannelID), strings.TrimSpace(in.OnlineDBID), strings.TrimSpace(in.DatabaseName), + ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, + &a.ChannelID, &a.OnlineDBID, &a.DatabaseName, + &a.Status, &a.CreatedBy, &a.CreatedAt) if err != nil { return nil, "", err } @@ -462,6 +501,7 @@ func (s *PostgresStore) Update(ctx context.Context, tenantID, agentID int64, in } name, status := a.Name, a.Status roleID := a.RoleID + channelID, onlineDBID, dbName := a.ChannelID, a.OnlineDBID, a.DatabaseName if in.Name != nil { name = strings.TrimSpace(*in.Name) } @@ -474,9 +514,20 @@ func (s *PostgresStore) Update(ctx context.Context, tenantID, agentID int64, in if in.RoleID != nil { roleID = *in.RoleID } + if in.ChannelID != nil { + channelID = strings.TrimSpace(*in.ChannelID) + } + if in.OnlineDBID != nil { + onlineDBID = strings.TrimSpace(*in.OnlineDBID) + } + if in.DatabaseName != nil { + dbName = strings.TrimSpace(*in.DatabaseName) + } if _, err := s.DB.ExecContext(ctx, ` -UPDATE platform_meta.agent_accounts SET name=$1, status=$2, role_id=NULLIF($3,0) WHERE agent_id=$4 AND tenant_id=$5`, - name, status, roleID, agentID, tenantID); err != nil { +UPDATE platform_meta.agent_accounts SET name=$1, status=$2, role_id=NULLIF($3,0), + channel_id=$4, online_db_id=$5, database_name=$6 +WHERE agent_id=$7 AND tenant_id=$8`, + name, status, roleID, channelID, onlineDBID, dbName, agentID, tenantID); err != nil { return nil, err } perms, slugs := a.Perms, a.AppSlugs @@ -525,9 +576,13 @@ func (s *PostgresStore) Authenticate(ctx context.Context, clientID, clientSecret var hash string var last sql.NullTime err := s.DB.QueryRowContext(ctx, ` -SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), client_secret_hash, status, created_by, created_at, last_token_at +SELECT agent_id, tenant_id, name, client_id, COALESCE(host_key,''), COALESCE(role_id,0), + COALESCE(channel_id,''), COALESCE(online_db_id,''), COALESCE(database_name,''), + client_secret_hash, status, created_by, created_at, last_token_at FROM platform_meta.agent_accounts WHERE client_id=$1`, clientID, - ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, &hash, &a.Status, &a.CreatedBy, &a.CreatedAt, &last) + ).Scan(&a.AgentID, &a.TenantID, &a.Name, &a.ClientID, &a.HostKey, &a.RoleID, + &a.ChannelID, &a.OnlineDBID, &a.DatabaseName, + &hash, &a.Status, &a.CreatedBy, &a.CreatedAt, &last) if errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("invalid client credentials") } diff --git a/platform/internal/apidef/catalog.go b/platform/internal/apidef/catalog.go index 9611f88..ef0e51e 100644 --- a/platform/internal/apidef/catalog.go +++ b/platform/internal/apidef/catalog.go @@ -65,13 +65,15 @@ var Catalog = []Entry{ {Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/stop", OperationID: "stopSyncChannel", Summary: "停止同步", Group: "admin"}, {Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/reconcile", OperationID: "reconcileSyncChannel", Summary: "主键对账（同步修复，有限流）", Group: "admin"}, {Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/ingest", OperationID: "ingestSyncRows", Summary: "外部行写入通道 local", Group: "admin"}, - {Method: "GET", Path: "/api/v1/admin/sync/bindings", OperationID: "listSyncBindings", Summary: "列出本机库↔线上库绑定", Group: "admin"}, - {Method: "POST", Path: "/api/v1/admin/sync/bindings", OperationID: "ensureSyncBinding", Summary: "登记/更新绑定", Group: "admin"}, + {Method: "GET", Path: "/api/v1/admin/sync/bindings", OperationID: "listSyncBindings", Summary: "列出本机库↔线上库绑定（用户仅本人）", Group: "admin"}, + {Method: "POST", Path: "/api/v1/admin/sync/bindings", OperationID: "ensureSyncBinding", Summary: "登记/更新绑定（用户可自助）", Group: "admin"}, {Method: "GET", Path: "/api/v1/admin/sync/conflicts", OperationID: "listSyncConflicts", Summary: "已废弃：公司侧 403，改用超管 LWW 审计", Group: "admin"}, {Method: "POST", Path: "/api/v1/admin/sync/conflicts/{id}/resolve", OperationID: "resolveSyncConflict", Summary: "已废弃：公司侧 403", Group: "admin"}, - {Method: "GET", Path: "/api/v1/agent/sync/channels/{id}/whitelist", OperationID: "agentSyncWhitelist", Summary: "本机 agent 拉取表白名单", Group: "agent"}, - {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push", OperationID: "agentSyncPush", Summary: "本机 agent 推变更到线上 A", Group: "agent"}, - {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push/batch", OperationID: "agentSyncPushBatch", Summary: "本机 agent 批量推送", Group: "agent"}, + {Method: "GET", Path: "/api/v1/agent/sync/channels/{id}/whitelist", OperationID: "agentSyncWhitelist", Summary: "拉取表白名单（管理员或用户+Binding）", Group: "agent"}, + {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push", OperationID: "agentSyncPush", Summary: "推变更到线上 A（用户须 online_db_id）", Group: "agent"}, + {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push/batch", OperationID: "agentSyncPushBatch", Summary: "批量推送（同上鉴权）", Group: "agent"}, + {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/pull", OperationID: "agentSyncPull", Summary: "从线上 A 下行（bootstrap/rows/pks）", Group: "agent"}, + {Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/bootstrap", OperationID: "agentSyncBootstrap", Summary: "全量灌库别名（mode=bootstrap）", Group: "agent"}, {Method: "GET", Path: "/api/v1/platform/dbsync/lww-overrides", OperationID: "platformLwwOverrides", Summary: "超管查看 LWW 覆盖审计", Group: "platform"}, {Method: "POST", Path: "/api/v1/platform/dbsync/lww-overrides/{id}/rollback", OperationID: "platformLwwRollback", Summary: "超管按落败快照回滚线上单行", Group: "platform"}, diff --git a/platform/internal/authx/authx.go b/platform/internal/authx/authx.go index bd42ccf..82cdcc4 100644 --- a/platform/internal/authx/authx.go +++ b/platform/internal/authx/authx.go @@ -218,6 +218,33 @@ func RequirePermission(perm string) func(http.HandlerFunc) http.HandlerFunc { } } +// RequireSyncPushAccess 放行「数据同步」权限，或已登录人类成员（用户自助 sync）。 +// 智能体无「数据同步」仍 403；无 sync 权的人类由 handler 再按 Binding 收窄范围。 +func RequireSyncPushAccess() func(http.HandlerFunc) http.HandlerFunc { + return func(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if HasPermissionCtx(r.Context(), Perm数据同步) { + next(w, r) + return + } + if IsAgent(Role(r.Context())) { + WriteError(w, http.StatusForbidden, "permission denied: 数据同步 (role=智能体)") + return + } + if UserID(r.Context()) <= 0 { + WriteError(w, http.StatusForbidden, "permission denied: need user login or 数据同步") + return + } + next(w, r) + } + } +} + +// SyncTenantWide 是否租户级同步权（管理员/智能体带「数据同步」）；否则须按 Binding 自助范围。 +func SyncTenantWide(ctx context.Context) bool { + return HasPermissionCtx(ctx, Perm数据同步) +} + // RequireTenant 要求已加入租户；pending / 无 tenant 不可访问业务数据。 // 超级管理员可在「进入某公司」后携带 tenant_id 访问公司内接口（可看可改，前端会多重确认）。 func RequireTenant() func(http.HandlerFunc) http.HandlerFunc { @@ -394,3 +421,12 @@ func WriteError(w http.ResponseWriter, code int, msg string) { "message": msg, }) } + +// WriteRetryableError 供 agent 识别可稍后重试（不换 UUID）。 +func WriteRetryableError(w http.ResponseWriter, code int, msg string) { + httpx.WriteJson(w, code, map[string]any{ + "code": code, + "message": msg, + "retryable": true, + }) +} diff --git a/platform/internal/blueprint/ensure_ops.go b/platform/internal/blueprint/ensure_ops.go new file mode 100644 index 0000000..6e28571 --- /dev/null +++ b/platform/internal/blueprint/ensure_ops.go @@ -0,0 +1,165 @@ +package blueprint + +import ( + "encoding/json" + "strings" +) + +// defaultImportExportOps Z9：业务表默认允许导入/导出。 +var defaultImportExportOps = []string{"import", "export"} + +// defaultListActions Z9：列表页默认工具栏（含导入）。 +var defaultListActions = []string{"create", "edit", "delete", "export", "import", "refresh"} + +// EnsureDefaultImportExport 发布/合并兜底（Z9d）： +// 可写业务 resource 缺 import/export 则补上；list 页 actions 缺 import 则补上。 +// 跳过：只读 resource（仅 list/get）；meta.ui.disable_default_import == true。 +func (bp *Blueprint) EnsureDefaultImportExport() { + if bp == nil { + return + } + if disableDefaultImport(bp) { + return + } + for i := range bp.Apis.Resources { + r := &bp.Apis.Resources[i] + if resourceReadOnly(r.Operations) { + continue + } + r.Operations = ensureOpList(r.Operations, defaultImportExportOps...) + } + for i := range bp.Pages { + p := &bp.Pages[i] + if !strings.EqualFold(strings.TrimSpace(p.Type), "list") { + continue + } + if p.Layout == nil { + p.Layout = &PageLayout{} + } + p.Layout.Actions = ensureListActions(p.Layout.Actions) + if p.Layout.ActionLabels == nil { + p.Layout.ActionLabels = map[string]string{} + } + if _, ok := p.Layout.ActionLabels["import"]; !ok { + p.Layout.ActionLabels["import"] = "导入 Excel" + } + if _, ok := p.Layout.ActionLabels["export"]; !ok { + p.Layout.ActionLabels["export"] = "导出 Excel" + } + } +} + +func disableDefaultImport(bp *Blueprint) bool { + if len(bp.Meta.UI) == 0 { + return false + } + var ui map[string]any + if err := json.Unmarshal(bp.Meta.UI, &ui); err != nil { + return false + } + v, ok := ui["disable_default_import"] + if !ok { + return false + } + switch t := v.(type) { + case bool: + return t + case string: + return strings.EqualFold(t, "true") || t == "1" + default: + return false + } +} + +func resourceReadOnly(ops []string) bool { + if len(ops) == 0 { + return false + } + writable := false + for _, op := range ops { + switch strings.ToLower(strings.TrimSpace(op)) { + case "create", "update", "delete", "import", "export": + writable = true + } + } + // 仅 list/get → 只读，不强加 import + if !writable { + onlyRead := true + for _, op := range ops { + switch strings.ToLower(strings.TrimSpace(op)) { + case "list", "get", "": + default: + onlyRead = false + } + } + return onlyRead + } + return false +} + +func ensureOpList(ops []string, add ...string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(ops)+len(add)) + for _, op := range ops { + op = strings.ToLower(strings.TrimSpace(op)) + if op == "" { + continue + } + if _, ok := seen[op]; ok { + continue + } + seen[op] = struct{}{} + out = append(out, op) + } + for _, op := range add { + op = strings.ToLower(strings.TrimSpace(op)) + if op == "" { + continue + } + if _, ok := seen[op]; ok { + continue + } + seen[op] = struct{}{} + out = append(out, op) + } + return out +} + +func ensureListActions(actions []string) []string { + if len(actions) == 0 { + return append([]string{}, defaultListActions...) + } + seen := map[string]struct{}{} + out := make([]string, 0, len(actions)+2) + for _, a := range actions { + a = strings.ToLower(strings.TrimSpace(a)) + if a == "" { + continue + } + if _, ok := seen[a]; ok { + continue + } + seen[a] = struct{}{} + out = append(out, a) + } + for _, need := range []string{"import", "export"} { + if _, ok := seen[need]; ok { + continue + } + // 插在 refresh 前；若无 refresh 则追加 + inserted := false + for i, a := range out { + if a == "refresh" { + out = append(out[:i], append([]string{need}, out[i:]...)...) + seen[need] = struct{}{} + inserted = true + break + } + } + if !inserted { + out = append(out, need) + seen[need] = struct{}{} + } + } + return out +} diff --git a/platform/internal/blueprint/ensure_ops_test.go b/platform/internal/blueprint/ensure_ops_test.go new file mode 100644 index 0000000..450e77d --- /dev/null +++ b/platform/internal/blueprint/ensure_ops_test.go @@ -0,0 +1,77 @@ +package blueprint_test + +import ( + "encoding/json" + "testing" + + "aijianzhan/platform/internal/blueprint" +) + +func TestEnsureDefaultImportExport(t *testing.T) { + bp := &blueprint.Blueprint{ + Apis: blueprint.Apis{ + Resources: []blueprint.APIResource{ + {Entity: "a", Path: "/a", Operations: []string{"list", "get", "create", "update", "delete"}}, + {Entity: "ro", Path: "/ro", Operations: []string{"list", "get"}}, + {Entity: "b", Path: "/b", Operations: []string{"list", "create", "import"}}, + }, + }, + Pages: []blueprint.Page{ + {ID: "p1", Type: "list", Entity: "a", Layout: &blueprint.PageLayout{Actions: []string{"create", "edit", "delete", "refresh"}}}, + {ID: "p2", Type: "form", Entity: "a"}, + }, + } + bp.EnsureDefaultImportExport() + + ops0 := bp.Apis.Resources[0].Operations + if !contains(ops0, "import") || !contains(ops0, "export") { + t.Fatalf("writable resource should get import/export, got %v", ops0) + } + opsRO := bp.Apis.Resources[1].Operations + if contains(opsRO, "import") { + t.Fatalf("read-only should not get import, got %v", opsRO) + } + opsB := bp.Apis.Resources[2].Operations + if !contains(opsB, "export") || countOp(opsB, "import") != 1 { + t.Fatalf("should add export once, keep import: %v", opsB) + } + acts := bp.Pages[0].Layout.Actions + if !contains(acts, "import") || !contains(acts, "export") { + t.Fatalf("list actions should include import/export: %v", acts) + } +} + +func TestEnsureDefaultImportExportDisabled(t *testing.T) { + ui, _ := json.Marshal(map[string]any{"disable_default_import": true}) + bp := &blueprint.Blueprint{ + Meta: blueprint.Meta{UI: ui}, + Apis: blueprint.Apis{ + Resources: []blueprint.APIResource{ + {Entity: "a", Path: "/a", Operations: []string{"list", "create"}}, + }, + }, + } + bp.EnsureDefaultImportExport() + if contains(bp.Apis.Resources[0].Operations, "import") { + t.Fatal("disable_default_import should skip") + } +} + +func contains(ss []string, want string) bool { + for _, s := range ss { + if s == want { + return true + } + } + return false +} + +func countOp(ss []string, want string) int { + n := 0 + for _, s := range ss { + if s == want { + n++ + } + } + return n +} diff --git a/platform/internal/blueprint/merge.go b/platform/internal/blueprint/merge.go index a2b3a32..e9c0f4f 100644 --- a/platform/internal/blueprint/merge.go +++ b/platform/internal/blueprint/merge.go @@ -5,16 +5,20 @@ import ( "strings" ) -// MergeResult describes what was added when merging incoming into base. +// MergeResult describes what was added/updated when merging incoming into base. type MergeResult struct { - AddedPages []string - AddedEntities []string - AddedResources []string + AddedPages []string + AddedEntities []string + AddedResources []string + UpdatedPages []string + UpdatedResources []string } // MergeInto merges incoming pages/entities/apis into base (existing published app). // Existing entities keep their fields; new fields on known entities are appended. -// Pages with the same id or route are rejected; new pages are appended. +// Same-path resources union operations (e.g. add import). +// Same page id：合并 actions，不再报错（支持「只改 API/按钮、不新建页」的编辑发布）。 +// 仅 route 冲突且 id 不同仍拒绝。 func MergeInto(base, incoming *Blueprint) (*MergeResult, error) { if base == nil { return nil, fmt.Errorf("base blueprint is nil") @@ -49,7 +53,11 @@ func MergeInto(base, incoming *Blueprint) (*MergeResult, error) { for _, r := range incoming.Apis.Resources { k := pathKey(r.Path) if idx, ok := resByPath[k]; ok { + before := len(base.Apis.Resources[idx].Operations) base.Apis.Resources[idx] = mergeResource(base.Apis.Resources[idx], r) + if len(base.Apis.Resources[idx].Operations) > before { + res.UpdatedResources = append(res.UpdatedResources, k) + } continue } base.Apis.Resources = append(base.Apis.Resources, r) @@ -57,22 +65,27 @@ func MergeInto(base, incoming *Blueprint) (*MergeResult, error) { res.AddedResources = append(res.AddedResources, k) } - pageByID := map[string]struct{}{} - routeBy := map[string]struct{}{} - for _, p := range base.Pages { - pageByID[p.ID] = struct{}{} - routeBy[p.Route] = struct{}{} + pageByID := map[string]int{} + routeBy := map[string]string{} // route -> page id + for i, p := range base.Pages { + pageByID[p.ID] = i + routeBy[p.Route] = p.ID } for _, p := range incoming.Pages { - if _, ok := pageByID[p.ID]; ok { - return nil, fmt.Errorf("page id already exists: %s (use a new page id when adding to an existing app)", p.ID) + if idx, ok := pageByID[p.ID]; ok { + before := pageActionCount(base.Pages[idx]) + base.Pages[idx] = mergePage(base.Pages[idx], p) + if pageActionCount(base.Pages[idx]) > before { + res.UpdatedPages = append(res.UpdatedPages, p.ID) + } + continue } - if _, ok := routeBy[p.Route]; ok { + if owner, ok := routeBy[p.Route]; ok && owner != p.ID { return nil, fmt.Errorf("page route already exists: %s", p.Route) } base.Pages = append(base.Pages, p) - pageByID[p.ID] = struct{}{} - routeBy[p.Route] = struct{}{} + pageByID[p.ID] = len(base.Pages) - 1 + routeBy[p.Route] = p.ID res.AddedPages = append(res.AddedPages, p.ID) } @@ -89,7 +102,11 @@ func MergeInto(base, incoming *Blueprint) (*MergeResult, error) { base.Seed = incoming.Seed } - if len(res.AddedPages) == 0 && len(res.AddedEntities) == 0 && len(res.AddedResources) == 0 { + if len(res.AddedPages) == 0 && + len(res.AddedEntities) == 0 && + len(res.AddedResources) == 0 && + len(res.UpdatedPages) == 0 && + len(res.UpdatedResources) == 0 { return nil, fmt.Errorf("nothing new to publish: provide newly generated pages (and entities/apis if needed) for an existing app") } return res, nil @@ -123,6 +140,34 @@ func mergeEntity(base, incoming Entity) Entity { return base } +func pageActionCount(p Page) int { + if p.Layout == nil { + return 0 + } + return len(p.Layout.Actions) +} + +func mergePage(base, incoming Page) Page { + if incoming.Layout == nil || len(incoming.Layout.Actions) == 0 { + return base + } + if base.Layout == nil { + base.Layout = &PageLayout{} + } + actSet := map[string]struct{}{} + for _, a := range base.Layout.Actions { + actSet[a] = struct{}{} + } + for _, a := range incoming.Layout.Actions { + if _, ok := actSet[a]; ok { + continue + } + base.Layout.Actions = append(base.Layout.Actions, a) + actSet[a] = struct{}{} + } + return base +} + func mergeResource(base, incoming APIResource) APIResource { opSet := map[string]struct{}{} for _, op := range base.Operations { diff --git a/platform/internal/blueprint/merge_test.go b/platform/internal/blueprint/merge_test.go index 14e1072..ec16652 100644 --- a/platform/internal/blueprint/merge_test.go +++ b/platform/internal/blueprint/merge_test.go @@ -40,14 +40,70 @@ func TestMergeIntoAddsPages(t *testing.T) { } } -func TestMergeIntoRejectsDuplicatePageID(t *testing.T) { +func TestMergeIntoUpdatesExistingPageAndResource(t *testing.T) { + base := &Blueprint{ + Apis: Apis{Resources: []APIResource{{ + Entity: "settlement_points", Path: "/settlement_points", + Operations: []string{"list", "get", "create", "update", "delete"}, + }}}, + Pages: []Page{{ + ID: "record_list", Route: "/records", Type: "list", Entity: "settlement_points", + Layout: &PageLayout{Actions: []string{"create", "edit", "delete"}}, + }}, + } + incoming := &Blueprint{ + Apis: Apis{Resources: []APIResource{{ + Entity: "settlement_points", Path: "/settlement_points", + Operations: []string{"list", "get", "create", "update", "delete", "import", "export"}, + }}}, + Pages: []Page{{ + ID: "record_list", Route: "/records", Type: "list", Entity: "settlement_points", + Layout: &PageLayout{Actions: []string{"create", "edit", "delete", "import"}}, + }}, + } + res, err := MergeInto(base, incoming) + if err != nil { + t.Fatal(err) + } + if len(res.AddedPages) != 0 { + t.Fatalf("should not add pages: %+v", res.AddedPages) + } + if len(res.UpdatedResources) != 1 || res.UpdatedResources[0] != "settlement_points" { + t.Fatalf("updated resources: %+v", res.UpdatedResources) + } + ops := base.Apis.Resources[0].Operations + hasImport := false + for _, op := range ops { + if op == "import" { + hasImport = true + } + } + if !hasImport { + t.Fatalf("ops missing import: %+v", ops) + } + acts := base.Pages[0].Layout.Actions + hasAct := false + for _, a := range acts { + if a == "import" { + hasAct = true + } + } + if !hasAct { + t.Fatalf("actions missing import: %+v", acts) + } +} + +func TestMergeIntoRejectsRouteConflictDifferentID(t *testing.T) { base := &Blueprint{ Pages: []Page{{ID: "a", Route: "/a", Type: "list", Entity: "x"}}, } incoming := &Blueprint{ - Pages: []Page{{ID: "a", Route: "/b", Type: "list", Entity: "x"}}, + Pages: []Page{{ID: "b", Route: "/a", Type: "list", Entity: "x"}}, + Apis: Apis{Resources: []APIResource{{ + Entity: "x", Path: "/x", Operations: []string{"list", "import"}, + }}}, } if _, err := MergeInto(base, incoming); err == nil { - t.Fatal("expected duplicate page id error") + t.Fatal("expected route conflict error") } } diff --git a/platform/internal/dbsync/binding.go b/platform/internal/dbsync/binding.go index 3ad5583..c6a0cd8 100644 --- a/platform/internal/dbsync/binding.go +++ b/platform/internal/dbsync/binding.go @@ -11,7 +11,7 @@ import ( "github.com/google/uuid" ) -// Binding 本机库 ↔ 线上库映射（P1；不挡推送，供登记/查询）。 +// Binding 本机库 ↔ 线上库映射（登记/查询；用户自助 push 时按 user_id+online_db_id 鉴权）。 type Binding struct { ID string `json:"id"` TenantID int64 `json:"tenant_id"` @@ -19,9 +19,12 @@ type Binding struct { LocalDatabaseID string `json:"local_database_id"` OnlineDBID string `json:"online_db_id"` ChannelID string `json:"channel_id,omitempty"` - Note string `json:"note,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + // 可读名（与宇恒 database_name / display_name 对齐；控制台优先展示） + DatabaseName string `json:"database_name,omitempty"` // 本地库可读名 + DisplayName string `json:"display_name,omitempty"` // 线上库可读名 + Note string `json:"note,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` } func (s *FileStore) bindingPath() string { @@ -37,6 +40,8 @@ func (s *FileStore) EnsureBinding(b Binding) (Binding, error) { } b.LocalDatabaseID = strings.TrimSpace(b.LocalDatabaseID) b.OnlineDBID = strings.TrimSpace(b.OnlineDBID) + b.DatabaseName = strings.TrimSpace(b.DatabaseName) + b.DisplayName = strings.TrimSpace(b.DisplayName) if b.TenantID <= 0 { return b, fmt.Errorf("tenant_id required") } @@ -53,6 +58,12 @@ func (s *FileStore) EnsureBinding(b Binding) (Binding, error) { if b.ChannelID != "" { list[i].ChannelID = b.ChannelID } + if b.DatabaseName != "" { + list[i].DatabaseName = b.DatabaseName + } + if b.DisplayName != "" { + list[i].DisplayName = b.DisplayName + } if b.Note != "" { list[i].Note = b.Note } @@ -79,6 +90,11 @@ func (s *FileStore) EnsureBinding(b Binding) (Binding, error) { } func (s *FileStore) ListBindings(tenantID int64, localDatabaseID string) ([]Binding, error) { + return s.ListBindingsFiltered(tenantID, 0, localDatabaseID) +} + +// ListBindingsFiltered 按租户列出；userID>0 时仅返回该用户的 Binding。 +func (s *FileStore) ListBindingsFiltered(tenantID, userID int64, localDatabaseID string) ([]Binding, error) { s.mu.Lock() defer s.mu.Unlock() list, err := s.readBindingsUnlocked() @@ -90,6 +106,9 @@ func (s *FileStore) ListBindings(tenantID int64, localDatabaseID string) ([]Bind if b.TenantID != tenantID { continue } + if userID > 0 && b.UserID != userID { + continue + } if localDatabaseID != "" && b.LocalDatabaseID != localDatabaseID { continue } @@ -98,6 +117,47 @@ func (s *FileStore) ListBindings(tenantID int64, localDatabaseID string) ([]Bind return out, nil } +// UserOwnsOnlineDB 用户是否登记了该 online_db_id（可选限定 channel）。 +func (s *FileStore) UserOwnsOnlineDB(tenantID, userID int64, channelID, onlineDBID string) bool { + if tenantID <= 0 || userID <= 0 || strings.TrimSpace(onlineDBID) == "" { + return false + } + list, err := s.ListBindingsFiltered(tenantID, userID, "") + if err != nil { + return false + } + online := strings.TrimSpace(onlineDBID) + ch := strings.TrimSpace(channelID) + for _, b := range list { + if strings.TrimSpace(b.OnlineDBID) != online { + continue + } + if ch != "" && b.ChannelID != "" && b.ChannelID != ch { + continue + } + return true + } + return false +} + +// UserCanAccessChannel 用户是否登记了指向该通道的 Binding（channel_id 空视为未限定通道）。 +func (s *FileStore) UserCanAccessChannel(tenantID, userID int64, channelID string) bool { + if tenantID <= 0 || userID <= 0 || strings.TrimSpace(channelID) == "" { + return false + } + list, err := s.ListBindingsFiltered(tenantID, userID, "") + if err != nil || len(list) == 0 { + return false + } + ch := strings.TrimSpace(channelID) + for _, b := range list { + if b.ChannelID == "" || b.ChannelID == ch { + return true + } + } + return false +} + func (s *FileStore) GetBinding(tenantID int64, localDatabaseID string) (*Binding, error) { list, err := s.ListBindings(tenantID, localDatabaseID) if err != nil { diff --git a/platform/internal/dbsync/binding_test.go b/platform/internal/dbsync/binding_test.go index 2168465..095b6e6 100644 --- a/platform/internal/dbsync/binding_test.go +++ b/platform/internal/dbsync/binding_test.go @@ -16,16 +16,19 @@ func TestEnsureBindingUpsert(t *testing.T) { LocalDatabaseID: "local-a", OnlineDBID: "online-1", ChannelID: "ch1", + DatabaseName: "本地演示库", + DisplayName: "AI建站智能体API", }) - if err != nil || b.ID == "" { + if err != nil || b.ID == "" || b.DisplayName != "AI建站智能体API" { t.Fatalf("ensure: %+v err=%v", b, err) } b2, err := st.EnsureBinding(Binding{ TenantID: 1, LocalDatabaseID: "local-a", OnlineDBID: "online-2", + DisplayName: "线上库B", }) - if err != nil || b2.OnlineDBID != "online-2" || b2.ID != b.ID { + if err != nil || b2.OnlineDBID != "online-2" || b2.ID != b.ID || b2.DisplayName != "线上库B" { t.Fatalf("upsert: %+v err=%v", b2, err) } list, err := st.ListBindings(1, "local-a") @@ -33,3 +36,42 @@ func TestEnsureBindingUpsert(t *testing.T) { t.Fatalf("list=%d err=%v", len(list), err) } } + +func TestUserBindingScope(t *testing.T) { + dir := t.TempDir() + st, err := NewFileStore(filepath.Join(dir, "dbsync")) + if err != nil { + t.Fatal(err) + } + _, err = st.EnsureBinding(Binding{ + TenantID: 1, UserID: 10, + LocalDatabaseID: "loc-a", OnlineDBID: "on-a", ChannelID: "ch1", + }) + if err != nil { + t.Fatal(err) + } + _, err = st.EnsureBinding(Binding{ + TenantID: 1, UserID: 20, + LocalDatabaseID: "loc-b", OnlineDBID: "on-b", ChannelID: "ch1", + }) + if err != nil { + t.Fatal(err) + } + mine, err := st.ListBindingsFiltered(1, 10, "") + if err != nil || len(mine) != 1 || mine[0].OnlineDBID != "on-a" { + t.Fatalf("filtered: %+v err=%v", mine, err) + } + if !st.UserOwnsOnlineDB(1, 10, "ch1", "on-a") { + t.Fatal("should own on-a") + } + if st.UserOwnsOnlineDB(1, 10, "ch1", "on-b") { + t.Fatal("must not own other's online_db_id") + } + if !st.UserCanAccessChannel(1, 10, "ch1") { + t.Fatal("should access ch1") + } + if st.UserCanAccessChannel(1, 10, "ch-other") { + t.Fatal("must not access unbound channel") + } +} + diff --git a/platform/internal/dbsync/dialect.go b/platform/internal/dbsync/dialect.go index 28b5110..4586ed9 100644 --- a/platform/internal/dbsync/dialect.go +++ b/platform/internal/dbsync/dialect.go @@ -19,7 +19,7 @@ func Open(driver Driver, dsn string) (*sql.DB, error) { if dsn == "" { return nil, fmt.Errorf("sqlite dsn required") } - // modernc.org/sqlite 注册名为 sqlite + dsn = normalizeSQLiteDSN(dsn) case DriverMySQL: name = "mysql" case DriverPostgres: @@ -31,8 +31,14 @@ func Open(driver Driver, dsn string) (*sql.DB, error) { if err != nil { return nil, err } - db.SetMaxOpenConns(5) - db.SetMaxIdleConns(2) + if driver == DriverSQLite { + // SQLite 文件锁：单连接更稳，避免 Windows 上并发 open 卡死/EOF + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + } else { + db.SetMaxOpenConns(5) + db.SetMaxIdleConns(2) + } if err := db.Ping(); err != nil { _ = db.Close() return nil, err @@ -40,6 +46,33 @@ func Open(driver Driver, dsn string) (*sql.DB, error) { return db, nil } +// normalizeSQLiteDSN：Windows 反斜杠 → 正斜杠；裸盘符路径补 file:；补 busy_timeout。 +func normalizeSQLiteDSN(dsn string) string { + dsn = strings.TrimSpace(dsn) + lower := strings.ToLower(dsn) + if strings.HasPrefix(lower, "file:") { + rest := dsn[5:] + path, query, hasQ := strings.Cut(rest, "?") + path = strings.ReplaceAll(path, `\`, `/`) + if hasQ { + dsn = "file:" + path + "?" + query + } else { + dsn = "file:" + path + } + } else if strings.Contains(dsn, `\`) || (len(dsn) >= 2 && dsn[1] == ':') { + path := strings.ReplaceAll(dsn, `\`, `/`) + dsn = "file:" + path + } + if !strings.Contains(strings.ToLower(dsn), "busy_timeout") { + if strings.Contains(dsn, "?") { + dsn += "&_pragma=busy_timeout(5000)" + } else { + dsn += "?_pragma=busy_timeout(5000)" + } + } + return dsn +} + func quoteIdent(driver Driver, name string) string { name = strings.ReplaceAll(name, "`", "") name = strings.ReplaceAll(name, `"`, "") diff --git a/platform/internal/dbsync/dialect_test.go b/platform/internal/dbsync/dialect_test.go new file mode 100644 index 0000000..781e01d --- /dev/null +++ b/platform/internal/dbsync/dialect_test.go @@ -0,0 +1,31 @@ +package dbsync + +import "testing" + +func TestNormalizeSQLiteDSN(t *testing.T) { + got := normalizeSQLiteDSN(`file:E:\data\remote.db`) + if got != "file:E:/data/remote.db?_pragma=busy_timeout(5000)" { + t.Fatalf("got %q", got) + } + got = normalizeSQLiteDSN(`E:\data\remote.db`) + if got != "file:E:/data/remote.db?_pragma=busy_timeout(5000)" { + t.Fatalf("bare path got %q", got) + } + got = normalizeSQLiteDSN(`file:E:/data/remote.db?_pragma=foreign_keys(1)`) + if got != "file:E:/data/remote.db?_pragma=foreign_keys(1)&_pragma=busy_timeout(5000)" { + t.Fatalf("keep query got %q", got) + } +} + +func TestIsRetryable(t *testing.T) { + if !IsRetryable(Retryable("x")) { + t.Fatal("expected retryable") + } + if IsRetryable(fmtError("plain")) { + t.Fatal("plain should not") + } +} + +type fmtError string + +func (e fmtError) Error() string { return string(e) } diff --git a/platform/internal/dbsync/drop_table.go b/platform/internal/dbsync/drop_table.go new file mode 100644 index 0000000..8ac90e8 --- /dev/null +++ b/platform/internal/dbsync/drop_table.go @@ -0,0 +1,87 @@ +package dbsync + +import ( + "context" + "fmt" + "strings" +) + +// DropTableResult 控制台删表结果。 +type DropTableResult struct { + OK bool `json:"ok"` + Side string `json:"side"` + Table string `json:"table"` + Dropped bool `json:"dropped"` + MetaCleared int64 `json:"meta_cleared,omitempty"` + Message string `json:"message,omitempty"` +} + +// DropTableOnEndpoint 删除业务表（禁止 _ajz_*）；并清理 _ajz_sync_meta 中该表残留。 +// 注意：仅作用于当前 endpoint（线上 A 或本机 B），不向另一侧同步 DDL。 +func DropTableOnEndpoint(ctx context.Context, ep Endpoint, side, table string) (*DropTableResult, error) { + table = strings.TrimSpace(table) + side = strings.TrimSpace(side) + if side == "" { + side = "remote" + } + if table == "" { + return nil, fmt.Errorf("table required") + } + if strings.HasPrefix(table, "_ajz_") { + return nil, fmt.Errorf("同步系统表不可删: %s", table) + } + if strings.EqualFold(table, "sqlite_master") || strings.HasPrefix(strings.ToLower(table), "sqlite_") { + return nil, fmt.Errorf("系统表不可删: %s", table) + } + + db, owned, err := openInspectDB(ep) + if err != nil { + return &DropTableResult{OK: false, Side: side, Table: table, Message: err.Error()}, err + } + if owned { + defer db.Close() + } + + exists, err := tableExists(ctx, db, ep.Driver, table) + if err != nil { + return nil, err + } + if !exists { + return &DropTableResult{ + OK: true, + Side: side, + Table: table, + Dropped: false, + Message: "表不存在（可能已删）", + }, nil + } + + ddl := fmt.Sprintf(`DROP TABLE IF EXISTS %s`, quoteIdent(ep.Driver, table)) + if _, err := db.ExecContext(ctx, ddl); err != nil { + return nil, fmt.Errorf("drop table: %w", err) + } + + var metaCleared int64 + if metaExists, _ := tableExists(ctx, db, ep.Driver, MetaTable); metaExists { + q := fmt.Sprintf(`DELETE FROM %s WHERE table_name = ?`, quoteIdent(ep.Driver, MetaTable)) + if ep.Driver == DriverPostgres { + q = fmt.Sprintf(`DELETE FROM %s WHERE table_name = $1`, quoteIdent(ep.Driver, MetaTable)) + } + res, err := db.ExecContext(ctx, q, table) + if err == nil && res != nil { + metaCleared, _ = res.RowsAffected() + } + } + + // 若该 DSN 在 remote 池里，丢掉缓存连接，避免旧 schema 缓存感。 + InvalidateRemote(ep.Driver, ep.DSN) + + return &DropTableResult{ + OK: true, + Side: side, + Table: table, + Dropped: true, + MetaCleared: metaCleared, + Message: "已删除；若另一侧仍有同名表，不会自动同步删除，需分别处理或避免再次 push/ensure", + }, nil +} diff --git a/platform/internal/dbsync/drop_table_test.go b/platform/internal/dbsync/drop_table_test.go new file mode 100644 index 0000000..bf4327b --- /dev/null +++ b/platform/internal/dbsync/drop_table_test.go @@ -0,0 +1,56 @@ +package dbsync + +import ( + "context" + "path/filepath" + "testing" +) + +func TestDropTableOnEndpoint(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "drop.db")) + "?_pragma=busy_timeout(5000)" + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := EnsureTableFromColumns(ctx, db, DriverSQLite, "模块·测2·item", "id", []string{"id", "before_day"}); err != nil { + t.Fatal(err) + } + if err := EnsureMeta(ctx, db, DriverSQLite); err != nil { + t.Fatal(err) + } + _ = UpsertMeta(ctx, db, DriverSQLite, "模块·测2·item", "pk1", 1) + _ = db.Close() + + ep := Endpoint{Driver: DriverSQLite, DSN: dsn} + res, err := DropTableOnEndpoint(ctx, ep, "remote", "模块·测2·item") + if err != nil { + t.Fatal(err) + } + if !res.OK || !res.Dropped { + t.Fatalf("unexpected: %+v", res) + } + if res.MetaCleared < 1 { + t.Fatalf("want meta cleared: %+v", res) + } + + db2, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + defer db2.Close() + exists, err := tableExists(ctx, db2, DriverSQLite, "模块·测2·item") + if err != nil || exists { + t.Fatalf("table should be gone exists=%v err=%v", exists, err) + } + + // idempotent + res2, err := DropTableOnEndpoint(ctx, ep, "remote", "模块·测2·item") + if err != nil || !res2.OK || res2.Dropped { + t.Fatalf("idempotent: %+v err=%v", res2, err) + } + if _, err := DropTableOnEndpoint(ctx, ep, "remote", "_ajz_sync_meta"); err == nil { + t.Fatal("expected reject system table") + } +} diff --git a/platform/internal/dbsync/ensure_table.go b/platform/internal/dbsync/ensure_table.go new file mode 100644 index 0000000..ba4a9c4 --- /dev/null +++ b/platform/internal/dbsync/ensure_table.go @@ -0,0 +1,105 @@ +package dbsync + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// EnsureTableFromRow 表不存在时按行字段自动建表（TEXT 列 + PK），满足 Z4「按 Binding 接受任意表」。 +func EnsureTableFromRow(ctx context.Context, db *sql.DB, driver Driver, table, pkCol string, row map[string]any) error { + table = strings.TrimSpace(table) + pkCol = strings.TrimSpace(pkCol) + if table == "" || pkCol == "" { + return fmt.Errorf("table and pk required") + } + if row == nil { + row = map[string]any{pkCol: ""} + } + cols := make([]string, 0, len(row)) + seen := map[string]struct{}{} + if _, ok := row[pkCol]; !ok { + cols = append(cols, pkCol) + seen[pkCol] = struct{}{} + } + for k := range row { + k = strings.TrimSpace(k) + if k == "" { + continue + } + if _, ok := seen[k]; ok { + continue + } + seen[k] = struct{}{} + cols = append(cols, k) + } + return EnsureTableFromColumns(ctx, db, driver, table, pkCol, cols) +} + +// EnsureTableFromColumns 表不存在时按列名建空表（全部 TEXT，指定 PK）。已存在则幂等跳过。 +// 用于空表结构同步：本机有空表 → 线上也建同名空表（无需 outbox 行）。 +func EnsureTableFromColumns(ctx context.Context, db *sql.DB, driver Driver, table, pkCol string, columns []string) error { + table = strings.TrimSpace(table) + pkCol = strings.TrimSpace(pkCol) + if table == "" { + return fmt.Errorf("table required") + } + if strings.HasPrefix(table, "_ajz_") { + return fmt.Errorf("sync system table not allowed: %s", table) + } + if pkCol == "" { + pkCol = "id" + } + exists, err := tableExists(ctx, db, driver, table) + if err != nil { + return err + } + if exists { + return nil + } + seen := map[string]struct{}{} + defs := make([]string, 0, len(columns)+1) + defs = append(defs, fmt.Sprintf("%s TEXT PRIMARY KEY", quoteIdent(driver, pkCol))) + seen[pkCol] = struct{}{} + for _, c := range columns { + c = strings.TrimSpace(c) + if c == "" { + continue + } + if _, ok := seen[c]; ok { + continue + } + seen[c] = struct{}{} + defs = append(defs, fmt.Sprintf("%s TEXT", quoteIdent(driver, c))) + } + if len(defs) == 0 { + return fmt.Errorf("columns required") + } + ddl := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (%s)`, quoteIdent(driver, table), strings.Join(defs, ", ")) + _, err = db.ExecContext(ctx, ddl) + return err +} + +func tableExists(ctx context.Context, db *sql.DB, driver Driver, table string) (bool, error) { + var q string + switch driver { + case DriverSQLite: + q = `SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1` + case DriverMySQL: + q = `SELECT 1 FROM information_schema.tables WHERE table_schema=DATABASE() AND table_name=? LIMIT 1` + case DriverPostgres: + q = `SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=$1 LIMIT 1` + default: + return false, fmt.Errorf("unsupported driver") + } + var n int + err := db.QueryRowContext(ctx, q, table).Scan(&n) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} diff --git a/platform/internal/dbsync/ensure_table_test.go b/platform/internal/dbsync/ensure_table_test.go new file mode 100644 index 0000000..81886bb --- /dev/null +++ b/platform/internal/dbsync/ensure_table_test.go @@ -0,0 +1,29 @@ +package dbsync + +import ( + "context" + "path/filepath" + "testing" +) + +func TestEnsureTableFromRow(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "t.db")) + "?_pragma=busy_timeout(5000)" + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + row := map[string]any{"id": "u1", "title": "x"} + if err := EnsureTableFromRow(ctx, db, DriverSQLite, "orders", "id", row); err != nil { + t.Fatal(err) + } + if err := EnsureTableFromRow(ctx, db, DriverSQLite, "orders", "id", row); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRowContext(ctx, `SELECT COUNT(1) FROM sqlite_master WHERE type='table' AND name='orders'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("table missing n=%d err=%v", n, err) + } +} diff --git a/platform/internal/dbsync/inspect.go b/platform/internal/dbsync/inspect.go new file mode 100644 index 0000000..10ffbe8 --- /dev/null +++ b/platform/internal/dbsync/inspect.go @@ -0,0 +1,277 @@ +package dbsync + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// TableInspect 线上/本机库一张表的摘要（控制台验同步用）。 +type TableInspect struct { + Name string `json:"name"` + RowCount int64 `json:"row_count"` + Columns []string `json:"columns"` + ColumnCount int `json:"column_count"` +} + +// InspectResult 库体检结果。 +type InspectResult struct { + OK bool `json:"ok"` + Side string `json:"side"` // remote | local + Driver string `json:"driver"` + DSNHint string `json:"dsn_hint,omitempty"` + Tables []TableInspect `json:"tables"` + Message string `json:"message,omitempty"` +} + +// PreviewResult 单表行预览。 +type PreviewResult struct { + OK bool `json:"ok"` + Table string `json:"table"` + Columns []string `json:"columns"` + Rows []map[string]any `json:"rows"` + Total int64 `json:"total"` + Limit int `json:"limit"` + Message string `json:"message,omitempty"` +} + +// InspectEndpoint 列出业务表 + 行数 + 列名（含 _ajz_ 同步系统表，便于对照）。 +func InspectEndpoint(ctx context.Context, ep Endpoint, side string, includeSyncMeta bool) (*InspectResult, error) { + db, owned, err := openInspectDB(ep) + if err != nil { + return &InspectResult{OK: false, Side: side, Driver: string(ep.Driver), Message: err.Error()}, err + } + if owned { + defer db.Close() + } + names, err := listTablesForInspect(ctx, db, ep.Driver, includeSyncMeta) + if err != nil { + return &InspectResult{OK: false, Side: side, Driver: string(ep.Driver), Message: err.Error()}, err + } + out := &InspectResult{ + OK: true, + Side: side, + Driver: string(ep.Driver), + DSNHint: dsnFileHint(ep.DSN), + Tables: make([]TableInspect, 0, len(names)), + } + for _, name := range names { + ti := TableInspect{Name: name} + cols, _ := listColumns(ctx, db, ep.Driver, name) + ti.Columns = cols + ti.ColumnCount = len(cols) + n, _ := countRows(ctx, db, ep.Driver, name) + ti.RowCount = n + out.Tables = append(out.Tables, ti) + } + if len(out.Tables) == 0 { + out.Message = "库可连接，但尚无业务表（可能还未 push）" + } + return out, nil +} + +// PreviewTable 预览表前 limit 行（默认 50，最大 200）。 +func PreviewTable(ctx context.Context, ep Endpoint, table string, limit int) (*PreviewResult, error) { + table = strings.TrimSpace(table) + if table == "" { + return nil, fmt.Errorf("table required") + } + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + db, owned, err := openInspectDB(ep) + if err != nil { + return nil, err + } + if owned { + defer db.Close() + } + cols, err := listColumns(ctx, db, ep.Driver, table) + if err != nil { + return nil, err + } + total, _ := countRows(ctx, db, ep.Driver, table) + rows, err := fetchPreviewRows(ctx, db, ep.Driver, table, cols, limit) + if err != nil { + return nil, err + } + return &PreviewResult{ + OK: true, + Table: table, + Columns: cols, + Rows: rows, + Total: total, + Limit: limit, + }, nil +} + +func openInspectDB(ep Endpoint) (*sql.DB, bool, error) { + // 控制台验库用短连接，不进 remote 池，避免 SQLite 文件被长期占用。 + db, err := Open(ep.Driver, ep.DSN) + if err != nil { + return nil, false, err + } + return db, true, nil +} + +func listTablesForInspect(ctx context.Context, db *sql.DB, driver Driver, includeSyncMeta bool) ([]string, error) { + var q string + switch driver { + case DriverSQLite: + if includeSyncMeta { + q = `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name` + } else { + q = `SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_ajz_%' ORDER BY name` + } + case DriverMySQL: + if includeSyncMeta { + q = `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name` + } else { + q = `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name NOT LIKE '\_ajz\_%' ORDER BY table_name` + } + case DriverPostgres: + if includeSyncMeta { + q = `SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename` + } else { + q = `SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE '\_ajz\_%' ORDER BY tablename` + } + default: + return nil, fmt.Errorf("unsupported driver") + } + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + return nil, err + } + out = append(out, n) + } + return out, rows.Err() +} + +func listColumns(ctx context.Context, db *sql.DB, driver Driver, table string) ([]string, error) { + var ( + rows *sql.Rows + err error + ) + switch driver { + case DriverSQLite: + rows, err = db.QueryContext(ctx, fmt.Sprintf(`PRAGMA table_info(%s)`, quoteIdent(driver, table))) + if err != nil { + return nil, err + } + defer rows.Close() + var cols []string + for rows.Next() { + var cid int + var name, typ string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil { + return nil, err + } + cols = append(cols, name) + } + return cols, rows.Err() + case DriverMySQL: + rows, err = db.QueryContext(ctx, ` +SELECT COLUMN_NAME FROM information_schema.columns +WHERE table_schema = DATABASE() AND table_name = ? ORDER BY ORDINAL_POSITION`, table) + case DriverPostgres: + rows, err = db.QueryContext(ctx, ` +SELECT column_name FROM information_schema.columns +WHERE table_schema = 'public' AND table_name = $1 ORDER BY ordinal_position`, table) + default: + return nil, fmt.Errorf("unsupported") + } + if err != nil { + return nil, err + } + defer rows.Close() + var cols []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + return nil, err + } + cols = append(cols, n) + } + return cols, rows.Err() +} + +func countRows(ctx context.Context, db *sql.DB, driver Driver, table string) (int64, error) { + q := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, quoteIdent(driver, table)) + var n int64 + err := db.QueryRowContext(ctx, q).Scan(&n) + return n, err +} + +func fetchPreviewRows(ctx context.Context, db *sql.DB, driver Driver, table string, cols []string, limit int) ([]map[string]any, error) { + if len(cols) == 0 { + return []map[string]any{}, nil + } + q := fmt.Sprintf(`SELECT * FROM %s LIMIT %d`, quoteIdent(driver, table), limit) + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + defer rows.Close() + colNames, err := rows.Columns() + if err != nil { + return nil, err + } + var out []map[string]any + for rows.Next() { + raw := make([]any, len(colNames)) + ptrs := make([]any, len(colNames)) + for i := range raw { + ptrs[i] = &raw[i] + } + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + m := make(map[string]any, len(colNames)) + for i, c := range colNames { + m[c] = normalizeSQLValue(raw[i]) + } + out = append(out, m) + } + return out, rows.Err() +} + +func normalizeSQLValue(v any) any { + switch x := v.(type) { + case nil: + return nil + case []byte: + return string(x) + default: + return x + } +} + +func dsnFileHint(dsn string) string { + dsn = strings.TrimSpace(dsn) + if i := strings.LastIndexAny(dsn, `/\`); i >= 0 { + rest := dsn[i+1:] + if j := strings.IndexAny(rest, "?#"); j >= 0 { + rest = rest[:j] + } + if strings.HasSuffix(strings.ToLower(rest), ".db") { + return rest + } + } + if len(dsn) > 64 { + return dsn[:64] + "…" + } + return dsn +} diff --git a/platform/internal/dbsync/inspect_test.go b/platform/internal/dbsync/inspect_test.go new file mode 100644 index 0000000..88e3fc1 --- /dev/null +++ b/platform/internal/dbsync/inspect_test.go @@ -0,0 +1,34 @@ +package dbsync + +import ( + "context" + "path/filepath" + "testing" +) + +func TestInspectAndPreviewSQLite(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "inspect.db")) + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE orders (id TEXT PRIMARY KEY, title TEXT); INSERT INTO orders VALUES ('a','one'),('b','two')`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + + ep := Endpoint{Driver: DriverSQLite, DSN: dsn} + ins, err := InspectEndpoint(context.Background(), ep, "remote", false) + if err != nil || !ins.OK { + t.Fatalf("inspect: %+v %v", ins, err) + } + if len(ins.Tables) != 1 || ins.Tables[0].Name != "orders" || ins.Tables[0].RowCount != 2 || ins.Tables[0].ColumnCount != 2 { + t.Fatalf("tables=%+v", ins.Tables) + } + prev, err := PreviewTable(context.Background(), ep, "orders", 10) + if err != nil || !prev.OK || prev.Total != 2 || len(prev.Rows) != 2 { + t.Fatalf("preview: %+v %v", prev, err) + } +} diff --git a/platform/internal/dbsync/manager.go b/platform/internal/dbsync/manager.go index f56632e..38b960f 100644 --- a/platform/internal/dbsync/manager.go +++ b/platform/internal/dbsync/manager.go @@ -55,6 +55,12 @@ func (m *Manager) StartChannel(id string) error { c.Enabled = true c.LastError = "" }) + // 预热 remote，避免首个 agent push 冷开 SQLite 触发超时 + go func(driver Driver, dsn string) { + if err := WarmRemote(driver, dsn); err != nil { + log.Printf("dbsync warm remote %s: %v", id, err) + } + }(ch.Remote.Driver, ch.Remote.DSN) return nil } @@ -65,6 +71,9 @@ func (m *Manager) StopChannel(id string) { cancel() delete(m.runners, id) } + if ch, err := m.store.GetChannel(id); err == nil { + InvalidateRemote(ch.Remote.Driver, ch.Remote.DSN) + } _ = m.store.PatchStats(id, func(c *Channel) { c.Enabled = false }) } diff --git a/platform/internal/dbsync/pool.go b/platform/internal/dbsync/pool.go new file mode 100644 index 0000000..e60a7dd --- /dev/null +++ b/platform/internal/dbsync/pool.go @@ -0,0 +1,85 @@ +package dbsync + +import ( + "database/sql" + "fmt" + "strings" + "sync" + "time" +) + +// remotePool 复用线上 A 连接，避免每次 push 冷开 SQLite（Windows 上常 >3s → 旧默认超时 503）。 +type remotePool struct { + mu sync.Mutex + dbs map[string]*pooledDB +} + +type pooledDB struct { + db *sql.DB + driver Driver + lastUsed time.Time +} + +var sharedRemotePool = &remotePool{dbs: map[string]*pooledDB{}} + +func poolKey(driver Driver, dsn string) string { + if driver == DriverSQLite { + dsn = normalizeSQLiteDSN(dsn) + } + return string(driver) + "|" + dsn +} + +// AcquireRemote 获取（或新建）remote 连接；调用方不要 Close。 +func AcquireRemote(driver Driver, dsn string) (*sql.DB, error) { + return sharedRemotePool.acquire(driver, dsn) +} + +func (p *remotePool) acquire(driver Driver, dsn string) (*sql.DB, error) { + key := poolKey(driver, dsn) + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.dbs[key]; ok && e.db != nil { + if err := e.db.Ping(); err == nil { + e.lastUsed = time.Now() + return e.db, nil + } + _ = e.db.Close() + delete(p.dbs, key) + } + db, err := Open(driver, dsn) + if err != nil { + return nil, err + } + p.dbs[key] = &pooledDB{db: db, driver: driver, lastUsed: time.Now()} + return db, nil +} + +// WarmRemote 通道启动时预热，把冷开成本挪出请求路径。 +func WarmRemote(driver Driver, dsn string) error { + if strings.TrimSpace(dsn) == "" { + return fmt.Errorf("dsn required") + } + _, err := AcquireRemote(driver, dsn) + return err +} + +// InvalidateRemote 通道 DSN 变更或停用时丢掉缓存连接。 +func InvalidateRemote(driver Driver, dsn string) { + key := poolKey(driver, dsn) + sharedRemotePool.mu.Lock() + defer sharedRemotePool.mu.Unlock() + if e, ok := sharedRemotePool.dbs[key]; ok { + _ = e.db.Close() + delete(sharedRemotePool.dbs, key) + } +} + +// CloseAllRemotes 测试或进程退出时关闭池内连接。 +func CloseAllRemotes() { + sharedRemotePool.mu.Lock() + defer sharedRemotePool.mu.Unlock() + for k, e := range sharedRemotePool.dbs { + _ = e.db.Close() + delete(sharedRemotePool.dbs, k) + } +} diff --git a/platform/internal/dbsync/pull.go b/platform/internal/dbsync/pull.go new file mode 100644 index 0000000..2d33261 --- /dev/null +++ b/platform/internal/dbsync/pull.go @@ -0,0 +1,254 @@ +package dbsync + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" +) + +const ( + PullModeBootstrap = "bootstrap" // 全量分页灌库 + PullModeRows = "rows" // 按 row_pks 取行 + PullModePKs = "pks" // 只列主键（客户端本地 diff） +) + +const DefaultPullLimit = 200 +const MaxPullLimit = 500 + +// PullRequest 形态 B：平台只读线上 A，把行/主键返回给本机 agent 写入 B。 +type PullRequest struct { + Mode string `json:"mode"` // bootstrap | rows | pks；空则 bootstrap + Table string `json:"table"` + RowPKs []string `json:"row_pks"` + AfterPK string `json:"after_pk"` // 分页游标（字典序） + Limit int `json:"limit"` + OnlineDBID string `json:"online_db_id"` +} + +// PullItem 下行一条（客户端对本机 B upsert，建议 WithApplying 防回声）。 +type PullItem struct { + Table string `json:"table"` + Op string `json:"op"` // upsert + RowPK string `json:"row_pk"` + Row map[string]any `json:"row,omitempty"` + Version int64 `json:"version"` +} + +// PullResult 下行响应。 +type PullResult struct { + OK bool `json:"ok"` + Mode string `json:"mode"` + Table string `json:"table"` + PKColumn string `json:"pk_column"` + Columns []string `json:"columns,omitempty"` // 表结构；空表时也返回，便于本机建空表 + Items []PullItem `json:"items,omitempty"` + PKs []string `json:"pks,omitempty"` + NextAfterPK string `json:"next_after_pk,omitempty"` + HasMore bool `json:"has_more"` + Message string `json:"message,omitempty"` +} + +// PullFromRemote 从通道 remote（线上 A）读出数据，供本机 agent 写入 B。 +func PullFromRemote(ctx context.Context, ch *Channel, store *FileStore, req PullRequest) (*PullResult, error) { + if ch == nil { + return nil, fmt.Errorf("channel is nil") + } + mode := strings.ToLower(strings.TrimSpace(req.Mode)) + if mode == "" { + mode = PullModeBootstrap + } + switch mode { + case PullModeBootstrap, PullModeRows, PullModePKs: + default: + return nil, fmt.Errorf("unsupported mode: %s (use bootstrap|rows|pks)", req.Mode) + } + + table := strings.TrimSpace(req.Table) + if table == "" { + return nil, fmt.Errorf("table required") + } + if !tableInChannel(ch, table) { + return nil, fmt.Errorf("table %s 不在通道白名单", table) + } + pkCol := "id" + if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" { + pkCol = ch.PKColumns[table] + } + + limit := req.Limit + if limit <= 0 { + limit = DefaultPullLimit + } + if limit > MaxPullLimit { + limit = MaxPullLimit + } + + db, err := AcquireRemote(ch.Remote.Driver, ch.Remote.DSN) + if err != nil { + return nil, wrapOpenRemote(err) + } + + if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil { + return nil, Retryablef("ensure meta: %v", err) + } + + exists, err := tableExists(ctx, db, ch.Remote.Driver, table) + if err != nil { + return nil, Retryablef("table exists: %v", err) + } + if !exists { + return nil, fmt.Errorf("table %s not found on online A; ensure-schema or push first", table) + } + cols, err := listColumns(ctx, db, ch.Remote.Driver, table) + if err != nil { + return nil, Retryablef("columns: %v", err) + } + + res := &PullResult{ + OK: true, + Mode: mode, + Table: table, + PKColumn: pkCol, + Columns: cols, + } + + switch mode { + case PullModePKs: + pks, next, more, err := listPKsPage(ctx, db, ch.Remote.Driver, table, pkCol, strings.TrimSpace(req.AfterPK), limit) + if err != nil { + return nil, Retryablef("list pks: %v", err) + } + res.PKs = pks + res.NextAfterPK = next + res.HasMore = more + res.Message = "pks from online A; client diffs then mode=rows" + case PullModeRows: + if len(req.RowPKs) == 0 { + return nil, fmt.Errorf("row_pks required for mode=rows") + } + if len(req.RowPKs) > MaxPullLimit { + return nil, fmt.Errorf("row_pks limit %d", MaxPullLimit) + } + items, err := fetchRowsByPKs(ctx, db, ch.Remote.Driver, table, pkCol, req.RowPKs) + if err != nil { + return nil, err + } + res.Items = items + res.Message = "rows from online A; apply to local B with applying flag" + default: // bootstrap + items, next, more, err := fetchRowsPage(ctx, db, ch.Remote.Driver, table, pkCol, strings.TrimSpace(req.AfterPK), limit) + if err != nil { + return nil, err + } + res.Items = items + res.NextAfterPK = next + res.HasMore = more + if len(items) == 0 && afterEmpty(req.AfterPK) { + res.Message = "empty table on online A; use columns to CREATE TABLE IF NOT EXISTS on local B" + } else { + res.Message = "bootstrap page from online A; loop until has_more=false; create local table from columns if missing" + } + } + + if store != nil && (len(res.Items) > 0 || len(res.PKs) > 0) { + n := int64(len(res.Items)) + if n == 0 { + n = int64(len(res.PKs)) + } + _ = store.PatchStats(ch.ID, func(c *Channel) { + c.Stats.PulledOK += n + }) + } + return res, nil +} + +func listPKsPage(ctx context.Context, db *sql.DB, driver Driver, table, pkCol, afterPK string, limit int) (pks []string, next string, more bool, err error) { + q, args := buildPKPageQuery(driver, table, pkCol, afterPK, limit+1) + rows, err := db.QueryContext(ctx, q, args...) + if err != nil { + return nil, "", false, err + } + defer rows.Close() + for rows.Next() { + var v any + if err := rows.Scan(&v); err != nil { + return nil, "", false, err + } + pks = append(pks, fmt.Sprint(v)) + } + if err := rows.Err(); err != nil { + return nil, "", false, err + } + if len(pks) > limit { + more = true + pks = pks[:limit] + } + if len(pks) > 0 { + next = pks[len(pks)-1] + } + return pks, next, more, nil +} + +func fetchRowsPage(ctx context.Context, db *sql.DB, driver Driver, table, pkCol, afterPK string, limit int) (items []PullItem, next string, more bool, err error) { + pks, next, more, err := listPKsPage(ctx, db, driver, table, pkCol, afterPK, limit) + if err != nil { + return nil, "", false, Retryablef("list pks: %v", err) + } + items, err = fetchRowsByPKs(ctx, db, driver, table, pkCol, pks) + if err != nil { + return nil, "", false, err + } + return items, next, more, nil +} + +func fetchRowsByPKs(ctx context.Context, db *sql.DB, driver Driver, table, pkCol string, pks []string) ([]PullItem, error) { + items := make([]PullItem, 0, len(pks)) + for _, pk := range pks { + pk = strings.TrimSpace(pk) + if pk == "" { + continue + } + _, row, err := FetchRowJSON(ctx, db, driver, table, pkCol, pk) + if err == sql.ErrNoRows { + continue + } + if err != nil { + return nil, Retryablef("fetch row: %v", err) + } + ver, has, _ := GetMetaVersion(ctx, db, driver, table, pk) + if !has || ver <= 0 { + ver = time.Now().UnixNano() + } + items = append(items, PullItem{ + Table: table, + Op: "upsert", + RowPK: pk, + Row: row, + Version: ver, + }) + } + return items, nil +} + +func afterEmpty(afterPK string) bool { + return strings.TrimSpace(afterPK) == "" +} + +func buildPKPageQuery(driver Driver, table, pkCol, afterPK string, limit int) (string, []any) { + t := quoteIdent(driver, table) + p := quoteIdent(driver, pkCol) + switch driver { + case DriverPostgres: + if afterPK != "" { + return fmt.Sprintf(`SELECT %s FROM %s WHERE %s > $1 ORDER BY %s ASC LIMIT $2`, p, t, p, p), []any{afterPK, limit} + } + return fmt.Sprintf(`SELECT %s FROM %s ORDER BY %s ASC LIMIT $1`, p, t, p), []any{limit} + default: + if afterPK != "" { + return fmt.Sprintf(`SELECT %s FROM %s WHERE %s > ? ORDER BY %s ASC LIMIT ?`, p, t, p, p), []any{afterPK, limit} + } + return fmt.Sprintf(`SELECT %s FROM %s ORDER BY %s ASC LIMIT ?`, p, t, p), []any{limit} + } +} diff --git a/platform/internal/dbsync/pull_test.go b/platform/internal/dbsync/pull_test.go new file mode 100644 index 0000000..036e86e --- /dev/null +++ b/platform/internal/dbsync/pull_test.go @@ -0,0 +1,73 @@ +package dbsync + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +func TestPullFromRemoteBootstrapAndRows(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "remote.db") + dsn := "file:" + filepath.ToSlash(dbPath) + "?_pragma=foreign_keys(1)" + + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + if _, err := db.ExecContext(ctx, `CREATE TABLE orders (id TEXT PRIMARY KEY, title TEXT)`); err != nil { + t.Fatal(err) + } + if err := EnsureMeta(ctx, db, DriverSQLite); err != nil { + t.Fatal(err) + } + for _, id := range []string{"a-1", "a-2", "b-1"} { + if _, err := db.ExecContext(ctx, `INSERT INTO orders(id,title) VALUES(?,?)`, id, "t-"+id); err != nil { + t.Fatal(err) + } + _ = UpsertMeta(ctx, db, DriverSQLite, "orders", id, time.Now().UnixNano()) + } + _ = db.Close() + + st, err := NewFileStore(filepath.Join(dir, "dbsync")) + if err != nil { + t.Fatal(err) + } + ch := &Channel{ + ID: "ch1", + TenantID: 1, + Remote: Endpoint{ + Driver: DriverSQLite, + DSN: dsn, + Tables: []string{"orders"}, + }, + PKColumns: map[string]string{"orders": "id"}, + } + + boot, err := PullFromRemote(ctx, ch, st, PullRequest{Mode: PullModeBootstrap, Table: "orders", Limit: 2}) + if err != nil || !boot.OK || len(boot.Items) != 2 || !boot.HasMore { + t.Fatalf("bootstrap page1: %+v err=%v", boot, err) + } + boot2, err := PullFromRemote(ctx, ch, st, PullRequest{ + Mode: PullModeBootstrap, Table: "orders", Limit: 2, AfterPK: boot.NextAfterPK, + }) + if err != nil || len(boot2.Items) != 1 || boot2.HasMore { + t.Fatalf("bootstrap page2: %+v err=%v", boot2, err) + } + + pks, err := PullFromRemote(ctx, ch, st, PullRequest{Mode: PullModePKs, Table: "orders", Limit: 10}) + if err != nil || len(pks.PKs) != 3 { + t.Fatalf("pks: %+v err=%v", pks, err) + } + + rows, err := PullFromRemote(ctx, ch, st, PullRequest{ + Mode: PullModeRows, Table: "orders", RowPKs: []string{"a-2", "missing"}, + }) + if err != nil || len(rows.Items) != 1 || rows.Items[0].RowPK != "a-2" { + t.Fatalf("rows: %+v err=%v", rows, err) + } + CloseAllRemotes() +} diff --git a/platform/internal/dbsync/push.go b/platform/internal/dbsync/push.go index 02f12f0..3a8bff7 100644 --- a/platform/internal/dbsync/push.go +++ b/platform/internal/dbsync/push.go @@ -40,9 +40,7 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI if table == "" { return nil, fmt.Errorf("table required") } - if !tableInChannel(ch, table) { - return nil, fmt.Errorf("table %s 不在通道白名单", table) - } + // Z4 冻结：通道表白名单不再作为拒收依据；Binding + JWT 才是权限源。 pkCol := "id" if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" { pkCol = ch.PKColumns[table] @@ -57,14 +55,21 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI version = time.Now().UnixNano() } - db, err := Open(ch.Remote.Driver, ch.Remote.DSN) + db, err := AcquireRemote(ch.Remote.Driver, ch.Remote.DSN) if err != nil { - return nil, fmt.Errorf("open remote: %w", err) + return nil, wrapOpenRemote(err) } - defer db.Close() if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil { - return nil, fmt.Errorf("ensure meta: %w", err) + return nil, Retryablef("ensure meta: %v", err) + } + + if op != "delete" { + var row map[string]any + _ = json.Unmarshal([]byte(payload), &row) + if err := EnsureTableFromRow(ctx, db, ch.Remote.Driver, table, pkCol, row); err != nil { + return nil, Retryablef("ensure table: %v", err) + } } res := &PushResult{ @@ -74,13 +79,16 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI tgtVer, has, err := GetMetaVersion(ctx, db, ch.Remote.Driver, table, rowPK) if err != nil { - return nil, err + return nil, Retryablef("meta version: %v", err) } // 幂等：同 version 已落地 → 跳过 if has && tgtVer == version { res.OK = true res.Skipped = true res.Message = "already applied (same version)" + if store != nil { + _ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedSkipped++ }) + } return res, nil } if has && tgtVer > version { @@ -109,11 +117,14 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI res.OK = true res.Skipped = true res.Message = "target newer; kept (lww_target)" + if store != nil { + _ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedSkipped++ }) + } return res, nil case PolicyLWWSource: loser := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, rowPK) if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil { - return nil, err + return nil, Retryablef("apply: %v", err) } RecordLwwOverride(store, LwwOverride{ TenantID: ch.TenantID, @@ -130,7 +141,10 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI SourceVer: version, }) if store != nil { - _ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ }) + _ = store.PatchStats(ch.ID, func(c *Channel) { + c.Stats.PushedOK++ + c.Stats.PushedApplied++ + }) } res.OK = true res.Applied = true @@ -150,7 +164,10 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI SourceVer: version, Message: "target version newer than agent push", }) - _ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.Conflicts++ }) + _ = store.PatchStats(ch.ID, func(c *Channel) { + c.Stats.Conflicts++ + c.Stats.PushedSkipped++ + }) } res.OK = true res.Skipped = true @@ -161,10 +178,13 @@ func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushI } if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil { - return nil, err + return nil, Retryablef("apply: %v", err) } if store != nil { - _ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ }) + _ = store.PatchStats(ch.ID, func(c *Channel) { + c.Stats.PushedOK++ + c.Stats.PushedApplied++ + }) } res.OK = true res.Applied = true @@ -192,12 +212,9 @@ func PushBatchToRemote(ctx context.Context, ch *Channel, store *FileStore, items } func tableInChannel(ch *Channel, table string) bool { - for _, t := range uniqueTables(ch.Local.Tables, ch.Remote.Tables) { - if strings.EqualFold(strings.TrimSpace(t), table) { - return true - } - } - return false + // Z4 冻结：通道 tables 仅历史兼容展示，push/pull 不再据此拒收。 + _ = ch + return strings.TrimSpace(table) != "" } func normalizePushPayload(item PushItem, pkCol string) (op string, payload string, rowPK string, err error) { diff --git a/platform/internal/dbsync/push_test.go b/platform/internal/dbsync/push_test.go index 205e613..28ce411 100644 --- a/platform/internal/dbsync/push_test.go +++ b/platform/internal/dbsync/push_test.go @@ -37,10 +37,11 @@ func TestTableInChannel(t *testing.T) { Local: Endpoint{Tables: []string{"orders"}}, Remote: Endpoint{Tables: []string{"orders"}}, } - if !tableInChannel(ch, "orders") { - t.Fatal("expected in") + // Z4：通道表白名单不再拒收 + if !tableInChannel(ch, "orders") || !tableInChannel(ch, "other") { + t.Fatal("expected any non-empty table accepted") } - if tableInChannel(ch, "other") { - t.Fatal("expected out") + if tableInChannel(ch, "") { + t.Fatal("empty table rejected") } } diff --git a/platform/internal/dbsync/retryable.go b/platform/internal/dbsync/retryable.go new file mode 100644 index 0000000..65ec342 --- /dev/null +++ b/platform/internal/dbsync/retryable.go @@ -0,0 +1,51 @@ +package dbsync + +import ( + "errors" + "fmt" + "strings" +) + +// RetryableError 上游 IO / remote 暂不可达，agent 可稍后重试（不换 UUID）。 +type RetryableError struct { + Msg string +} + +func (e *RetryableError) Error() string { + if e == nil { + return "retryable error" + } + return e.Msg +} + +func Retryable(msg string) error { + return &RetryableError{Msg: msg} +} + +func Retryablef(format string, args ...any) error { + return &RetryableError{Msg: fmt.Sprintf(format, args...)} +} + +func IsRetryable(err error) bool { + var r *RetryableError + return errors.As(err, &r) +} + +func wrapOpenRemote(err error) error { + if err == nil { + return nil + } + msg := err.Error() + lower := strings.ToLower(msg) + if strings.Contains(lower, "open remote") || + strings.Contains(lower, "unable to open") || + strings.Contains(lower, "no such file") || + strings.Contains(lower, "locked") || + strings.Contains(lower, "busy") || + strings.Contains(lower, "connection refused") || + strings.Contains(lower, "timeout") || + strings.Contains(lower, "i/o") { + return Retryablef("open remote: %v", err) + } + return Retryablef("open remote: %v", err) +} diff --git a/platform/internal/dbsync/schema.go b/platform/internal/dbsync/schema.go new file mode 100644 index 0000000..74a3f69 --- /dev/null +++ b/platform/internal/dbsync/schema.go @@ -0,0 +1,184 @@ +package dbsync + +import ( + "context" + "fmt" + "strings" +) + +// SchemaTableSpec 本机 → 线上：空表/表结构 ensure 请求中的一张表。 +type SchemaTableSpec struct { + Table string `json:"table"` + PKColumn string `json:"pk_column,omitempty"` + Columns []string `json:"columns"` + OnlineDBID string `json:"online_db_id,omitempty"` +} + +// SchemaEnsureRequest agent 批量 ensure 空表到线上 A。 +type SchemaEnsureRequest struct { + OnlineDBID string `json:"online_db_id"` + Tables []SchemaTableSpec `json:"tables"` +} + +// SchemaEnsureItemResult 单表 ensure 结果。 +type SchemaEnsureItemResult struct { + Table string `json:"table"` + OK bool `json:"ok"` + Created bool `json:"created"` // true=本次新建；false=已存在跳过 + Message string `json:"message,omitempty"` +} + +// SchemaEnsureResult 批量 ensure 响应。 +type SchemaEnsureResult struct { + OK bool `json:"ok"` + Results []SchemaEnsureItemResult `json:"results"` + Message string `json:"message,omitempty"` +} + +// SchemaDescribeRequest 从线上 A 拉取表结构（供本机建空表）。 +type SchemaDescribeRequest struct { + OnlineDBID string `json:"online_db_id"` + Tables []string `json:"tables,omitempty"` // 空=全部业务表 +} + +// SchemaTableDesc 线上表结构描述。 +type SchemaTableDesc struct { + Name string `json:"name"` + PKColumn string `json:"pk_column"` + Columns []string `json:"columns"` + RowCount int64 `json:"row_count"` +} + +// SchemaDescribeResult 表结构列表。 +type SchemaDescribeResult struct { + OK bool `json:"ok"` + Tables []SchemaTableDesc `json:"tables"` + Message string `json:"message,omitempty"` +} + +const MaxSchemaEnsureTables = 100 + +// EnsureSchemasOnRemote 在线上 A 为每张表 CREATE IF NOT EXISTS（空表也建）。 +func EnsureSchemasOnRemote(ctx context.Context, ch *Channel, req SchemaEnsureRequest) (*SchemaEnsureResult, error) { + if ch == nil { + return nil, fmt.Errorf("channel is nil") + } + if len(req.Tables) == 0 { + return nil, fmt.Errorf("tables required") + } + if len(req.Tables) > MaxSchemaEnsureTables { + return nil, fmt.Errorf("tables limit %d", MaxSchemaEnsureTables) + } + db, err := AcquireRemote(ch.Remote.Driver, ch.Remote.DSN) + if err != nil { + return nil, wrapOpenRemote(err) + } + if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil { + return nil, Retryablef("ensure meta: %v", err) + } + + out := &SchemaEnsureResult{OK: true, Results: make([]SchemaEnsureItemResult, 0, len(req.Tables))} + for _, spec := range req.Tables { + table := strings.TrimSpace(spec.Table) + item := SchemaEnsureItemResult{Table: table} + if table == "" { + item.Message = "table required" + out.OK = false + out.Results = append(out.Results, item) + continue + } + if !tableInChannel(ch, table) { + item.Message = "table rejected" + out.OK = false + out.Results = append(out.Results, item) + continue + } + pkCol := strings.TrimSpace(spec.PKColumn) + if pkCol == "" && ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" { + pkCol = ch.PKColumns[table] + } + if pkCol == "" { + pkCol = "id" + } + existed, err := tableExists(ctx, db, ch.Remote.Driver, table) + if err != nil { + return nil, Retryablef("table exists: %v", err) + } + if err := EnsureTableFromColumns(ctx, db, ch.Remote.Driver, table, pkCol, spec.Columns); err != nil { + item.Message = err.Error() + out.OK = false + out.Results = append(out.Results, item) + continue + } + item.OK = true + item.Created = !existed + if existed { + item.Message = "already exists" + } else { + item.Message = "created" + } + out.Results = append(out.Results, item) + } + out.Message = "empty tables ensured on online A; client should also create missing local tables from schema describe" + return out, nil +} + +// DescribeSchemasFromRemote 列出线上 A 业务表结构（含空表），供本机 CREATE IF NOT EXISTS。 +func DescribeSchemasFromRemote(ctx context.Context, ch *Channel, req SchemaDescribeRequest) (*SchemaDescribeResult, error) { + if ch == nil { + return nil, fmt.Errorf("channel is nil") + } + db, err := AcquireRemote(ch.Remote.Driver, ch.Remote.DSN) + if err != nil { + return nil, wrapOpenRemote(err) + } + if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil { + return nil, Retryablef("ensure meta: %v", err) + } + + want := map[string]struct{}{} + for _, t := range req.Tables { + t = strings.TrimSpace(t) + if t != "" { + want[t] = struct{}{} + } + } + + names, err := ListTables(ctx, db, ch.Remote.Driver) + if err != nil { + return nil, Retryablef("list tables: %v", err) + } + + out := &SchemaDescribeResult{OK: true, Tables: make([]SchemaTableDesc, 0, len(names))} + for _, name := range names { + if len(want) > 0 { + if _, ok := want[name]; !ok { + continue + } + } + if !tableInChannel(ch, name) { + continue + } + pkCol := "id" + if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[name]) != "" { + pkCol = ch.PKColumns[name] + } + cols, err := listColumns(ctx, db, ch.Remote.Driver, name) + if err != nil { + return nil, Retryablef("columns %s: %v", name, err) + } + n, _ := countRows(ctx, db, ch.Remote.Driver, name) + out.Tables = append(out.Tables, SchemaTableDesc{ + Name: name, + PKColumn: pkCol, + Columns: cols, + RowCount: n, + }) + } + if len(out.Tables) == 0 { + out.Message = "online A has no business tables yet; push or ensure-schema first" + } else { + out.Message = "use columns to CREATE TABLE IF NOT EXISTS on local B (including empty tables)" + } + return out, nil +} diff --git a/platform/internal/dbsync/schema_test.go b/platform/internal/dbsync/schema_test.go new file mode 100644 index 0000000..ddae361 --- /dev/null +++ b/platform/internal/dbsync/schema_test.go @@ -0,0 +1,94 @@ +package dbsync + +import ( + "context" + "path/filepath" + "testing" +) + +func TestEnsureTableFromColumnsEmpty(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "empty.db")) + "?_pragma=busy_timeout(5000)" + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + cols := []string{"id", "username", "password", "email"} + if err := EnsureTableFromColumns(ctx, db, DriverSQLite, "accounts", "id", cols); err != nil { + t.Fatal(err) + } + var n int64 + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM accounts`).Scan(&n); err != nil || n != 0 { + t.Fatalf("want empty table n=%d err=%v", n, err) + } + // idempotent + if err := EnsureTableFromColumns(ctx, db, DriverSQLite, "accounts", "id", cols); err != nil { + t.Fatal(err) + } +} + +func TestEnsureSchemasOnRemoteCreatesEmpty(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "remote.db")) + "?_pragma=busy_timeout(5000)" + ch := &Channel{ + ID: "ch1", + Remote: Endpoint{ + Driver: DriverSQLite, + DSN: dsn, + }, + } + ctx := context.Background() + defer InvalidateRemote(DriverSQLite, dsn) + res, err := EnsureSchemasOnRemote(ctx, ch, SchemaEnsureRequest{ + Tables: []SchemaTableSpec{{ + Table: "accounts", + PKColumn: "id", + Columns: []string{"id", "username", "email"}, + }}, + }) + if err != nil { + t.Fatal(err) + } + if !res.OK || len(res.Results) != 1 || !res.Results[0].Created { + t.Fatalf("unexpected: %+v", res) + } + desc, err := DescribeSchemasFromRemote(ctx, ch, SchemaDescribeRequest{}) + if err != nil { + t.Fatal(err) + } + if len(desc.Tables) != 1 || desc.Tables[0].Name != "accounts" || desc.Tables[0].RowCount != 0 { + t.Fatalf("describe: %+v", desc) + } + if len(desc.Tables[0].Columns) < 2 { + t.Fatalf("columns: %v", desc.Tables[0].Columns) + } +} + +func TestPullEmptyTableReturnsColumns(t *testing.T) { + dir := t.TempDir() + dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "pull.db")) + "?_pragma=busy_timeout(5000)" + db, err := Open(DriverSQLite, dsn) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := EnsureTableFromColumns(ctx, db, DriverSQLite, "accounts", "id", []string{"id", "username"}); err != nil { + t.Fatal(err) + } + _ = db.Close() + + ch := &Channel{ID: "ch1", Remote: Endpoint{Driver: DriverSQLite, DSN: dsn}} + defer InvalidateRemote(DriverSQLite, dsn) + res, err := PullFromRemote(ctx, ch, nil, PullRequest{Mode: PullModeBootstrap, Table: "accounts"}) + if err != nil { + t.Fatal(err) + } + if len(res.Items) != 0 { + t.Fatalf("want empty items: %+v", res.Items) + } + if len(res.Columns) < 2 { + t.Fatalf("want columns for empty table: %+v", res) + } +} diff --git a/platform/internal/dbsync/types.go b/platform/internal/dbsync/types.go index c10f45e..059104d 100644 --- a/platform/internal/dbsync/types.go +++ b/platform/internal/dbsync/types.go @@ -48,20 +48,25 @@ type Channel struct { Local Endpoint `json:"local"` Remote Endpoint `json:"remote"` PKColumns map[string]string `json:"pk_columns"` // table -> pk col，默认 id - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - LastError string `json:"last_error,omitempty"` - LastSyncAt *time.Time `json:"last_sync_at,omitempty"` - LastReconcileAt *time.Time `json:"last_reconcile_at,omitempty"` - Stats ChannelStats `json:"stats"` + // AgentID / AppSlug：把通道挂到某个智能体及其模块，便于「模块数据进该智能体库」对照。 + AgentID int64 `json:"agent_id,omitempty"` + AppSlug string `json:"app_slug,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + LastError string `json:"last_error,omitempty"` + LastSyncAt *time.Time `json:"last_sync_at,omitempty"` + LastReconcileAt *time.Time `json:"last_reconcile_at,omitempty"` + Stats ChannelStats `json:"stats"` } type ChannelStats struct { - PushedOK int64 `json:"pushed_ok"` - PulledOK int64 `json:"pulled_ok"` - Conflicts int64 `json:"conflicts"` - Retries int64 `json:"retries"` - LastBatch int `json:"last_batch"` + PushedOK int64 `json:"pushed_ok"` // 兼容：成功 apply 次数 + PushedApplied int64 `json:"pushed_applied,omitempty"` // agent/drain 实际写入 + PushedSkipped int64 `json:"pushed_skipped,omitempty"` // 同 version 幂等跳过 + PulledOK int64 `json:"pulled_ok"` + Conflicts int64 `json:"conflicts"` + Retries int64 `json:"retries"` + LastBatch int `json:"last_batch"` } type Conflict struct { diff --git a/platform/internal/handler/agent_sync.go b/platform/internal/handler/agent_sync.go index b18e556..460fd98 100644 --- a/platform/internal/handler/agent_sync.go +++ b/platform/internal/handler/agent_sync.go @@ -2,8 +2,12 @@ package handler import ( "encoding/json" + "log" "net/http" + "strings" + "time" + "aijianzhan/platform/internal/audit" "aijianzhan/platform/internal/authx" "aijianzhan/platform/internal/dbsync" "aijianzhan/platform/internal/svc" @@ -12,19 +16,27 @@ import ( "github.com/zeromicro/go-zero/rest/pathvar" ) -// agent 同步只读白名单 + 推远程 A（形态 B）；需 JWT 含「数据同步」权限（人类管理员或智能体均可）。 +// agent 同步：管理员/智能体（「数据同步」）租户级；普通登录用户按本人 Binding 范围。 func agentSyncWhitelistHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if !requireDBSync(svcCtx, w) { return } - ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) if err != nil { authx.WriteError(w, http.StatusNotFound, err.Error()) return } + if !authorizeUserSyncChannel(svcCtx, w, r, channelID) { + return + } tables := uniqueStringSlice(ch.Local.Tables, ch.Remote.Tables) + hint := "通道 tables 仅历史兼容；用户自助以 Binding+JWT 为权限，整库表均可 push（可不改白名单）" + if len(tables) == 0 { + hint = "通道表白名单为空；以 Binding 为准，接受任意表 push（可自动建表）" + } httpx.OkJson(w, map[string]any{ "channel_id": ch.ID, "name": ch.Name, @@ -33,17 +45,20 @@ func agentSyncWhitelistHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { "conflict_policy": ch.ConflictPolicy, "tables": tables, "pk_columns": ch.PKColumns, - "hint": "本机 agent 缓存此表白名单；仅白名单表走 local_dbsync", + "hint": hint, }) } } func agentSyncPushHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + reqID := r.Header.Get("X-Request-Id") if !requireDBSync(svcCtx, w) { return } - ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) if err != nil { authx.WriteError(w, http.StatusNotFound, err.Error()) return @@ -53,21 +68,46 @@ func agentSyncPushHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { authx.WriteError(w, http.StatusBadRequest, err.Error()) return } + if !authorizeUserSyncPush(svcCtx, w, r, channelID, item) { + return + } res, err := dbsync.PushToRemote(r.Context(), ch, svcCtx.DBSync.Store(), item) + dur := time.Since(started) if err != nil { + logSyncReq(r, "push", channelID, item.Table, item.RowPK, "err", err.Error(), dur, reqID) + auditSync(svcCtx, r, "dbsync.push.error", map[string]any{ + "channel_id": channelID, "table": item.Table, "row_pk": item.RowPK, + "online_db_id": item.OnlineDBID, "error": err.Error(), "ms": dur.Milliseconds(), "req_id": reqID, + }) + if dbsync.IsRetryable(err) { + authx.WriteRetryableError(w, http.StatusServiceUnavailable, err.Error()) + return + } authx.WriteError(w, http.StatusBadRequest, err.Error()) return } + outcome := "applied" + if res != nil && res.Skipped { + outcome = "skipped" + } + logSyncReq(r, "push", channelID, item.Table, item.RowPK, outcome, "", dur, reqID) + auditSync(svcCtx, r, "dbsync.push", map[string]any{ + "channel_id": channelID, "table": item.Table, "row_pk": item.RowPK, + "online_db_id": item.OnlineDBID, "outcome": outcome, "ms": dur.Milliseconds(), "req_id": reqID, + }) httpx.OkJson(w, map[string]any{"success": true, "result": res}) } } func agentSyncPushBatchHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + reqID := r.Header.Get("X-Request-Id") if !requireDBSync(svcCtx, w) { return } - ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) if err != nil { authx.WriteError(w, http.StatusNotFound, err.Error()) return @@ -87,19 +127,282 @@ func agentSyncPushBatchHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { authx.WriteError(w, http.StatusBadRequest, "items limit 100") return } + for _, item := range body.Items { + if !authorizeUserSyncPush(svcCtx, w, r, channelID, item) { + return + } + } results, err := dbsync.PushBatchToRemote(r.Context(), ch, svcCtx.DBSync.Store(), body.Items) + dur := time.Since(started) if err != nil { - httpx.OkJson(w, map[string]any{ + logSyncReq(r, "push_batch", channelID, "", "", "err", err.Error(), dur, reqID) + auditSync(svcCtx, r, "dbsync.push_batch.error", map[string]any{ + "channel_id": channelID, "n": len(body.Items), "error": err.Error(), + "ms": dur.Milliseconds(), "req_id": reqID, + }) + payload := map[string]any{ "success": false, "error": err.Error(), "results": results, - }) + } + if dbsync.IsRetryable(err) { + payload["retryable"] = true + httpx.WriteJson(w, http.StatusServiceUnavailable, payload) + return + } + httpx.OkJson(w, payload) return } + logSyncReq(r, "push_batch", channelID, "", "", "ok", "", dur, reqID) + auditSync(svcCtx, r, "dbsync.push_batch", map[string]any{ + "channel_id": channelID, "n": len(body.Items), "ms": dur.Milliseconds(), "req_id": reqID, + }) httpx.OkJson(w, map[string]any{"success": true, "results": results}) } } +func logSyncReq(r *http.Request, op, channelID, table, rowPK, outcome, errMsg string, dur time.Duration, reqID string) { + uid := authx.UserID(r.Context()) + if errMsg != "" { + log.Printf("dbsync %s channel=%s table=%s pk=%s user=%d outcome=%s err=%s dur=%s req=%s", + op, channelID, table, rowPK, uid, outcome, errMsg, dur, reqID) + return + } + log.Printf("dbsync %s channel=%s table=%s pk=%s user=%d outcome=%s dur=%s req=%s", + op, channelID, table, rowPK, uid, outcome, dur, reqID) +} + +func auditSync(svcCtx *svc.ServiceContext, r *http.Request, action string, detail map[string]any) { + if svcCtx == nil || svcCtx.Audit == nil { + return + } + _ = svcCtx.Audit.Log(r.Context(), syncTenantID(r), authx.UserID(r.Context()), action, audit.DetailJSON(detail)) +} + +// authorizeUserSyncChannel 无「数据同步」时，须本人 Binding 覆盖该通道。 +func authorizeUserSyncChannel(svcCtx *svc.ServiceContext, w http.ResponseWriter, r *http.Request, channelID string) bool { + if authx.SyncTenantWide(r.Context()) { + return true + } + uid := authx.UserID(r.Context()) + tid := syncTenantID(r) + if !svcCtx.DBSync.Store().UserCanAccessChannel(tid, uid, channelID) { + authx.WriteError(w, http.StatusForbidden, "无权访问该同步通道：请先登记本人 Binding（local_database_id ↔ online_db_id + channel_id）") + return false + } + return true +} + +// authorizeUserSyncPush 无「数据同步」时，须本人 Binding 且 online_db_id 匹配。 +func authorizeUserSyncPush(svcCtx *svc.ServiceContext, w http.ResponseWriter, r *http.Request, channelID string, item dbsync.PushItem) bool { + return authorizeUserOnlineDB(svcCtx, w, r, channelID, item.OnlineDBID, "用户自助 push 须带 online_db_id，且须为本人 Binding") +} + +// authorizeUserOnlineDB：带 online_db_id 时一律按本人 Binding 校验（含有「数据同步」的人类管理员）。 +// 仅「数据同步」且未带 online_db_id 时保持租户级管理路径（智能体/管理员兼容）。 +func authorizeUserOnlineDB(svcCtx *svc.ServiceContext, w http.ResponseWriter, r *http.Request, channelID, onlineDBID, emptyMsg string) bool { + online := strings.TrimSpace(onlineDBID) + wide := authx.SyncTenantWide(r.Context()) + if online == "" { + if wide { + return true + } + if emptyMsg == "" { + emptyMsg = "用户自助须带 online_db_id，且须为本人 Binding" + } + authx.WriteError(w, http.StatusForbidden, emptyMsg) + return false + } + uid := authx.UserID(r.Context()) + tid := syncTenantID(r) + if !svcCtx.DBSync.Store().UserOwnsOnlineDB(tid, uid, channelID, online) { + authx.WriteError(w, http.StatusForbidden, "无权访问该 online_db_id（非本人 Binding）") + return false + } + return true +} + +func agentSyncPullHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return agentSyncPullWithDefaultMode(svcCtx, "") +} + +func agentSyncBootstrapHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return agentSyncPullWithDefaultMode(svcCtx, dbsync.PullModeBootstrap) +} + +func agentSyncPullWithDefaultMode(svcCtx *svc.ServiceContext, forceMode string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + reqID := r.Header.Get("X-Request-Id") + if !requireDBSync(svcCtx, w) { + return + } + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + var req dbsync.PullRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if forceMode != "" { + req.Mode = forceMode + } + if !authorizeUserOnlineDB(svcCtx, w, r, channelID, req.OnlineDBID, "用户自助 pull/bootstrap 须带 online_db_id，且须为本人 Binding") { + return + } + if !authorizeUserSyncChannel(svcCtx, w, r, channelID) { + return + } + res, err := dbsync.PullFromRemote(r.Context(), ch, svcCtx.DBSync.Store(), req) + dur := time.Since(started) + mode := req.Mode + if mode == "" { + mode = dbsync.PullModeBootstrap + } + if err != nil { + logSyncReq(r, "pull/"+mode, channelID, req.Table, "", "err", err.Error(), dur, reqID) + auditSync(svcCtx, r, "dbsync.pull.error", map[string]any{ + "channel_id": channelID, "mode": mode, "table": req.Table, + "error": err.Error(), "ms": dur.Milliseconds(), "req_id": reqID, + }) + if dbsync.IsRetryable(err) { + authx.WriteRetryableError(w, http.StatusServiceUnavailable, err.Error()) + return + } + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + n := 0 + if res != nil { + n = len(res.Items) + len(res.PKs) + } + logSyncReq(r, "pull/"+mode, channelID, req.Table, "", "ok", "", dur, reqID) + auditSync(svcCtx, r, "dbsync.pull", map[string]any{ + "channel_id": channelID, "mode": mode, "table": req.Table, + "n": n, "ms": dur.Milliseconds(), "req_id": reqID, + }) + httpx.OkJson(w, map[string]any{"success": true, "result": res}) + } +} + +func agentSyncSchemaEnsureHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + reqID := r.Header.Get("X-Request-Id") + if !requireDBSync(svcCtx, w) { + return + } + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + var req dbsync.SchemaEnsureRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + online := strings.TrimSpace(req.OnlineDBID) + if online == "" && len(req.Tables) > 0 { + online = strings.TrimSpace(req.Tables[0].OnlineDBID) + req.OnlineDBID = online + } + if !authorizeUserOnlineDB(svcCtx, w, r, channelID, online, "用户自助 schema/ensure 须带 online_db_id，且须为本人 Binding") { + return + } + if !authorizeUserSyncChannel(svcCtx, w, r, channelID) { + return + } + res, err := dbsync.EnsureSchemasOnRemote(r.Context(), ch, req) + dur := time.Since(started) + if err != nil { + logSyncReq(r, "schema/ensure", channelID, "", "", "err", err.Error(), dur, reqID) + auditSync(svcCtx, r, "dbsync.schema_ensure.error", map[string]any{ + "channel_id": channelID, "n": len(req.Tables), "error": err.Error(), + "ms": dur.Milliseconds(), "req_id": reqID, + }) + if dbsync.IsRetryable(err) { + authx.WriteRetryableError(w, http.StatusServiceUnavailable, err.Error()) + return + } + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + n := 0 + created := 0 + if res != nil { + n = len(res.Results) + for _, it := range res.Results { + if it.Created { + created++ + } + } + } + logSyncReq(r, "schema/ensure", channelID, "", "", "ok", "", dur, reqID) + auditSync(svcCtx, r, "dbsync.schema_ensure", map[string]any{ + "channel_id": channelID, "n": n, "created": created, + "ms": dur.Milliseconds(), "req_id": reqID, + }) + httpx.OkJson(w, map[string]any{"success": true, "result": res}) + } +} + +func agentSyncSchemaDescribeHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + started := time.Now() + reqID := r.Header.Get("X-Request-Id") + if !requireDBSync(svcCtx, w) { + return + } + channelID := pathvar.Vars(r)["id"] + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(channelID, syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + var req dbsync.SchemaDescribeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if !authorizeUserOnlineDB(svcCtx, w, r, channelID, req.OnlineDBID, "用户自助 schema 须带 online_db_id，且须为本人 Binding") { + return + } + if !authorizeUserSyncChannel(svcCtx, w, r, channelID) { + return + } + res, err := dbsync.DescribeSchemasFromRemote(r.Context(), ch, req) + dur := time.Since(started) + if err != nil { + logSyncReq(r, "schema", channelID, "", "", "err", err.Error(), dur, reqID) + auditSync(svcCtx, r, "dbsync.schema.error", map[string]any{ + "channel_id": channelID, "error": err.Error(), + "ms": dur.Milliseconds(), "req_id": reqID, + }) + if dbsync.IsRetryable(err) { + authx.WriteRetryableError(w, http.StatusServiceUnavailable, err.Error()) + return + } + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + n := 0 + if res != nil { + n = len(res.Tables) + } + logSyncReq(r, "schema", channelID, "", "", "ok", "", dur, reqID) + auditSync(svcCtx, r, "dbsync.schema", map[string]any{ + "channel_id": channelID, "n": n, "ms": dur.Milliseconds(), "req_id": reqID, + }) + httpx.OkJson(w, map[string]any{"success": true, "result": res}) + } +} + func uniqueStringSlice(a, b []string) []string { seen := map[string]struct{}{} var out []string diff --git a/platform/internal/handler/openapi.yaml b/platform/internal/handler/openapi.yaml index e804553..1a8dcc5 100644 --- a/platform/internal/handler/openapi.yaml +++ b/platform/internal/handler/openapi.yaml @@ -360,15 +360,39 @@ paths: - { $ref: "#/components/parameters/Authorization" } - { $ref: "#/components/parameters/Id" } responses: { "200": { description: OK } } + /api/v1/admin/sync/channels/{id}/inspect: + get: + operationId: inspectSyncChannel + summary: 列出通道线上/本机库表（验同步） + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: { "200": { description: OK } } + /api/v1/admin/sync/channels/{id}/preview: + post: + operationId: previewSyncTable + summary: 预览单表行 + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: { "200": { description: OK } } + /api/v1/admin/sync/channels/{id}/drop-table: + post: + operationId: dropSyncTable + summary: 删除线上/本机业务表（仅当前 side，不同步 DDL） + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: { "200": { description: OK } } /api/v1/admin/sync/bindings: get: operationId: listSyncBindings - summary: 列出本机库↔线上库绑定 + summary: 列出本机库↔线上库绑定（管理员全量；用户仅本人） parameters: [{ $ref: "#/components/parameters/Authorization" }] responses: { "200": { description: OK } } post: operationId: ensureSyncBinding - summary: 登记/更新绑定 + summary: 登记/更新绑定（登录用户可自助；强制本人 user_id） parameters: [{ $ref: "#/components/parameters/Authorization" }] responses: { "200": { description: OK } } /api/v1/admin/sync/conflicts: @@ -380,7 +404,7 @@ paths: /api/v1/agent/sync/channels/{id}/whitelist: get: operationId: agentSyncWhitelist - summary: 本机 agent 拉取表白名单 + summary: 拉取表白名单（「数据同步」或登录用户+本人 Binding） parameters: - { $ref: "#/components/parameters/Authorization" } - { $ref: "#/components/parameters/Id" } @@ -388,11 +412,42 @@ paths: /api/v1/agent/sync/channels/{id}/push: post: operationId: agentSyncPush - summary: 本机 agent 推变更到线上 A + summary: 推变更到线上 A（用户自助须带本人 online_db_id） parameters: - { $ref: "#/components/parameters/Authorization" } - { $ref: "#/components/parameters/Id" } - responses: { "200": { description: OK } } + responses: + "200": + description: OK（含 applied / skipped 幂等） + content: + application/json: + examples: + applied: + value: + success: true + result: + ok: true + applied: true + skipped: false + applied_version: 1710000000000000000 + message: applied + skipped: + value: + success: true + result: + ok: true + applied: false + skipped: true + applied_version: 1710000000000000000 + message: already applied (same version) + "503": + description: remote 暂不可达，可重试 + content: + application/json: + example: + code: 503 + message: "open remote: ..." + retryable: true /api/v1/agent/sync/channels/{id}/push/batch: post: operationId: agentSyncPushBatch @@ -400,7 +455,67 @@ paths: parameters: - { $ref: "#/components/parameters/Authorization" } - { $ref: "#/components/parameters/Id" } - responses: { "200": { description: OK } } + responses: + "200": { description: OK } + "503": { description: remote 暂不可达，retryable } + /api/v1/agent/sync/channels/{id}/pull: + post: + operationId: agentSyncPull + summary: 从线上 A 下行（bootstrap / rows / pks）；用户自助须 online_db_id + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: + "200": + description: OK + content: + application/json: + example: + success: true + result: + ok: true + mode: bootstrap + table: orders + pk_column: id + items: + - table: orders + op: upsert + row_pk: aaaa-bbbb + row: { id: aaaa-bbbb, title: x } + version: 1710000000000000000 + next_after_pk: aaaa-bbbb + has_more: true + "503": { description: remote 暂不可达，retryable } + /api/v1/agent/sync/channels/{id}/bootstrap: + post: + operationId: agentSyncBootstrap + summary: 全量灌库（等同 pull + mode=bootstrap） + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: + "200": { description: OK } + "503": { description: remote 暂不可达，retryable } + /api/v1/agent/sync/channels/{id}/schema: + post: + operationId: agentSyncSchemaDescribe + summary: 拉取线上 A 表结构（含空表）；本机按 columns CREATE IF NOT EXISTS + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: + "200": { description: OK } + "503": { description: remote 暂不可达，retryable } + /api/v1/agent/sync/channels/{id}/schema/ensure: + post: + operationId: agentSyncSchemaEnsure + summary: 本机空表结构推到线上 A（CREATE IF NOT EXISTS，无需 outbox 行） + parameters: + - { $ref: "#/components/parameters/Authorization" } + - { $ref: "#/components/parameters/Id" } + responses: + "200": { description: OK } + "503": { description: remote 暂不可达，retryable } /api/v1/platform/dbsync/lww-overrides: get: operationId: platformLwwOverrides diff --git a/platform/internal/handler/routes.go b/platform/internal/handler/routes.go index ba5b0d0..b32072a 100644 --- a/platform/internal/handler/routes.go +++ b/platform/internal/handler/routes.go @@ -34,6 +34,7 @@ func RegisterHandlers(server *rest.Server, svcCtx *svc.ServiceContext) { perm := authx.RequirePermission tenant := authx.RequireTenant() platformAdmin := authx.RequirePlatformAdmin() + syncPush := authx.RequireSyncPushAccess() appGrant := requireAgentAppGrant(svcCtx) // —— 公开 —— @@ -133,13 +134,20 @@ func RegisterHandlers(server *rest.Server, svcCtx *svc.ServiceContext) { {Method: http.MethodPost, Path: "/api/v1/admin/sync/conflicts/:id/resolve", Handler: chain(syncResolveConflictHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, {Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/reconcile", Handler: chain(syncReconcileHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, {Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/ingest", Handler: chain(syncIngestHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, - {Method: http.MethodGet, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsListHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, - {Method: http.MethodPost, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsEnsureHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, + {Method: http.MethodGet, Path: "/api/v1/admin/sync/channels/:id/inspect", Handler: chain(syncInspectHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, + {Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/preview", Handler: chain(syncPreviewHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, + {Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/drop-table", Handler: chain(syncDropTableHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, + {Method: http.MethodGet, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsListHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsEnsureHandler(svcCtx), rl, authMW, tenant, syncPush)}, - // —— 本机 sync agent（形态 B）：白名单拉取 + 推线上 A；需「数据同步」权限 —— - {Method: http.MethodGet, Path: "/api/v1/agent/sync/channels/:id/whitelist", Handler: chain(agentSyncWhitelistHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, - {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push", Handler: chain(agentSyncPushHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, - {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push/batch", Handler: chain(agentSyncPushBatchHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))}, + // —— 本机 sync agent（形态 B）：白名单 + push/pull；「数据同步」租户级，或登录用户按 Binding 自助 —— + {Method: http.MethodGet, Path: "/api/v1/agent/sync/channels/:id/whitelist", Handler: chain(agentSyncWhitelistHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push", Handler: chain(agentSyncPushHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push/batch", Handler: chain(agentSyncPushBatchHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/pull", Handler: chain(agentSyncPullHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/bootstrap", Handler: chain(agentSyncBootstrapHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/schema", Handler: chain(agentSyncSchemaDescribeHandler(svcCtx), rl, authMW, tenant, syncPush)}, + {Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/schema/ensure", Handler: chain(agentSyncSchemaEnsureHandler(svcCtx), rl, authMW, tenant, syncPush)}, // 存储：POST 创建对象，GET 读取（无 /upload 动词路径；旧路径保留别名防断裂） {Method: http.MethodPost, Path: "/api/v1/storage", Handler: chain(uploadHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm上传文件))}, diff --git a/platform/internal/handler/sync.go b/platform/internal/handler/sync.go index 6688d6d..f0a45d7 100644 --- a/platform/internal/handler/sync.go +++ b/platform/internal/handler/sync.go @@ -3,8 +3,10 @@ package handler import ( "encoding/json" "net/http" + "strings" "time" + "aijianzhan/platform/internal/audit" "aijianzhan/platform/internal/authx" "aijianzhan/platform/internal/dbsync" "aijianzhan/platform/internal/svc" @@ -272,3 +274,128 @@ func syncIngestHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { httpx.OkJson(w, map[string]any{"ok": true, "ingested": n, "hint": "已写入本地并进入 outbox，将同步到线上"}) } } + +// syncInspectHandler 控制台验同步：列出线上/本机库表 + 行数 + 列。 +func syncInspectHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !requireDBSync(svcCtx, w) { + return + } + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + side := strings.TrimSpace(r.URL.Query().Get("side")) + if side == "" { + side = "remote" + } + includeMeta := r.URL.Query().Get("include_sync_meta") == "1" || r.URL.Query().Get("include_sync_meta") == "true" + var ep dbsync.Endpoint + switch side { + case "remote": + ep = ch.Remote + case "local": + ep = ch.Local + default: + authx.WriteError(w, http.StatusBadRequest, "side must be remote|local") + return + } + res, err := dbsync.InspectEndpoint(r.Context(), ep, side, includeMeta) + if err != nil && (res == nil || !res.OK) { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + httpx.OkJson(w, res) + } +} + +// syncPreviewHandler 预览单表内容（默认前 50 行）。 +func syncPreviewHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !requireDBSync(svcCtx, w) { + return + } + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + var body struct { + Side string `json:"side"` // remote|local，默认 remote + Table string `json:"table"` + Limit int `json:"limit"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + side := strings.TrimSpace(body.Side) + if side == "" { + side = "remote" + } + var ep dbsync.Endpoint + switch side { + case "remote": + ep = ch.Remote + case "local": + ep = ch.Local + default: + authx.WriteError(w, http.StatusBadRequest, "side must be remote|local") + return + } + res, err := dbsync.PreviewTable(r.Context(), ep, body.Table, body.Limit) + if err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + httpx.OkJson(w, res) + } +} + +// syncDropTableHandler 控制台删业务表（仅当前 side 的 endpoint；不同步 DDL 到另一侧）。 +func syncDropTableHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !requireDBSync(svcCtx, w) { + return + } + ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r)) + if err != nil { + authx.WriteError(w, http.StatusNotFound, err.Error()) + return + } + var body struct { + Side string `json:"side"` // remote|local，默认 remote + Table string `json:"table"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + side := strings.TrimSpace(body.Side) + if side == "" { + side = "remote" + } + var ep dbsync.Endpoint + switch side { + case "remote": + ep = ch.Remote + case "local": + ep = ch.Local + default: + authx.WriteError(w, http.StatusBadRequest, "side must be remote|local") + return + } + res, err := dbsync.DropTableOnEndpoint(r.Context(), ep, side, body.Table) + if err != nil { + authx.WriteError(w, http.StatusBadRequest, err.Error()) + return + } + if svcCtx.Audit != nil { + _ = svcCtx.Audit.Log(r.Context(), syncTenantID(r), authx.UserID(r.Context()), "dbsync.drop_table", audit.DetailJSON(map[string]any{ + "channel_id": ch.ID, "side": side, "table": body.Table, "dropped": res != nil && res.Dropped, + })) + } + httpx.OkJson(w, res) + } +} diff --git a/platform/internal/handler/sync_binding.go b/platform/internal/handler/sync_binding.go index 43aadf9..7aa9831 100644 --- a/platform/internal/handler/sync_binding.go +++ b/platform/internal/handler/sync_binding.go @@ -17,7 +17,16 @@ func syncBindingsListHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return } localID := r.URL.Query().Get("local_database_id") - list, err := svcCtx.DBSync.Store().ListBindings(syncTenantID(r), localID) + tid := syncTenantID(r) + var ( + list []dbsync.Binding + err error + ) + if authx.SyncTenantWide(r.Context()) { + list, err = svcCtx.DBSync.Store().ListBindings(tid, localID) + } else { + list, err = svcCtx.DBSync.Store().ListBindingsFiltered(tid, authx.UserID(r.Context()), localID) + } if err != nil { authx.WriteError(w, http.StatusBadRequest, err.Error()) return @@ -37,8 +46,14 @@ func syncBindingsEnsureHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { return } body.TenantID = syncTenantID(r) - if body.UserID == 0 { - body.UserID = authx.UserID(r.Context()) + uid := authx.UserID(r.Context()) + if authx.SyncTenantWide(r.Context()) { + if body.UserID == 0 { + body.UserID = uid + } + } else { + // 用户自助：强制绑定到本人，禁止冒用他人 user_id + body.UserID = uid } saved, err := svcCtx.DBSync.Store().EnsureBinding(body) if err != nil { diff --git a/platform/internal/logic/applogic/agents.go b/platform/internal/logic/applogic/agents.go index 41353da..ebe4e84 100644 --- a/platform/internal/logic/applogic/agents.go +++ b/platform/internal/logic/applogic/agents.go @@ -99,11 +99,14 @@ func (l *AgentAdminLogic) Create(req *types.AgentCreateReq) (*types.AgentCreateR return nil, err } acc, secret, err := st.Create(l.ctx, authx.TenantID(l.ctx), authx.UserID(l.ctx), agentstore.CreateInput{ - Name: req.Name, - Perms: perms, - AppSlugs: req.AppSlugs, - Status: agentstore.StatusActive, - RoleID: roleID, + Name: req.Name, + Perms: perms, + AppSlugs: req.AppSlugs, + Status: agentstore.StatusActive, + RoleID: roleID, + ChannelID: req.ChannelID, + OnlineDBID: req.OnlineDBID, + DatabaseName: req.DatabaseName, }) if err != nil { return nil, err @@ -161,6 +164,15 @@ func (l *AgentAdminLogic) Update(agentID int64, req *types.AgentUpdateReq) (*age if req.AppSlugs != nil { in.AppSlugs = req.AppSlugs } + if req.ChannelID != nil { + in.ChannelID = req.ChannelID + } + if req.OnlineDBID != nil { + in.OnlineDBID = req.OnlineDBID + } + if req.DatabaseName != nil { + in.DatabaseName = req.DatabaseName + } if req.RoleID != nil { roleID, perms, err := l.resolvePerms(*req.RoleID, nil) if err != nil { diff --git a/platform/internal/logic/applogic/publish.go b/platform/internal/logic/applogic/publish.go index 43a2b0e..c3ecc0a 100644 --- a/platform/internal/logic/applogic/publish.go +++ b/platform/internal/logic/applogic/publish.go @@ -68,17 +68,18 @@ func (l *PublishLogic) ListApps() (*types.AppListResp, error) { } st := it.Status out = append(out, types.AppListItem{ - AppID: it.AppID, - Slug: it.Slug, - Name: it.Name, - Status: string(st), - StatusLabel: meta.StatusLabelCN(st), - Building: meta.IsBuilding(st), - SchemaName: it.SchemaName, - PageCount: it.PageCount, - EntityCount: it.EntityCount, - UpdatedAt: it.UpdatedAt.UTC().Format(time.RFC3339), - CreatedAt: it.CreatedAt.UTC().Format(time.RFC3339), + AppID: it.AppID, + Slug: it.Slug, + Name: it.Name, + Status: string(st), + StatusLabel: meta.StatusLabelCN(st), + Building: meta.IsBuilding(st), + SchemaName: it.SchemaName, + DatabaseName: it.DatabaseName, + PageCount: it.PageCount, + EntityCount: it.EntityCount, + UpdatedAt: it.UpdatedAt.UTC().Format(time.RFC3339), + CreatedAt: it.CreatedAt.UTC().Format(time.RFC3339), }) } return &types.AppListResp{Items: out, Scope: scope}, nil @@ -115,6 +116,7 @@ func (l *PublishLogic) SaveDraft(slug string, req *types.DraftReq) (*types.AppLi if bp.Storage.Engine == "" { bp.Storage.Engine = "postgres" } + bp.EnsureDefaultImportExport() // 草稿允许尚未完全合法；尽量校验，失败则仍以宽松方式保存关键字段 _ = bp.Validate(slug) @@ -255,12 +257,27 @@ func (l *PublishLogic) Publish(slug string, req *types.PublishReq) (*types.Publi } } + bp.EnsureDefaultImportExport() if err := bp.Validate(slug); err != nil { return nil, err } schemaName := bp.AssignSchemaName(tenantID) dbName := bp.AssignDatabaseName(tenantID) + // Z8c：智能体若绑定了 database_name，新建模块优先落到该库（database_per_app） + if authx.Role(l.ctx) == authx.RoleAgent && l.svcCtx.Agents != nil { + if aid := authx.AgentID(l.ctx); aid > 0 { + if acc, err := l.svcCtx.Agents.Get(l.ctx, tenantID, aid); err == nil && acc != nil { + if dn := strings.TrimSpace(acc.DatabaseName); dn != "" { + if !exists || existing.DatabaseName == "" { + bp.Storage.Mode = "database_per_app" + schemaName = bp.AssignSchemaName(tenantID) + dbName = dn + } + } + } + } + } ddl, err := schema.BuildPostgresDDL(bp) if err != nil { return nil, err diff --git a/platform/internal/meta/migrate.go b/platform/internal/meta/migrate.go index 55ecf2d..bedcfe5 100644 --- a/platform/internal/meta/migrate.go +++ b/platform/internal/meta/migrate.go @@ -124,6 +124,14 @@ CREATE TABLE IF NOT EXISTS platform_meta.role_permissions ( ALTER TABLE platform_meta.agent_accounts ADD COLUMN IF NOT EXISTS role_id BIGINT; +-- Z8b：智能体绑定同步通道 / 线上库 / 模块落库名 +ALTER TABLE platform_meta.agent_accounts + ADD COLUMN IF NOT EXISTS channel_id VARCHAR(128) NOT NULL DEFAULT ''; +ALTER TABLE platform_meta.agent_accounts + ADD COLUMN IF NOT EXISTS online_db_id VARCHAR(128) NOT NULL DEFAULT ''; +ALTER TABLE platform_meta.agent_accounts + ADD COLUMN IF NOT EXISTS database_name VARCHAR(128) NOT NULL DEFAULT ''; + -- 租户邀请码：pending 用户凭码加入已有公司 CREATE TABLE IF NOT EXISTS platform_meta.tenant_invites ( invite_id BIGSERIAL PRIMARY KEY, diff --git a/platform/internal/meta/store.go b/platform/internal/meta/store.go index 0e45dba..feca696 100644 --- a/platform/internal/meta/store.go +++ b/platform/internal/meta/store.go @@ -71,6 +71,7 @@ type AppSummary struct { Name string `json:"name"` Status AppStatus `json:"status"` SchemaName string `json:"schema_name,omitempty"` + DatabaseName string `json:"database_name,omitempty"` PageCount int `json:"page_count"` EntityCount int `json:"entity_count"` UpdatedAt time.Time `json:"updated_at"` @@ -141,13 +142,14 @@ func (s *MemoryStore) ListByTenant(_ context.Context, tenantID int64) ([]AppSumm func summarizeApp(app *AppRecord) AppSummary { sum := AppSummary{ - AppID: app.AppID, - Slug: app.Slug, - Name: app.Name, - Status: app.Status, - SchemaName: app.SchemaName, - UpdatedAt: app.UpdatedAt, - CreatedAt: app.CreatedAt, + AppID: app.AppID, + Slug: app.Slug, + Name: app.Name, + Status: app.Status, + SchemaName: app.SchemaName, + DatabaseName: app.DatabaseName, + UpdatedAt: app.UpdatedAt, + CreatedAt: app.CreatedAt, } if app.Blueprint != nil { sum.PageCount = len(app.Blueprint.Pages) diff --git a/platform/internal/types/types.go b/platform/internal/types/types.go index 6cbbafc..bba033e 100644 --- a/platform/internal/types/types.go +++ b/platform/internal/types/types.go @@ -43,17 +43,18 @@ type PublishResp struct { } type AppListItem struct { - AppID string `json:"app_id"` - Slug string `json:"slug"` - Name string `json:"name"` - Status string `json:"status"` - StatusLabel string `json:"status_label"` // 已发布 / 在建 / 失败 - Building bool `json:"building"` // 在建（含草稿、发布中） - SchemaName string `json:"schema_name,omitempty"` - PageCount int `json:"page_count"` - EntityCount int `json:"entity_count"` - UpdatedAt string `json:"updated_at"` - CreatedAt string `json:"created_at"` + AppID string `json:"app_id"` + Slug string `json:"slug"` + Name string `json:"name"` + Status string `json:"status"` + StatusLabel string `json:"status_label"` // 已发布 / 在建 / 失败 + Building bool `json:"building"` // 在建（含草稿、发布中） + SchemaName string `json:"schema_name,omitempty"` + DatabaseName string `json:"database_name,omitempty"` // database_per_app 时有值 + PageCount int `json:"page_count"` + EntityCount int `json:"entity_count"` + UpdatedAt string `json:"updated_at"` + CreatedAt string `json:"created_at"` } type AppListResp struct { @@ -105,18 +106,24 @@ type TokenResp struct { } type AgentCreateReq struct { - Name string `json:"name"` - RoleID int64 `json:"role_id"` - Permissions []string `json:"permissions"` - AppSlugs []string `json:"app_slugs"` + Name string `json:"name"` + RoleID int64 `json:"role_id"` + Permissions []string `json:"permissions"` + AppSlugs []string `json:"app_slugs"` + ChannelID string `json:"channel_id,omitempty"` + OnlineDBID string `json:"online_db_id,omitempty"` + DatabaseName string `json:"database_name,omitempty"` } type AgentUpdateReq struct { - Name *string `json:"name,omitempty"` - Status *string `json:"status,omitempty"` - RoleID *int64 `json:"role_id,omitempty"` - Permissions *[]string `json:"permissions,omitempty"` - AppSlugs *[]string `json:"app_slugs,omitempty"` + Name *string `json:"name,omitempty"` + Status *string `json:"status,omitempty"` + RoleID *int64 `json:"role_id,omitempty"` + Permissions *[]string `json:"permissions,omitempty"` + AppSlugs *[]string `json:"app_slugs,omitempty"` + ChannelID *string `json:"channel_id,omitempty"` + OnlineDBID *string `json:"online_db_id,omitempty"` + DatabaseName *string `json:"database_name,omitempty"` } type AgentCreateResp struct { diff --git a/platform/internal/userstore/store.go b/platform/internal/userstore/store.go index b1b27c8..a61ea1a 100644 --- a/platform/internal/userstore/store.go +++ b/platform/internal/userstore/store.go @@ -1491,7 +1491,7 @@ func EnsurePlatformAdminUser(ctx context.Context, store Store) { if _, err := store.EnsurePlatformAdmin(ctx, "ljk_admin", "ljk_admin", "平台超级管理员"); err != nil { fmt.Printf("ensure platform admin ljk_admin: %v\n", err) } - ensureDevPhone(ctx, store, "ljk_admin", "13800000000") + ensureDevPhone(ctx, store, "ljk_admin", "13531041945") } func ensureDevPhone(ctx context.Context, store Store, username, phone string) { @@ -1508,12 +1508,17 @@ func ensureDevPhone(ctx context.Context, store Store, username, phone string) { if err != nil || user == nil { return } - if strings.TrimSpace(user.Phone) != "" { + want, nerr := NormalizePhone(phone) + if nerr != nil { + fmt.Printf("ensure phone for %s: %v\n", username, nerr) return } - if _, err := store.BindPhone(ctx, user.UserID, phone); err != nil { - fmt.Printf("ensure phone for %s: %v\n", username, err) - return + cur := strings.TrimSpace(user.Phone) + if cur != want { + if _, err := store.BindPhone(ctx, user.UserID, want); err != nil { + fmt.Printf("ensure phone for %s: %v\n", username, err) + return + } } _, _ = store.SetUsernameLoginDisabled(ctx, user.UserID, true) } diff --git a/scripts/Stop-Stack.ps1 b/scripts/Stop-Stack.ps1 index 8499b3b..592f56f 100644 --- a/scripts/Stop-Stack.ps1 +++ b/scripts/Stop-Stack.ps1 @@ -55,20 +55,46 @@ function Kill-Port([int]$Port) { } function Kill-ByCommand([string]$ProcessName, [string]$Pattern) { - Get-CimInstance Win32_Process -Filter ("Name='{0}'" -f $ProcessName) -ErrorAction SilentlyContinue | Where-Object { - $_.CommandLine -and ($_.CommandLine -match $Pattern) - } | ForEach-Object { - Write-Info (" kill {0} pid={1}" -f $ProcessName, $_.ProcessId) - Kill-Tree ([int]$_.ProcessId) + # WMI/CIM 在 Docker 半挂或本机负载高时会卡死；用限时 Job，超时则跳过 + $job = Start-Job -ScriptBlock { + param($ProcessName, $Pattern) + Get-CimInstance Win32_Process -Filter ("Name='{0}'" -f $ProcessName) -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and ($_.CommandLine -match $Pattern) + } | ForEach-Object { [int]$_.ProcessId } + } -ArgumentList $ProcessName, $Pattern + if (-not (Wait-Job $job -Timeout 5)) { + Stop-Job $job -ErrorAction SilentlyContinue + Remove-Job $job -Force -ErrorAction SilentlyContinue + return + } + $ids = @(Receive-Job $job -ErrorAction SilentlyContinue) + Remove-Job $job -Force -ErrorAction SilentlyContinue + foreach ($procId in $ids) { + if ($procId -gt 0) { + Write-Info (" kill {0} pid={1}" -f $ProcessName, $procId) + Kill-Tree $procId + } } } # ----- 1) Docker first (do NOT return; native may still be running) ----- +# Docker Desktop 半挂时 `docker info` 会无限阻塞，必须限时。 $dockerOk = $false try { - $null = & docker info 2>&1 - if ($LASTEXITCODE -eq 0) { $dockerOk = $true } -} catch { } + $outFile = Join-Path $env:TEMP "aijz-docker-info.out" + $errFile = Join-Path $env:TEMP "aijz-docker-info.err" + $p = Start-Process -FilePath "docker" -ArgumentList @("info") -NoNewWindow -PassThru ` + -RedirectStandardOutput $outFile -RedirectStandardError $errFile + if (-not $p.WaitForExit(8000)) { + try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch {} + Write-Info " docker info timeout (8s); skip compose down" + $dockerOk = $false + } else { + $dockerOk = ($p.ExitCode -eq 0) + } +} catch { + $dockerOk = $false +} if ($dockerOk) { Write-Info "==> docker compose down --remove-orphans" @@ -127,15 +153,9 @@ Kill-ByCommand "node.exe" "vite" Kill-ByCommand "node.exe" "esbuild" Kill-ByCommand "cmd.exe" "npm.*vite|vite\.js" -# Playwright Chromium leftovers from fidelity shots +# Playwright Chromium leftovers from fidelity shots（限时，避免 CIM 卡死） Kill-ByCommand "chrome.exe" "ms-playwright|playwright" Kill-ByCommand "chromium.exe" "ms-playwright|playwright" -Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { - $_.Name -match '^(chrome|chromium)\.exe$' -and $_.CommandLine -match 'ms-playwright|playwright_chromium' -} | ForEach-Object { - Write-Info (" kill playwright browser pid={0}" -f $_.ProcessId) - Kill-Tree ([int]$_.ProcessId) -} # ----- 5) Ports (authoritative) ----- foreach ($port in 8888, 8001, 8180, 5173) { @@ -171,3 +191,4 @@ if ($still.Count -gt 0) { Write-Info "OK stopped (docker + native; ports 8888/8001/8180/5173 free)" exit 0 + diff --git a/web/src/AgentUsersPage.tsx b/web/src/AgentUsersPage.tsx index 7d4568f..e9a2b31 100644 --- a/web/src/AgentUsersPage.tsx +++ b/web/src/AgentUsersPage.tsx @@ -26,10 +26,14 @@ import { AgentAccount, Role, Session, + SyncBinding, + SyncChannel, createAgent, deleteAgent, listAgents, listRoles, + listSyncBindings, + listSyncChannels, rotateAgentSecret, updateAgent, } from "./api"; @@ -39,6 +43,9 @@ type FormValues = { name: string; role_id: number; app_slugs: string[]; + channel_id?: string; + online_db_id?: string; + database_name?: string; }; export function AgentUsersPage(props: { @@ -52,6 +59,8 @@ export function AgentUsersPage(props: { const { message, modal } = AntApp.useApp(); const [agents, setAgents] = useState([]); const [roles, setRoles] = useState([]); + const [channels, setChannels] = useState([]); + const [bindings, setBindings] = useState([]); const [loading, setLoading] = useState(false); const [open, setOpen] = useState(false); const [editing, setEditing] = useState(null); @@ -66,9 +75,16 @@ export function AgentUsersPage(props: { async function refresh() { setLoading(true); try { - const [a, r] = await Promise.all([listAgents(session), listRoles(session)]); + const [a, r, ch, bind] = await Promise.all([ + listAgents(session), + listRoles(session), + listSyncChannels(session).catch(() => ({ items: [] as SyncChannel[] })), + listSyncBindings(session).catch(() => ({ items: [] as SyncBinding[] })), + ]); setAgents(a.items || []); setRoles(r.items || []); + setChannels(ch.items || []); + setBindings(bind.items || []); } catch (e: any) { const msg = e.message || String(e); setError(msg); @@ -94,6 +110,9 @@ export function AgentUsersPage(props: { name: "", role_id: pref?.role_id, app_slugs: [], + channel_id: undefined, + online_db_id: undefined, + database_name: undefined, }); setOpen(true); } @@ -104,6 +123,9 @@ export function AgentUsersPage(props: { name: a.name || "", role_id: a.role_id || undefined, app_slugs: a.app_slugs || [], + channel_id: a.channel_id || undefined, + online_db_id: a.online_db_id || undefined, + database_name: a.database_name || undefined, }); setOpen(true); } @@ -112,11 +134,17 @@ export function AgentUsersPage(props: { const values = await form.validateFields(); setBusy(true); try { + const bindBody = { + channel_id: values.channel_id || "", + online_db_id: values.online_db_id || "", + database_name: (values.database_name || "").trim() || "", + }; if (!editing) { const res = await createAgent(session, { name: values.name.trim(), role_id: values.role_id, app_slugs: values.app_slugs || [], + ...bindBody, }); modal.success({ title: "用户已创建", @@ -135,6 +163,7 @@ export function AgentUsersPage(props: { name: values.name.trim(), role_id: values.role_id, app_slugs: values.app_slugs || [], + ...bindBody, ...(activate ? { status: "active" } : {}), }); message.success(activate ? "已保存并启用" : "已保存"); @@ -173,7 +202,7 @@ export function AgentUsersPage(props: { > 当前租户 #{session.tenantId}。用户挂在本租户下；同角色跨公司靠租户隔离，不靠再拆一套角色。 - 角色请在「角色管理」中维护。 + 角色请在「角色管理」中维护。可绑定同步通道 / 线上库：该智能体发布的模块与松离线数据进同一目标库（Z8）。 { + if (!r.channel_id && !r.online_db_id && !r.database_name) return "—"; + const ch = channels.find((c) => c.id === r.channel_id); + return ( + + {ch?.name || r.channel_id || "无通道"} +
+ + {r.online_db_id || "—"} + {r.database_name ? ` · ${r.database_name}` : ""} + + + ); + }, + }, { title: "操作", width: 320, @@ -363,6 +409,53 @@ export function AgentUsersPage(props: { allowClear /> + 松离线 / 模块落库（Z8） + + + + + {[...new Set(bindings.map((b) => b.online_db_id).filter(Boolean))].map((id) => ( + + + + diff --git a/web/src/AuthShell.tsx b/web/src/AuthShell.tsx index b761a32..65ec9d8 100644 --- a/web/src/AuthShell.tsx +++ b/web/src/AuthShell.tsx @@ -265,7 +265,7 @@ export function LoginShell(props: LoginProps) { type="link" size="small" onClick={() => { - props.onUsername("13800000000"); + props.onUsername("13531041945"); props.onPassword("ljk_admin"); }} > diff --git a/web/src/SyncPage.tsx b/web/src/SyncPage.tsx index 9167a05..e114c2a 100644 --- a/web/src/SyncPage.tsx +++ b/web/src/SyncPage.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from "react"; import { App as AntApp, Button, + Checkbox, Form, Input, InputNumber, @@ -18,13 +19,26 @@ import { PlayCircleOutlined, PauseCircleOutlined, ApiOutlined, + TableOutlined, + EyeOutlined, + DeleteOutlined, } from "@ant-design/icons"; import { + AgentAccount, Session, + SyncBinding, SyncChannel, + SyncInspectResult, + SyncPreviewResult, createSyncChannel, deleteSyncChannel, + dropSyncTable, + inspectSyncChannel, + listAgents, + listApps, + listSyncBindings, listSyncChannels, + previewSyncTable, reconcileSyncChannel, startSyncChannel, stopSyncChannel, @@ -43,6 +57,8 @@ type FormValues = { direction: string; conflict_policy: string; poll_interval_ms: number; + agent_id?: number | null; + app_slug?: string; local: EndpointForm; remote: EndpointForm; pk_columns: string; // table:pk,table:pk @@ -57,6 +73,56 @@ function parsePKs(s: string): Record { return out; } +/** DSN 脱敏；file 库尽量只露文件名 */ +function formatDsnHint(dsn: string): string { + const masked = (dsn || "").replace(/:[^:@/]+@/, ":***@"); + const fileMatch = masked.match(/(?:^|[\\/])([^\\/?#]+\.db)(?:\?|$)/i); + if (fileMatch) return fileMatch[1]; + if (masked.length > 56) return `${masked.slice(0, 56)}…`; + return masked; +} + +function bindingLocalLabel(b: SyncBinding): string { + return (b.database_name || "").trim() || b.local_database_id; +} + +function bindingOnlineLabel(b: SyncBinding): string { + return (b.display_name || "").trim() || b.online_db_id; +} + +/** 联调烟雾 Binding：占位 id，易被误认为「库名对不上」 */ +function isSmokeBinding(b: SyncBinding): boolean { + const note = (b.note || "").toLowerCase(); + if (note.includes("smoke") || note.includes("user-jwt")) return true; + const local = (b.local_database_id || "").toLowerCase(); + const online = (b.online_db_id || "").toLowerCase(); + return local.startsWith("local_") || online.startsWith("online_"); +} + +/** 产品：与本地一致的库名优先，勿用落库文件名当主标题 */ +function channelOnlineTitle(ch: SyncChannel, bindings: SyncBinding[]): string { + const linked = bindings.filter((b) => b.channel_id === ch.id); + const localNames = linked.map((b) => (b.database_name || "").trim()).filter(Boolean); + if (localNames.length) return localNames[0]; + const onlineNames = linked.map((b) => (b.display_name || "").trim()).filter(Boolean); + if (onlineNames.length) return onlineNames[0]; + if ((ch.name || "").trim()) return ch.name.trim(); + return ch.remote?.driver || "线上库"; +} + +function channelOnlineIds(ch: SyncChannel, bindings: SyncBinding[]): string { + const linked = bindings.filter((b) => b.channel_id === ch.id); + const ids = linked.map((b) => b.online_db_id).filter(Boolean); + const uniq = [...new Set(ids)]; + if (!uniq.length) return ""; + // 有可读本地名时副文案带上 local↔online id，便于对账 + const localName = linked.map((b) => (b.database_name || "").trim()).find(Boolean); + if (localName && linked[0]?.local_database_id) { + return `${linked[0].local_database_id} ↔ ${uniq.join(" · ")}`; + } + return uniq.join(" · "); +} + function toChannelBody(v: FormValues, id?: string): Partial { return { id, @@ -64,6 +130,8 @@ function toChannelBody(v: FormValues, id?: string): Partial { direction: v.direction, conflict_policy: v.conflict_policy, poll_interval_ms: v.poll_interval_ms || 500, + agent_id: v.agent_id || undefined, + app_slug: (v.app_slug || "").trim() || undefined, local: { driver: v.local.driver, dsn: v.local.dsn, @@ -94,16 +162,48 @@ export function SyncPage(props: { const { session, busy, setBusy, setError, setInfo } = props; const { message } = AntApp.useApp(); const [items, setItems] = useState([]); + const [bindings, setBindings] = useState([]); + const [agents, setAgents] = useState([]); + const [appOptions, setAppOptions] = useState<{ value: string; label: string }[]>([]); const [loading, setLoading] = useState(false); const [open, setOpen] = useState(false); const [editing, setEditing] = useState(null); + const [showSmokeBindings, setShowSmokeBindings] = useState(false); + const [filterAgentId, setFilterAgentId] = useState(undefined); + const [inspectOpen, setInspectOpen] = useState(false); + const [inspectCh, setInspectCh] = useState(null); + const [inspectSide, setInspectSide] = useState<"remote" | "local">("remote"); + const [inspectData, setInspectData] = useState(null); + const [inspectLoading, setInspectLoading] = useState(false); + const [previewOpen, setPreviewOpen] = useState(false); + const [previewData, setPreviewData] = useState(null); + const [previewLoading, setPreviewLoading] = useState(false); const [form] = Form.useForm(); + const visibleBindings = showSmokeBindings + ? bindings + : bindings.filter((b) => !isSmokeBinding(b)); + const smokeHidden = bindings.length - visibleBindings.length; + const visibleChannels = filterAgentId + ? items.filter((c) => c.agent_id === filterAgentId || agents.find((a) => a.agent_id === filterAgentId)?.channel_id === c.id) + : items; async function refresh() { setLoading(true); try { - const ch = await listSyncChannels(session); + const [ch, bind, ag, apps] = await Promise.all([ + listSyncChannels(session), + listSyncBindings(session).catch(() => ({ items: [] as SyncBinding[] })), + listAgents(session).catch(() => ({ items: [] as AgentAccount[] })), + listApps(session).catch(() => ({ items: [] as { slug: string; name: string; status: string }[] })), + ]); setItems(ch.items || []); + setBindings(bind.items || []); + setAgents(ag.items || []); + setAppOptions( + (apps.items || []) + .filter((a) => a.status === "published" || !a.status) + .map((a) => ({ value: a.slug, label: `${a.name || a.slug} (${a.slug})` })) + ); } catch (e: any) { const msg = e.message || String(e); setError(msg); @@ -125,10 +225,12 @@ export function SyncPage(props: { direction: "local_to_remote", conflict_policy: "lww_source", poll_interval_ms: 500, + agent_id: undefined, + app_slug: undefined, local: { driver: "sqlite", dsn: "file:./data/local.db", tables: "article" }, remote: { - driver: "mysql", - dsn: "user:pass@tcp(127.0.0.1:3306)/app?parseTime=true&charset=utf8mb4", + driver: "postgres", + dsn: "postgres://user:pass@127.0.0.1:5432/app?sslmode=disable", tables: "article", }, pk_columns: "article:id", @@ -143,6 +245,8 @@ export function SyncPage(props: { direction: ch.direction, conflict_policy: ch.conflict_policy, poll_interval_ms: ch.poll_interval_ms, + agent_id: ch.agent_id || undefined, + app_slug: ch.app_slug || undefined, local: { driver: ch.local.driver, dsn: ch.local.dsn, @@ -160,6 +264,78 @@ export function SyncPage(props: { setOpen(true); } + async function openInspect(ch: SyncChannel, side: "remote" | "local" = "remote") { + setInspectCh(ch); + setInspectSide(side); + setInspectOpen(true); + setInspectData(null); + setInspectLoading(true); + try { + const data = await inspectSyncChannel(session, ch.id, { side, include_sync_meta: true }); + setInspectData(data); + } catch (e: any) { + message.error(e.message || String(e)); + setInspectOpen(false); + } finally { + setInspectLoading(false); + } + } + + async function openPreview(table: string) { + if (!inspectCh) return; + setPreviewLoading(true); + setPreviewOpen(true); + setPreviewData(null); + try { + const data = await previewSyncTable(session, inspectCh.id, { + table, + side: inspectSide, + limit: 50, + }); + setPreviewData(data); + } catch (e: any) { + message.error(e.message || String(e)); + setPreviewOpen(false); + } finally { + setPreviewLoading(false); + } + } + + function confirmDropTable(table: string) { + if (!inspectCh) return; + const sideLabel = inspectSide === "remote" ? "线上库" : "本机端（通道 local）"; + Modal.confirm({ + title: `删除表「${table}」？`, + content: ( +
+

+ 将从{sideLabel}删除该表及同步元数据残留。此操作不可恢复。 +

+

+ 松离线不会把删表同步到另一侧。若本机仍有同名表，下次 push/ensure 可能再次出现在线上。 +

+
+ ), + okText: "确认删除", + okType: "danger", + cancelText: "取消", + onOk: async () => { + try { + const res = await dropSyncTable(session, inspectCh.id, { + table, + side: inspectSide, + }); + message.success(res.message || (res.dropped ? "已删除" : "表不存在")); + if (previewData?.table === table) setPreviewOpen(false); + await openInspect(inspectCh, inspectSide); + } catch (e: any) { + message.error(e.message || String(e)); + throw e; + } + }, + }); + } + async function onSave() { const v = await form.validateFields(); setBusy(true); @@ -204,25 +380,39 @@ export function SyncPage(props: { 数据同步 - 仅本公司顶级权限（管理员）可配置；通道按公司隔离。 - 默认客户无感：终端须显式开通 local_dbsync 才走松离线。 - 首期推荐方向「本地 → 线上」、策略「源端覆盖」；覆盖审计仅平台超管可见。 - 开通/迁移说明见文档「数据同步-开通说明」「数据同步-迁移手册」。 + 仅本公司顶级权限（管理员）可配置通道 DSN / 线上库落点；通道按公司隔离。 + 终端用户以 Binding + 登录 JWT 为同步权限； + 库级选「同步」后该库全部表可上云，管理员不必按业务表维护通道白名单。 + 通道可绑定智能体 + 模块：在「用户管理」为智能体填写同步通道/线上库/落库名后，可按智能体筛通道； + 模块发布（智能体 Token）优先写入绑定库。本机库管理模块表见宇恒侧（Z8f）。 + 仅线上/首启可用 pull·bootstrap 从线上灌回本机。默认客户无感：未开通不走松离线。 + 线上库 A 推荐 postgres；多服务器每库一条通道。 - + +
( - - {r.remote.driver} -
- - {(r.remote.dsn || "").replace(/:[^:@/]+@/, ":***@")} - - - ), + render: (_: unknown, r: SyncChannel) => { + const title = channelOnlineTitle(r, bindings); + const ids = channelOnlineIds(r, bindings); + const path = formatDsnHint(r.remote?.dsn || ""); + return ( + + {title} +
+ + {r.remote?.driver} + {ids ? ` · ${ids}` : ""} + {path ? ` · ${path}` : ""} + + + ); + }, + }, + { + title: "智能体/模块", + render: (_: unknown, r: SyncChannel) => { + const ag = agents.find((a) => a.agent_id === r.agent_id); + return ( + + {ag ? ag.name : r.agent_id ? `#${r.agent_id}` : "—"} +
+ + {r.app_slug || "未绑模块"} + + + ); + }, }, { title: "状态", @@ -253,14 +465,25 @@ export function SyncPage(props: { r.enabled ? 运行中 : 已停止, }, { - title: "统计", - render: (_: unknown, r: SyncChannel) => - `↑${r.stats?.pushed_ok || 0} ↓${r.stats?.pulled_ok || 0}`, + title: "推送次数", + render: (_: unknown, r: SyncChannel) => ( + + ↑{r.stats?.pushed_ok || 0} + {r.stats?.pushed_skipped ? `（跳过 ${r.stats.pushed_skipped}）` : ""} +
+ + push 成功次数，非表数 + + + ), }, { title: "操作", render: (_: unknown, r: SyncChannel) => ( + @@ -336,10 +559,234 @@ export function SyncPage(props: { ]} /> + + 库绑定（本地 ↔ 线上） + + + local_database_id 与{" "} + online_db_id{" "} + 本就可以不同（映射关系，不是要求同名）。 + 可读名靠宇恒 ensure 时带 database_name /{" "} + display_name + ；通道「线上」列优先用本地可读名。默认隐藏联调烟雾 Binding（ + local_* / note=smoke）。 + + + setShowSmokeBindings(e.target.checked)} + > + 显示联调烟雾 Binding + {smokeHidden > 0 ? `（已隐藏 ${smokeHidden} 条）` : ""} + + +
( + + {bindingLocalLabel(b)} + {b.database_name ? ( + <> +
+ + {b.local_database_id} + + + ) : null} + + ), + }, + { + title: "线上库名", + render: (_: unknown, b: SyncBinding) => ( + + {bindingOnlineLabel(b)} + {b.display_name ? ( + <> +
+ + {b.online_db_id} + + + ) : null} + + ), + }, + { + title: "映射", + render: (_: unknown, b: SyncBinding) => ( + + {bindingLocalLabel(b)} ↔ {bindingOnlineLabel(b)} + + ), + }, + { + title: "通道", + dataIndex: "channel_id", + render: (id: string) => { + if (!id) return "—"; + const ch = items.find((c) => c.id === id); + if (!ch) { + return ( + + 通道已删除 · {id.slice(0, 8)}… + + ); + } + const tables = (ch.remote?.tables || []).join(", ") || "（按 Binding 任意表）"; + return ( + + {ch.name || id} +
+ + 通道表白名单配置：{tables} + + + ); + }, + }, + { title: "备注", dataIndex: "note", ellipsis: true }, + ]} + /> + 冲突已改为自动 LWW；覆盖审计仅平台超级管理员可查。「同步修复」有最小间隔限流（默认 5 分钟）。 + ↑ 推送次数 = agent push 成功次数，不是业务表数量。 + 本地有、线上没有的表（如尚未 push 的业务表）属正常，不是 Binding 映射错了。 + setInspectOpen(false)} + width={820} + footer={ + + + + + + } + > + {inspectData?.dsn_hint ? ( + + {inspectData.driver} · {inspectData.dsn_hint} + {inspectData.message ? ` · ${inspectData.message}` : ""} + + ) : null} +
!String(t.name || "").startsWith("_ajz_"))} + pagination={false} + size="small" + locale={{ emptyText: "暂无业务表（尚未 push 或连不上库）" }} + columns={[ + { title: "表名", dataIndex: "name" }, + { + title: "行数", + dataIndex: "row_count", + width: 90, + render: (n: number) => {n}, + }, + { + title: "字段数", + dataIndex: "column_count", + width: 90, + }, + { + title: "字段", + dataIndex: "columns", + ellipsis: true, + render: (cols: string[]) => (cols || []).join(", "), + }, + { + title: "操作", + width: 168, + render: (_: unknown, t: { name: string; row_count: number }) => ( + + + + + ), + }, + ]} + /> + + {(inspectData?.tables || []).some((t) => String(t.name || "").startsWith("_ajz_")) ? ( + <> + 已隐藏同步系统表 _ajz_* + （版本/outbox，不是业务模块表）。 + + ) : null} + 本机有、这里没有的表 = 尚未 push/ensure。「删表」只删当前查看侧，不会同步到另一侧。 + + + + setPreviewOpen(false)} + width={960} + footer={ + + } + > +
({ ...row, __k: i }))} + rowKey="__k" + size="small" + scroll={{ x: true }} + pagination={false} + columns={(previewData?.columns || []).map((c) => ({ + title: c, + dataIndex: c, + ellipsis: true, + render: (v: unknown) => + v === null || v === undefined ? ( + null + ) : ( + String(v) + ), + }))} + /> + + + + + - 线上库（可随时改地址） - - @@ -424,7 +903,7 @@ export function SyncPage(props: { name={["remote", "tables"]} label="表（逗号分隔）" rules={[{ required: true }]} - extra="与本地表白名单一致；须满足 UUID TEXT PK + FK 闭包" + extra="与本地提示表一致即可；验同步请用「查看线上表」" > diff --git a/web/src/api.ts b/web/src/api.ts index 4c51b84..2275902 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -537,6 +537,12 @@ export type AgentAccount = { status: string; permissions: string[]; app_slugs: string[]; + /** Z8b：绑定同步通道 */ + channel_id?: string; + /** Z8b：绑定线上库 id */ + online_db_id?: string; + /** Z8b：模块落库名（database_per_app） */ + database_name?: string; created_by: number; created_at: string; last_token_at?: string; @@ -563,7 +569,15 @@ export async function listAgents(session: Session) { export async function createAgent( session: Session, - body: { name: string; role_id: number; app_slugs: string[]; permissions?: string[] } + body: { + name: string; + role_id: number; + app_slugs: string[]; + permissions?: string[]; + channel_id?: string; + online_db_id?: string; + database_name?: string; + } ) { const res = await apiFetch(`/api/v1/admin/agents`, { method: "POST", @@ -587,6 +601,9 @@ export async function updateAgent( role_id?: number; permissions?: string[]; app_slugs?: string[]; + channel_id?: string; + online_db_id?: string; + database_name?: string; } ) { const res = await apiFetch(`/api/v1/admin/agents/${id}`, { @@ -831,10 +848,16 @@ export type SyncChannel = { local: SyncEndpoint; remote: SyncEndpoint; pk_columns: Record; + /** 绑定智能体：模块/库归属对照 */ + agent_id?: number; + /** 绑定已发布模块 slug */ + app_slug?: string; last_error?: string; last_sync_at?: string; stats?: { pushed_ok: number; + pushed_applied?: number; + pushed_skipped?: number; pulled_ok: number; conflicts: number; retries: number; @@ -842,6 +865,32 @@ export type SyncChannel = { }; }; +export type SyncTableInspect = { + name: string; + row_count: number; + columns: string[]; + column_count: number; +}; + +export type SyncInspectResult = { + ok: boolean; + side: string; + driver: string; + dsn_hint?: string; + tables: SyncTableInspect[]; + message?: string; +}; + +export type SyncPreviewResult = { + ok: boolean; + table: string; + columns: string[]; + rows: Record[]; + total: number; + limit: number; + message?: string; +}; + export type SyncConflict = { id: string; channel_id: string; @@ -1002,12 +1051,77 @@ export async function ingestSyncRows( return data as { ok: boolean; ingested: number; hint?: string }; } +/** 控制台验同步：列出通道线上/本机库的表、行数、字段 */ +export async function inspectSyncChannel( + session: Session, + id: string, + opts?: { side?: "remote" | "local"; include_sync_meta?: boolean } +) { + const side = opts?.side || "remote"; + const q = new URLSearchParams({ side }); + if (opts?.include_sync_meta) q.set("include_sync_meta", "1"); + const res = await apiFetch( + `/api/v1/admin/sync/channels/${encodeURIComponent(id)}/inspect?${q}`, + { headers: { Authorization: `Bearer ${session.accessToken}` } } + ); + const data = await readJson(res); + throwIfBad(res, data, "inspect failed"); + return data as SyncInspectResult; +} + +/** 预览单表内容 */ +export async function previewSyncTable( + session: Session, + id: string, + body: { table: string; side?: "remote" | "local"; limit?: number } +) { + const res = await apiFetch(`/api/v1/admin/sync/channels/${encodeURIComponent(id)}/preview`, { + method: "POST", + headers: { + Authorization: `Bearer ${session.accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + const data = await readJson(res); + throwIfBad(res, data, "preview failed"); + return data as SyncPreviewResult; +} + +/** 控制台删业务表（仅当前 side；不同步 DDL 到另一侧） */ +export async function dropSyncTable( + session: Session, + id: string, + body: { table: string; side?: "remote" | "local" } +) { + const res = await apiFetch(`/api/v1/admin/sync/channels/${encodeURIComponent(id)}/drop-table`, { + method: "POST", + headers: { + Authorization: `Bearer ${session.accessToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + const data = await readJson(res); + throwIfBad(res, data, "drop table failed"); + return data as { + ok: boolean; + side: string; + table: string; + dropped: boolean; + meta_cleared?: number; + message?: string; + }; +} + export async function ensureSyncBinding( session: Session, body: { local_database_id: string; online_db_id: string; channel_id?: string; + database_name?: string; + display_name?: string; note?: string; user_id?: number; } @@ -1022,9 +1136,23 @@ export async function ensureSyncBinding( }); const data = await readJson(res); throwIfBad(res, data, "ensure binding failed"); - return data; + return data as SyncBinding; } +export type SyncBinding = { + id: string; + tenant_id: number; + user_id?: number; + local_database_id: string; + online_db_id: string; + channel_id?: string; + database_name?: string; + display_name?: string; + note?: string; + created_at?: string; + updated_at?: string; +}; + export async function listSyncBindings(session: Session, localDatabaseId?: string) { const q = localDatabaseId ? `?local_database_id=${encodeURIComponent(localDatabaseId)}` @@ -1034,7 +1162,7 @@ export async function listSyncBindings(session: Session, localDatabaseId?: strin }); const data = await readJson(res); throwIfBad(res, data, "list bindings failed"); - return data as { items: Array> }; + return data as { items: SyncBinding[] }; } export async function listPlatformLwwOverrides( diff --git a/宇恒-松离线数据同步使用文档.md b/宇恒-松离线数据同步使用文档.md new file mode 100644 index 0000000..123a756 --- /dev/null +++ b/宇恒-松离线数据同步使用文档.md @@ -0,0 +1,430 @@ +# 宇恒 × 智建 · 松离线数据同步使用文档 + +> 面向：**宇恒一号客户端**对接同学 +> 平台侧仓库：智建（ai建站） +> 依据：`松离线-dbsync方案-最终版.md`（冻结） +> 本地测试基址示例：`http://127.0.0.1:8180`；生产示例：`https://aisite.yuxindazhineng.com` + +--- + +## 1. 你要做什么（一句话） + +未开通用户**零改动**。仅当显式 `YXD_SYNC_MODE=local_dbsync` 且表在白名单时：业务写进本机正式库 B（UUID）→ 写 outbox → **本机 agent** 调智建 push API → 落到线上库 A。agent 停了也**不能挡保存**。 + +```text +业务保存 ──► 本机 B（最终 UUID，立刻可见） + │ + ▼ + outbox（积压可接受） + │ + ▼ 本机 agent（可选装） + 智建 POST .../push ──► 线上 A +``` + +--- + +## 2. 硬约束（违反即不合入） + +| # | 要求 | +|---|------| +| H1 | 未开通：保存 / 自增表 / 插件与现网 **行为 diff = 0** | +| H2 | **禁止**自动升为 `local_dbsync`（仅显式配置） | +| H3 | agent 停运 / 推送失败 → **保存仍成功**；心跳只驱动 UI | +| H4 | 默认建表仍自增；仅「同步表」用 UUID TEXT PK | +| H5 | 旧 `online_primary` 客户禁止静默关双写 | +| H6 | 未开通用户：**零同步文案**（无强制状态条、无「已自动合并」） | + +开通判定（写路径）： + +```text +mode == local_dbsync AND 表白名单缓存命中该表 +→ 走松离线旁路 +否则 → 原写路径 +``` + +同表互斥：某表不可同时 HTTP 双写 + dbsync 白名单。 + +--- + +## 3. 模式三分（环境变量） + +| `YXD_SYNC_MODE` | 含义 | 说明 | +|-----------------|------|------| +| `local_only` | 仅本地 | 默认之一 | +| `online_primary` | HTTP 双写 / 离线 pending | **旧客保留**；仅配了 `YXD_ONLINE_API_BASE` 且未写 MODE 时可兼容升为此模式 | +| `local_dbsync` | 松离线 + 白名单 + agent | **必须显式写出**，永不因「配了线上地址」自动升 | + +建议本地测试： + +```bash +# 默认无感：不要设 local_dbsync，或显式： +YXD_SYNC_MODE=local_only +``` + +开通增值： + +```bash +YXD_SYNC_MODE=local_dbsync +YXD_SYNC_AGENT=1 +YXD_ONLINE_API_BASE=http://127.0.0.1:8180 +YXD_SYNC_CHANNEL_ID=<智建控制台通道 ID> +YXD_SYNC_ACCESS_TOKEN= +# 可选 +YXD_SYNC_WHITELIST_TTL_SEC=600 +YXD_SYNC_AGENT_INTERVAL_SEC=5 +YXD_SYNC_WHITELIST_CACHE=cache/db_sync/whitelist.json +``` + +调试兜底表白名单（通道拉取失败前）： + +```bash +YXD_SYNC_WHITELIST=orders,order_items +# 或 +YXD_SYNC_DBSYNC_TABLES=orders,order_items +``` + +--- + +## 4. 智建侧前置（公司管理员） + +1. 登录智建控制台 → **数据同步** → 新建通道。 +2. **生产**：`remote.driver=postgres`，DSN 例：`postgres://user:pass@host:5432/db?sslmode=disable`（须可达）。 +3. 表白名单须 **UUID TEXT/UUID PK** + **外键闭包**。 +4. 方向推荐 **本地 → 线上**；冲突策略推荐 **源端覆盖（lww_source）**。 +5. 保存通过校验后记下 **通道 ID**。 +6. **二选一鉴权**： + - 管理路径：发带「数据同步」的智能体 Token / 管理员 JWT；或 + - **用户自助**：终端用登录用户 JWT；先登记 Binding，push 带本人 `online_db_id`。 + +**单服务器**：一条通道 + 一个 `YXD_SYNC_CHANNEL_ID` 即可。 +**多服务器**：每台线上 Postgres **各建一条通道**；按本机库选择对应 `channel_id`（建议登记 Binding）。 + +Binding（多库 / 用户自助时建议登记；登录 JWT 即可，不必管理员权）： + +```http +POST /api/v1/admin/sync/bindings +Authorization: Bearer +Content-Type: application/json + +{ + "local_database_id": "用户本机库 ID", + "online_db_id": "线上库 ID", + "channel_id": "<通道ID>", + "database_name": "可选，本机库可读名（控制台优先展示）", + "display_name": "可选，线上库可读名（控制台优先展示）", + "note": "可选" +} +``` + +不传可读名时控制台仍显示 id；宇恒有命名库名时建议一并写入，便于 SyncPage「本地库名 ↔ 线上库名」。 + +```http +GET /api/v1/admin/sync/bindings?local_database_id=... +``` + +--- + +## 5. 宇恒侧实现清单 + +### 5.1 写网关旁路（M1，可无 agent） + +对 `insert / update / delete / 写 SQL`： + +1. 读模式与白名单。 +2. **未命中**：原路径，响应形状与现网一致（可多字段，不可少成功语义）。 +3. **命中**： + - insert：缺 id 或非法 id → **自动补**小写带连字符 UUID； + - 写入本机正式库 B； + - append outbox（失败只打日志，**不挡保存**）； + - `sync_status`：`pending` / `local_only_table` 等仅开通用户可见。 + +文案：`需 agent 才上云`（不得暗示已上云）。 + +### 5.2 Outbox + +建议路径（等价即可）： + +```text +cache/db_sync/outbox//pending.jsonl +``` + +每条建议字段： + +| 字段 | 说明 | +|------|------| +| `id` | outbox 记录 UUID | +| `seq` | 单调版本（如 `time.time_ns()`），作 push 的 `version` | +| `op` | `insert` / `update` / `update_by_id` / `delete` | +| `table_name` | 表名 | +| `row_pk` | 行主键 UUID | +| `payload` | 含 `data` / `update_data` / `where_*` | +| `status` | `pending` | + +规则：按 `seq` **保序**消费；失败标记 error 并**停该库后续**，恢复后重试；**禁止**失败后换新 UUID。 + +### 5.3 本机 Agent（M2） + +启用：`YXD_SYNC_AGENT=1` 且 `mode=local_dbsync`。默认可不启。 + +循环建议： + +1. （TTL）拉白名单 → 写本地缓存。 +2. `list_pending` → 逐条 push → 成功 `mark_done` / 失败 `mark_error` 并 break。 +3. 写心跳文件（供 `/sync/status` 展示 `agent.running` / `last_beat`）。 + +**拉白名单** + +```http +GET /api/v1/agent/sync/channels/{channel_id}/whitelist +Authorization: Bearer +``` + +响应要点：`tables`、`pk_columns`、`conflict_policy`。 + +**推单条** + +```http +POST /api/v1/agent/sync/channels/{channel_id}/push +Authorization: Bearer +Content-Type: application/json + +{ + "table": "orders", + "op": "insert", + "row_pk": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "row": { + "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + "title": "demo" + }, + "version": 1710000000000000000, + "client_outbox_id": "outbox-record-uuid", + "online_db_id": "用户自助必填（本人 Binding）；管理员/智能体可选" +} +``` + +| `op` | 平台行为 | +|------|----------| +| `insert` / `update` / `update_by_id` | upsert 到 remote A | +| `delete` | 按 pk 删 A | + +幂等：同 `version` 已落地 → 跳过成功（`skipped=true`）。 +LWW：目标更新 → 按通道策略；源端覆盖会记**超管**审计（公司管理员不可见）。 +重复 push（双 drain）正确性由 version 保证；客户端可用 drain 锁减流量，非平台必做。 +remote 暂不可达：`503` + `retryable:true`。 + +**下行（仅线上 / 灌库 / 补齐）** + +```http +POST /api/v1/agent/sync/channels/{channel_id}/bootstrap +Authorization: Bearer +Content-Type: application/json + +{ + "table": "orders", + "after_pk": "", + "limit": 200, + "online_db_id": "用户自助必填" +} +``` + +或 `POST .../pull`，`mode` 取： + +| mode | 作用 | +|------|------| +| `bootstrap` | 分页全量（循环至 `has_more=false`） | +| `pks` | 只取 A 主键，本机 diff | +| `rows` | 按 `row_pks` 取行 | + +客户端把 `result.items` upsert 进本机 B 时务必 **WithApplying**（或等价），避免回声进 outbox。 +空表：`result.columns` 仍会返回；本机应用其 `CREATE TABLE IF NOT EXISTS`（见下「表结构同步」）。 + +成功响应示例： + +```json +{ + "success": true, + "result": { + "ok": true, + "applied": false, + "skipped": true, + "conflict": false, + "applied_version": 1710000000000000000, + "message": "already applied (same version)", + "client_outbox_id": "outbox-record-uuid" + } +} +``` + +批量：`POST .../push/batch`，body `{ "items": [ ... ] }`，最多 100，保序遇错即停。 + +也可设完整 URL：`YXD_SYNC_PUSH_URL=...`（覆盖默认拼装）。 + +**表结构同步（空表也要建）** + +仅靠 outbox 行 push 时，**空表不会出现在线上**（无变更事件）。同步周期应额外： + +1. **本机 → 线上**：对本机每张业务表（含 0 行）调用 ensure： + +```http +POST /api/v1/agent/sync/channels/{channel_id}/schema/ensure +Authorization: Bearer +Content-Type: application/json + +{ + "online_db_id": "用户自助必填", + "tables": [ + { + "table": "accounts", + "pk_column": "id", + "columns": ["id", "username", "password", "email", "nickname", "status", "created_at", "updated_at"] + } + ] +} +``` + +平台对每张表 `CREATE TABLE IF NOT EXISTS`（列一律 TEXT + 指定 PK）；已存在则跳过。 + +2. **线上 → 本机**：先拉结构，本机缺表则建空表，再 bootstrap 行： + +```http +POST /api/v1/agent/sync/channels/{channel_id}/schema +Authorization: Bearer +Content-Type: application/json + +{ "online_db_id": "用户自助必填" } +``` + +响应 `result.tables[].columns` + `row_count`；对本地缺失表执行 `CREATE TABLE IF NOT EXISTS`（建议 TEXT + PK=`pk_column`）。 +`pull`/`bootstrap` 的 `result.columns` 同样可用于单表建空表。 + +### 5.4 状态接口（建议） + +`GET /database/sync/status`（或你们现有等价路由）对开通用户返回： + +- `mode` +- `local_dbsync.whitelist_tables` / `outbox_pending` +- `agent.enabled_flag` / `running` / `last_error` +- `note`：未开通勿强塞同步 UI + +--- + +## 6. UUID 约定 + +- 标准形式：**小写 + 连字符** `8-4-4-4-12` +- 入库前 `normalize`；无连字符 32 hex 可规范化 +- 默认建表模板**不要**改成 UUID；只给同步表白名单表用 + +SQLite 同步表示例： + +```sql +CREATE TABLE IF NOT EXISTS orders ( + id TEXT PRIMARY KEY NOT NULL, + title TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); +``` + +--- + +## 7. 从旧双写迁入（O7，摘要） + +1. 该表先停 `online_primary` 双写。 +2. 观察无残留。 +3. 再进白名单并切 `local_dbsync`。 +4. 装 agent，看 outbox 清空。 +5. **禁止**静默全员迁移；旧客默认不变。 + +完整步骤见智建仓：`docs/数据同步-迁移手册.md`。 + +--- + +## 8. 联调检查表 + +### 未开通回归（必过） + +- [ ] 不设 `local_dbsync`：插入自增表与现网一致 +- [ ] 无同步强制文案、无强制装 agent +- [ ] 保存不因同步模块报错失败 + +### 开通 + 无 agent + +- [ ] 白名单表本地立刻可见（UUID） +- [ ] outbox 增长；响应提示需 agent 才上云 +- [ ] 非白名单表仍走原路径 + +### 开通 + agent + +- [ ] 拉白名单成功并缓存 +- [ ] push 后线上 A 出现**同一 UUID**（不双行） +- [ ] 重复 push 同 version 幂等 +- [ ] agent 停：仍可本地保存；恢复后按序追上 +- [ ] delete 保序，不换新 UUID + +### 冲突 + +- [ ] 终端**无**冲突处理台 +- [ ] 公司管理员打 conflicts API → 403 +- [ ] 超管可在平台工作台看 LWW 日志 / 回滚 + +--- + +## 9. 智建 API 速查 + +基址：`{YXD_ONLINE_API_BASE}`，鉴权：`Authorization: Bearer ...` +- 管理路径：需「数据同步」 +- **用户自助**：登录用户 JWT + 本人 Binding；push 须带 `online_db_id` +- LWW 仅超管 + +| 方法 | 路径 | 谁用 | +|------|------|------| +| GET | `/api/v1/agent/sync/channels/{id}/whitelist` | agent / 用户 JWT | +| POST | `/api/v1/agent/sync/channels/{id}/push` | agent / 用户 JWT | +| POST | `/api/v1/agent/sync/channels/{id}/push/batch` | agent / 用户 JWT | +| POST | `/api/v1/agent/sync/channels/{id}/pull` | agent / 用户 JWT | +| POST | `/api/v1/agent/sync/channels/{id}/bootstrap` | agent / 用户 JWT | +| POST | `/api/v1/agent/sync/channels/{id}/schema` | agent / 用户 JWT（拉线上表结构，含空表） | +| POST | `/api/v1/agent/sync/channels/{id}/schema/ensure` | agent / 用户 JWT（本机空表建到线上） | +| POST | `/api/v1/agent/sync/channels/{id}/pull` | agent / 用户 JWT（下行） | +| POST | `/api/v1/agent/sync/channels/{id}/bootstrap` | 全量灌库别名 | +| GET/POST | `/api/v1/admin/sync/bindings` | 管理员或用户自助 | +| GET | `/api/v1/platform/dbsync/lww-overrides` | 仅超管 | +| POST | `/api/v1/platform/dbsync/lww-overrides/{id}/rollback` | 仅超管 | + +OpenAPI：`GET /api/v1/meta/openapi.yaml` + +--- + +## 10. 本地联调最小步骤 + +1. 智建本机起栈（`.env` 已是本地：`AIJZ_PUBLIC_BASE_URL=http://127.0.0.1:8180`）。 +2. 控制台建通道：`remote` 优先本机 **Postgres**（与平台同实例或独立库均可）；拷贝通道 ID。 +3. 拿 token（管理员登录或智能体 client_credentials）。 +4. 宇恒设： + +```bash +YXD_SYNC_MODE=local_dbsync +YXD_SYNC_AGENT=1 +YXD_ONLINE_API_BASE=http://127.0.0.1:8180 +YXD_SYNC_CHANNEL_ID=... +YXD_SYNC_ACCESS_TOKEN=... +``` + +5. 对白名单表插一行 → 看本机 B + outbox → agent 推上 A → 比对 UUID。 +6. 多服务器演练：再建第二条通道指向另一 Postgres，换 `CHANNEL_ID` 验证不串库。 + +--- + +## 11. 相关文档（智建仓） + +| 文档 | 内容 | +|------|------| +| `松离线-dbsync方案-最终版.md` | 双方冻结方案 | +| `docs/同步表约定.md` | UUID / 接口约定 | +| `docs/数据同步-开通说明.md` | 管理员开通 | +| `docs/数据同步-迁移手册.md` | 旧客迁移 | +| `docs/数据同步-中间件.md` | 平台能力与 API | +| `docs/发版说明-数据同步.md` | 默认无感发版说明 | + +--- + +**分工提醒**：智建只提供平台通道、校验、push/whitelist/Binding/LWW；**写网关、outbox、agent、未开通回归**由宇恒在己方仓库实现，勿改智建仓业务代码。 diff --git a/智建修改意见.md b/智建修改意见.md index 79d0776..3cfb04f 100644 --- a/智建修改意见.md +++ b/智建修改意见.md @@ -1,8 +1,8 @@ # 智建修改意见（松离线 + UUID 主键一致性） -> **最终方案（冻结）**：[`松离线-dbsync方案-最终版.md`](./松离线-dbsync方案-最终版.md) -> 过程稿：[`松离线-dbsync方案.md`](./松离线-dbsync方案.md) -> 配套：`database_fastapi修改意见.md`；对照 `docs/数据同步-中间件.md`。 +> 配套文档：同目录 `database_fastapi修改意见.md`（宇恒客户端写网关 / 双轨写） +> 目标：离线写入不破坏联网后的**数据一致 + 主键一致**；对账可兜底。 +> 对照能力：本仓库 `docs/数据同步-中间件.md`（dbsync outbox / ingest / reconcile / 冲突队列）。 --- @@ -134,6 +134,16 @@ --- -## 十、一句话 +## 十一、模块导入默认能力（方案 A） -智建 dbsync **已经按主键 upsert / 对账**；「松 + UUID」要改的是 **业务表主键约定 + 写接口认客户端 id**，并与宇恒绑定、回放对齐——不是再造一套同步中间件。 +> **正式条目已写入联调意见，请以此为准：** +> `E:\project\ai建站\联调后修改意见-宇恒松离线.md` → **§5.6 Z9（方案 A）** +> （§0 结论表已挂「Z9 模块导入默认能力」） + +摘要：生成/发布时业务 resource **默认**带 `import`（及 `export`），列表 `actions` 对齐；publish/merge 可兜底。绑定权限解决「谁能调」，方案 A 解决「表是否允许导入」。 + +--- + +## 十二、一句话（同步 + 导入） + +松离线要对齐的是 **UUID 主键 + upsert**；模块灌数要对齐的是 **绑定权限 + 蓝图默认开放 import（方案 A / Z9）**——细则见联调意见 §5.6。 diff --git a/松离线-dbsync方案-最终版.md b/松离线-dbsync方案-最终版.md index 002370a..647d0e0 100644 --- a/松离线-dbsync方案-最终版.md +++ b/松离线-dbsync方案-最终版.md @@ -290,5 +290,3 @@ | 2026-07-31 | 约定 | M1（宇恒 `local_dbsync` 旁路）与 agent 消费 outbox：以本文 + `docs/同步表约定.md` 为接口约定，不在智建仓改宇恒代码 | | 2026-07-31 | M3 智建 | LWW 覆盖审计 `lww_overrides.json`；`GET /api/v1/platform/dbsync/lww-overrides`（仅超管）；公司 conflicts API 403；对账限流；默认 `lww_source`；SyncPage 去冲突台 | | 2026-07-31 | M4 智建 | Binding API；开通说明/O7 迁移手册；中间件文档 LWW 对齐；SyncPage「同步修复」+ opt-in 文案；apidef 补全。**全员默认 / 宇恒灰度由对方与运营决策，本仓不改宇恒** | -| 2026-07-31 | P2 智建 | 超管按 LWW 落败快照回滚单行：`POST /api/v1/platform/dbsync/lww-overrides/:id/rollback` + 平台工作台按钮 | -| 2026-07-31 | 收尾 智建 | OpenAPI 补 sync/agent/lww；yaml TTL/限流；INTEGER PK 拒绝集成测；发版说明 | diff --git a/联调后修改意见-宇恒松离线.md b/联调后修改意见-宇恒松离线.md new file mode 100644 index 0000000..617bdd9 --- /dev/null +++ b/联调后修改意见-宇恒松离线.md @@ -0,0 +1,433 @@ +# 联调后修改意见 · 宇恒松离线（形态 B） + +> 初稿：2026-08-01 +> 修订：2026-08-03（对照智建已合入改动 + 宇恒压测结果回写） +> 再修订：2026-08-03 下午（用户自助 JWT / Binding / 库级策略复测，见 §5.3） +> 再修订：2026-08-04（§5.6 Z9 模块导入默认能力 · 方案 A） +> 再修订：2026-08-04 晚（§5.5 Z8f 本机镜像已通；§5.5.2 Z8g SyncPage） +> 再修订：2026-08-04（§5.7 **Z10 空表双侧建齐**；§0/§1/§5.4 现状回写） +> 再修订：2026-08-05（§5.6 **Z9e/Z9f 存量 import 必须扫库**；`coerce_fields` 复现） +> 来源：宇恒客户端 `yuhengyihao_client` ↔ 本机智建 `127.0.0.1:8180`（gateway）/ `:8888`（platform） +> 依据：`松离线-dbsync方案-最终版.md`、`宇恒-松离线数据同步使用文档.md` + +--- + +## 0. 当前结论 + +| 级别 | 状态 | 说明 | +|------|------|------| +| **硬改（阻塞宇恒上线）** | **无**（松离线主路径） | Z1–Z7、**Z9**、**Z10a/b（智建 API）** 已落实；主路径可上线 | +| **原建议项 2.1–2.5** | **智建侧基本已落实** | 见 §2 对照表；接口请保持兼容 | +| **用户自助 Z1–Z7** | **已落实** | Z2 宇恒复测通过；Z7 Binding 可读名已合入（需宇恒 ensure 时带 `database_name`/`display_name`） | +| **Z8 模块↔本机库+智能体库** | **大部分落实** | Z8a/b/c/e 智建；**Z8f 宇恒本机 `模块·名·资源` 已可管+可 sync**；**Z8d 路径已通**；**Z8g** 展示增强仍待智建（P1，非硬阻塞）见 §5.5.2 | +| **Z9 模块导入默认能力（方案 A）** | **新建已落实；存量未闭环** | Z9a–d 发布时兜底；**Z9e/Z9f 必须做**：旧模块未再发布仍会 `operation import not allowed`（2026-08-05 再复现），见 §5.6 | +| **Z10 空表双侧建齐** | **智建 API 已落实；宇恒待接** | `schema/ensure` + `schema` + pull `columns`；**Z10c 待宇恒** sync 周期调用，见 §5.7 | +| **Z11 蓝图↔库列一致** | **待智建** | 增字段未迁 Postgres → 导入 42703；见 §5.8 | +| **仍建议关注** | 性能/运维 | SQLite remote **高并发 push** 易锁；agent 宜单库串行 drain（属宇恒用法，非平台硬改） | + +宇恒对照脚本： + +- `yxd/app_fastapi/smoke_zhijian_e2e.py` +- `yxd/app_fastapi/smoke_offline_concurrent.py` +- `yxd/app_fastapi/smoke_zhijian_gateway_burst.py`（连续 push / stats） +- `yxd/app_fastapi/smoke_stress_sync.py`（高并发 + gateway 压测） + +复测摘要（改后）：gateway 串行 60/60、`eof502=0`；同 version → `skipped`；`pushed_applied` / `pushed_skipped` 有值；并发 push 加重试可压满（锁冲突靠重试消化）。 + +--- + +## 1. 已验证可用（请保持兼容） + +请**勿破坏**下列契约（宇恒 agent 已按此对接）： + +1. `GET /api/v1/agent/sync/channels/{id}/whitelist` + - 返回含 `tables`、`pk_columns`、`conflict_policy` +2. `POST /api/v1/agent/sync/channels/{id}/push` + - Body：`table, op, row_pk, row/rows, version, client_outbox_id`（用户自助另加 `online_db_id`） + - `op`：`insert|update|update_by_id|delete`（平台侧 insert/update 归一为 upsert 可接受） + - 成功：`{ success: true, result: { ok, applied, skipped, conflict, applied_version, message } }` + - **同 `(table,row_pk,version)` 已落地 → `skipped=true` 且 HTTP 2xx** +3. `POST .../pull`、`POST .../bootstrap` + - 下行灌库；响应含 `columns`（空表也返回列，便于本机建表） +4. `POST .../schema`、`POST .../schema/ensure`（**Z10**） + - 拉线上表结构 / 本机空表结构推到线上（`CREATE IF NOT EXISTS`，无需 outbox 行） +5. 鉴权：管理员 JWT（含「数据同步」）、智能体 Token，或**登录用户 JWT**（须本人 Binding + `online_db_id`）均可调 agent API +6. 错误体优先带可读 `message`；忙/上游失败宜带 `retryable: true`（502/503） + +演示账号联调可用：`phone=13800000001` / `demo123`（超管快捷号见控制台；另见 seed `13531041945`）。 + +--- + +## 2. 原建议项落实对照（2026-08-03） + +| 编号 | 原建议 | 智建现状 | 是否还要改 | +|------|--------|----------|------------| +| **2.1 P1** | gateway 偶发 502 EOF；超时与可重试 JSON | `ProxyTimeoutSec` 已生效；sync 路径加长超时；502 体含 `retryable`；SQLite DSN 规范化 + `busy_timeout`；remote IO → 503+retryable（见方案 changelog 2026-08-01） | **否（已落实）**。若生产仍偶发 EOF，按运维排障，不升格为接口变更 | +| **2.2 P2** | 文档写明多 agent / version 幂等 | `docs/数据同步-中间件.md` 已写「允许重复 push / 客户端 drain 锁优化」 | **否** | +| **2.3 P3** | `pushed_applied` / `pushed_skipped` | `ChannelStats` 已有；burst 联调可见 | **否**。OpenAPI 已标注 applied/skipped，示例可再补一句即可（非必须） | +| **2.4 P4** | Windows `file:` DSN 示例 | 开通/同步表约定已补正斜杠示例；平台规范化反斜杠 | **否** | +| **2.5 P5** | 503/502 + retryable | 已按表落实 | **否** | + +### 2.6 【新增·知会】高并发 push 与 SQLite remote + +**现象**（宇恒 `smoke_stress_sync`）：对同一通道 remote（本机 SQLite）**并行** push 时，大量 `database is locked` / 失败，串行则稳定；加重试后可最终成功，但 p95 延迟明显升高。 + +**边界**： + +- 正确性仍靠 **version 幂等**，不要求平台做 push 租约。 +- **推荐用法**：宇恒 agent **按库串行 drain**（勿多线程齐推同一 SQLite A）。 +- 若线上 A 为 MySQL/Postgres，并发能力通常好于 SQLite；文档可一句带过「SQLite remote 不适合高并发多 writer」。 + +**智建是否必须改**：**否**。可选更低优：remote 打开时统一更长 `busy_timeout`、或文档强调 SQLite 并发限制（已有 busy_timeout 则足够）。 + +--- + +## 3. 明确不需要智建改的部分 + +| 项 | 负责方 | 说明 | +|----|--------|------| +| 本机 drain 互斥锁 / 串行 drain | 宇恒 | 减重复 HTTP 与 SQLite 锁风暴 | +| 写网关 / outbox / UUID 补齐 | 宇恒 | 未开通零感；仅白名单表同步 | +| 「agent 停仍可保存」 | 宇恒 | H3 | +| 终端冲突处理台 | 双方已否决 | 冲突只在平台超管侧 | +| 改宇恒仓库代码 | 禁止由智建代改 | 分工已冻结 | + +--- + +## 4. 联调回归清单（改后复测） + +| 项 | 宇恒侧结果 | +|----|------------| +| whitelist 与通道一致 | ✅ | +| push insert → A 同 UUID | ✅ | +| push update / delete | ✅ | +| 同 version → skipped，A 不增行 | ✅ | +| gateway 连续 push ≥20 无 EOF | ✅（burst 25、stress 串行 60） | +| stats `pushed_applied` / `pushed_skipped` | ✅ | +| 并发 push（SQLite） | ⚠️ 需重试；建议客户端串行 | + +未在每次脚本覆盖：无「数据同步」权限 → 403；旧 version + `lww_source` 全矩阵——以平台单测/文档为准即可。 + +--- + +## 5. 宇恒侧说明（知会） + +- 测试已收紧：mock 对齐 skipped、双 drain 断言 applied==N、update/delete 保序、压测脚本带锁重试。 +- **开通仍是可选**：`YXD_SYNC_MODE=local_dbsync`；用户自助以 Binding 为准（**Z4**：不按通道表白名单拒收）。 +- **空表**：sync 周期须调 **Z10** `schema/ensure`（见 §5.7），否则本机空表不会出现在线上。 +- 原「若采纳 2.1 则稳定性提升」：**已验证提升**；本意见可归档为「历史建议 + 落实对照」，无需再开一轮同名改造。 + +--- + +## 5.1 【新增·2026-08-03】用户自助库级三态（宇恒已开做） + +产品目标：用户登录智能体后，用**自己的全部权限**管理名下数据库/网页；可对每个库选择： + +| 产品态 | 含义 | 宇恒引擎映射 | +|--------|------|--------------| +| **仅本地** `local_only` | 只写本机 | `local_only` | +| **仅线上** `online_only` | 以线上为主 | `online_primary` | +| **同步** `sync` | 本机落库 + outbox 上云 | `local_dbsync` | + +宇恒已落地（本机）： + +- `POST/GET /database/sync/mode`、`GET /database/sync/policies` +- 策略键：`(tenant_id, user_id, local_database_id, database_name)` +- 写网关按命名库策略覆盖全局 `YXD_SYNC_MODE`（默认仍不静默升 sync） +- 数据库区卡片「同步设置」可视化三态；鉴权优先**登录用户 JWT**（`app.client.access_token`） +- 选「同步」后可拉起本机 agent；push 带头里的用户 Bearer + +--- + +## 5.2 【请智建配合改 / 兼容】用户态同步 + +> 不破坏现有管理员通道 + agent Token 联调路径；以下为**用户自助开通**所需平台能力。 + +| 编号 | 优先级 | 诉求 | 说明 | 智建现状（2026-08-04） | +|------|--------|------|------|------------------------| +| **Z1** | **P0** | **登录用户 JWT 可调 agent sync API** | whitelist / push / batch 允许已登录普通用户（本人资源范围）。 | **已落实** | +| **Z2** | **P0** | **权限范围 = 用户自己的库** | 带 `online_db_id` 时一律校验本人 Binding；异库 → 403（含有「数据同步」的人类）。 | **已落实** | +| **Z3** | **P1** | **Binding 支持用户自助登记** | `POST/GET .../bindings` 认用户 JWT；强制本人 `user_id`。 | **已落实** | +| **Z4** | **P1** | **用户选 sync 时白名单策略** | **冻结②**：不按通道表白名单拒收；Binding 库内任意表可 push，可自动建表。 | **已落实** | +| **Z5** | **P2** | **仅线上 / 下行补齐** | A→B / pull / bootstrap。 | **已落实** | +| **Z6** | **P2** | **控制台文案** | Binding 即权限；管理员不负责维护表白名单。 | **已落实** | +| **Z7** | **P1** | **SyncPage / Binding 展示「可读库名 + 映射」** | 通道「线上」列优先可读名；副文案 online_db_id / 落库路径；统计标注为 push 次数；Binding 列表「本地库名 ↔ 线上库名」。 | **已落实（2026-08-04）**：Binding 增 `database_name`/`display_name`；SyncPage 主标题优先本地 `database_name`，↑ 标注「推送次数 / 非表数」。宇恒 ensure 须带可读名，否则仍会回退到通道名或 driver | +| **Z8** | **P0** | **模块数据：本机库可管理 + 按智能体上云** | 见 §5.5：本机 B 管理模块表；push 到智能体绑定的 A；废弃默认双轨 | **大部分落实**：Z8a/b/c/d/e/f；**Z8g 展示增强待智建**（P1） | +| **Z10** | **P0** | **空表两侧建齐** | 见 §5.7：仅靠 outbox 不会建空表 | **智建 API 已落实**；**Z10c 待宇恒**接 ensure/schema | + +### 建议验收（智建改后） + +1. 演示账号登录拿用户 JWT（**无**单独「数据同步」管理员权）→ `GET .../channels/{id}/whitelist` → 2xx。 +2. 同 JWT → `POST .../push` 写入本人 Binding 库 → A 出现同 UUID。 +3. 同 JWT 推他人 `online_db_id` → 403。 +4. 原管理员 / 智能体 Token 路径回归仍通过（不破坏联调结论）。 + +### 宇恒 ↔ 智建分工（本项） + +| 方 | 负责 | +|----|------| +| 宇恒 | 库级三态 UI/API、写路径分流、outbox、用登录态调 push；**默认不按通道表白名单过滤**；**Z10c** 调 schema/ensure | +| 智建 | 用户 JWT 鉴权、Binding 校验、任意表 push；**SyncPage/Binding 按可读名展示（Z7）**；**Z10a/b** 空表 API | + +--- + +## 5.3 【复测·2026-08-03 下午】智建改后 × 宇恒库级策略 + +> 环境：`127.0.0.1:8180` + `:8888`；演示账号 `13800000001` / `demo123` +> 脚本：`yuhengyihao_client/yxd/app_fastapi/smoke_zhijian_e2e.py`、`smoke_user_jwt_policy.py` +> 说明：直连智建验收（当时本机 8080 未起）；push 遇 gateway **502 EOF / 503** 时按 `retryable` 重试后成功。 + +### 对照表 + +| 编号 | 项 | 结果 | 备注 | +|------|----|------|------| +| — | 登录拿用户 JWT | ✅ | `token_len=225`；`/api/v1/auth/me` 可读 | +| — | 经典 e2e（B→outbox→push→A 同 UUID） | ✅ **PASSED** | 首次 push 常 502/503，约 2–4 次重试后成功；二次 insert 同行数≥2 | +| **Z1** | 用户 JWT 拉 whitelist | ✅ | HTTP 200，`tables=['orders']` | +| **Z1** | 用户 JWT 直推本人库 | ✅ | 带 `online_db_id`；A 可见同 UUID（重试后 200，或同 version → `skipped`） | +| **Z3** | 用户 JWT 自助 Binding | ✅ | `POST .../admin/sync/bindings` → 200，返回 `user_id/local/online/channel_id` | +| — | 库级策略 `sync` → 写网关 `local_dbsync` | ✅ | 全局未强制 `YXD_SYNC_MODE=local_dbsync` 时，命名库策略仍进 outbox | +| — | 登录态 agent drain 上云 | ✅ | drain 重试后 `pushed=1`，A 行 title=`policy-sync` | +| **Z2** | 推他人 `online_db_id` → 403 | ✅ **宇恒复测通过（2026-08-04）** | `403 无权访问该 online_db_id（非本人 Binding）`；`smoke_user_jwt_policy.py` PASSED | +| **Z4–Z6** | 白名单 / 下行 / 文案 | ✅ | Z4 冻结②已落实（任意表+自动建表）；Z5 pull；Z6 SyncPage | + +### 现象与建议（运维） + +1. **Gateway 偶发 502 EOF / 503 Request Timeout**，体带 `retryable: true`；正确性仍靠 version 幂等（重试后常见 `skipped=already applied`）。 +2. **推荐**：宇恒 agent / 联调脚本对 push **串行 + 按 retryable 重试**（已在上述 smoke 加重试与 error→pending 恢复）。 +3. **Z2**：**2026-08-04 宇恒复测通过**——异库返回 `403 无权访问该 online_db_id（非本人 Binding）`。 +4. 前端「数据库区 → 同步设置」需本机服务起来后人工点验；本次后端契约已通。 + +### 再测摘要（2026-08-03 17:53） + +| 脚本 | 结果 | +|------|------| +| `smoke_zhijian_e2e.py` | ✅ PASSED（首推即成功，无 502） | +| `smoke_user_jwt_policy.py` | ✅ 主路径 PASSED；**当时 Z2 WARN** 异库仍 applied | + +### 再测摘要（2026-08-04 · 智建 Z2/Z4/Z6 重启后 · 宇恒确认） + +| 脚本 | 结果 | +|------|------| +| `smoke_user_jwt_policy.py` | ✅ **PASSED**；**Z2 OK**（异库 403）；直推 + 库级 sync drain 上云均通过 | + +### 结论 + +- **用户自助主路径（Z1 + Z2 + Z3 + 宇恒库级 sync）已打通**。 +- **Z2（2026-08-04）**：异库 `online_db_id` → **403**（智建已修，宇恒已确认）。 +- **Z4 已冻结并落地**：push 不按通道表白名单拒收；Binding + 用户 JWT 即权限；可自动建表。 + +### 宇恒澄清（2026-08-04 · 对照截图） + +| 现象 | 说明 | 责任方 | +|------|------|--------| +| SyncPage `↑37` | **不是表数**，是通道 `pushed_ok`（push 成功次数）；与线上 `orders` 行数巧合接近 | 智建 Z7 已改列名为「推送次数」并加副文案 | +| 显示 `ai_site_….db` 而非「AI建站智能体API」 | SyncPage 曾用落库 DSN 当主展示；应对齐本地库名，id/路径作副文案 | **智建 Z7**（主标题优先 Binding.`database_name`）；宇恒 ensure 带 `database_name`/`display_name` | +| Binding `local_*` ↔ `online_*`「名字对不上」 | **id 本就可不同**（映射）；烟雾联调造的占位 id。真实库 `6a3df5…` 已补可读名「AI建站智能体API」；默认隐藏 smoke Binding | 智建已清烟雾数据 + UI 过滤；宇恒正式 ensure 带可读名 | +| 本地多表 vs 线上曾只有 `orders` | **当时** remote 仅 `orders`；`accounts` / `填土高度` 尚未 push，**不是**映射错库 | 后续 `填土高度` 已 push 对齐；空表仍需 **Z10 ensure**（见 §5.7） | +| 弹窗 4 表「0 条记录 / 0 字段」 | 卡片统计不准；库内仍有数据 | **宇恒前端**（智建 SyncPage 无此卡片）；可用数据预览或打开 remote db | +| **要看线上表内容验同步** | SyncPage 原先只显示通道名/推送次数 | **智建已加**：「查看线上表」→ 表名/行数/字段 + 行预览；**可「删表」**（仅当前侧，需确认；不同步 DDL） | +| **图1「线上表」≠ 图2 宇恒「数据表」** | 见 §5.4：一边是线上 A，一边是本机 B | **不是串库**；未 push / 未 ensure 的表不会出现在线上 | +| **模块数据按智能体进库** | 模块与 dbsync 原无关联 | **§5.5 Z8 大部分已通**；剩余 **Z8g** 展示与 **Z10c** 空表 | + +同步链路本身可用；展示层勿把「通道文件名 + 推送次数」当成「本地库名 + 表行数」。 + +### 5.4 图1 vs 图2：表名「对不上」说明（2026-08-04 · 现状已回写） + +| | 图1 智建「查看线上表」 | 图2 宇恒「数据库管理 · 数据表」 | +|--|------------------------|--------------------------------| +| **看的是哪边** | 通道 **线上 A**（例：`ai_site_1785779755.db`） | 本机正式库 **B**（「AI建站智能体API」） | +| **联调当时（早）** | 仅 `orders`（约 37 行）+ `_ajz_*` | `accounts`、`orders`、`填土高度…` 等 | +| **回写后（已 push）** | `orders`（1）+ **`填土高度(6标一工区)`（96）**；outbox 全 `done` | 同行数对齐；`accounts` 仍 0 行 | +| **仍可能不一致** | 空表（如 `accounts`）未 ensure → 线上无此表 | 本机有空表；须走 **Z10** `schema/ensure`，单靠 outbox 不会建空表 | +| **如何验同步** | 「查看线上表」预览行；或对照 `_ajz_sync_meta` | 宇恒「数据预览」看本机真实行 | + +**结论**：表名集合不同 = **本机有、线上尚未 push/ensure**，不是 Binding/通道指错库。有行的表走 drain push；**空表必须走 Z10 ensure**。 + +`_ajz_sync_*` 是中间件系统表（版本/outbox），**不是业务模块表**；控制台「查看线上表」默认隐藏（Z8e）。 + +### 5.5 【进展 Z8】模块数据 ↔ 本机库 + 智能体库 + +> **产品原则（2026-08-04 确认）** +> 1. **模块数据也要能在本机库 B 里管理**（宇恒数据库区：建表/增删改查/导入），与普通业务表同一套体验，不是只能在聊天「模块操作」或线上 Postgres 里改。 +> 2. 本机改完后，经松离线 **push → 该智能体绑定的线上库 A**；换机/仅线上可用 **pull/bootstrap** 灌回本机。 +> 3. **哪个智能体的模块 → 进哪个智能体的库**（通道/`online_db_id` 绑定）。 +> **进展**：Z8f 本机镜像已通；智建绑库/筛通道已通；剩余主要是 **Z8g SyncPage 展示** 与空表 **Z10c**。 + +**目标数据流** + +```text +宇恒本机库 B（含模块表，可本地管理） + │ sync / agent drain push + ▼ +智能体绑定的线上库 A（与通道 remote 一致） + │ 智能体 / 控制台「查看线上表」核对 + ▼ +（可选）模块页 CRUD 读同一 A，或本机 B 为唯一写入口 + 同步 +``` + +| 编号 | 优先级 | 诉求 | 说明 | 现状 | +|------|--------|------|------|------| +| **Z8f** | **P0** | **模块表可在本机库管理** | 模块实体表出现在宇恒「AI建站智能体API」类本机库中；数据预览/录入/导入与业务表一致；开通 sync 后进 outbox | **宇恒已通（2026-08-04）**：`local_module_db` 镜像；表名 **`模块·{模块名}·{资源}`**（无下划线）；列含「所属模块」「模块slug」；写网关 → outbox → push | +| **Z8a** | **P0** | **统一「一份表」叙事** | 废弃「模块一套库、松离线又一套」双轨并存为默认；默认：**本机 B 为模块数据管理面**，A 为同步副本（或同库） | **已补开通说明**（`docs/数据同步-开通说明.md`） | +| **Z8b** | **P0** | **智能体选库 / 选通道** | 该智能体绑定 `channel_id` + `online_db_id`；宇恒可选「同步到哪个智能体库」 | **智建已落实**：用户管理可绑通道/线上库/落库名；SyncPage 按智能体筛通道。宇恒开通 sync 时会 `PUT` 通道 `agent_id`/`app_slug`（读 `YXD_SYNC_*`） | +| **Z8c** | **P1** | **线上与本机对齐** | 智能体读模块数据时优先本机 B（离线）或已同步的 A；发布蓝图在本机建表（UUID PK）再 sync，避免只在平台 schema 建一份 | **部分**：智能体新建发布若设 `database_name` → `database_per_app` 落该库；本机建表靠宇恒 Z8f | +| **Z8d** | **P1** | **本地上传 / 编辑 → 上云** | 本机模块表变更经 Binding + push 进该智能体 A；「查看线上表」可核对 | **路径已通**：本机改 `模块·…` 表 → drain → 「查看线上表」应见同表；若仍缺行 = 未 push / 未 drain | +| **Z8e** | **P2** | **查看线上表默认藏 `_ajz_*`** | 减少「表名不对」误解 | **智建已做默认隐藏**（2026-08-04） | +| **Z8g** | **P1** | **本机可见 = 线上可见（展示闭环）** | SyncPage「智能体/模块」勿长期「未绑模块」；「查看线上表」列出本机已有业务表（含 `模块·…`）；模块列优先展示名 | **待智建**，见 §5.5.2 | + +**分工建议** + +| 方 | 负责 | +|----|------| +| 智建 | Z8a 文案；Z8b 智能体↔通道/库绑定；**Z8g SyncPage 模块展示名**；Z10a/b API；开通说明 | +| 宇恒 | **本机库管理模块表（Z8f 已通）**；开通/全量推送时挂通道 `agent_id`/`app_slug`；**Z10c** sync 周期调 schema/ensure | + +**验收（Z8 完成后）** + +1. ✅ 在宇恒本机库能看到并编辑模块相关表（Z8f）。 +2. ✅ 本机改模块行 → sync 后智建「查看线上表」出现同表同行（有数据路径；空表见 Z10）。 +3. ✅ 智能体 X 只落到绑定库；管理员可按智能体筛通道验数（Z8b）。 +4. ✅ 文档写明：本机有、线上无 = 尚未 push/ensure，不是串库。 +5. ⬜ SyncPage「智能体/模块」显示智能体名 + 模块**展示名**（Z8g；目前多显示 slug /「未绑模块」）。 + +### 5.5.1 【联调补丁·2026-08-04】编辑发布撞 page id + +**现象**：编辑模块 `mode=add_pages` 时报 +`page id already exists: record_list`（模型复用了已有页 id，实为要改 API `import`）。 + +**智建已改**：`platform/internal/blueprint/merge.go` — 同 page id **合并 actions**（不再 400）；同 path resource **并集 operations**；仅有 API/按钮更新也视为可发布（`UpdatedResources`/`UpdatedPages`）。需**重启 platform** 后生效。 + +**与 Z9**：Z9 默认带 import 后，此类「仅为开导入而编辑」会减少；本补丁仍保留给「改已有页能力」用。 + +### 5.5.2 【待智建 Z8g】本机可见 ↔ 线上可见（展示增强 · 非硬阻塞） + +> **联调现象（对照两图）** +> - 宇恒「数据表」已有：`orders`、`填土高度(6标一工区)`、**`模块·测·Sheet1`**（及业务表）。 +> - 智建 SyncPage 通道行若未写 `agent_id`/`app_slug` 会显示 **「未绑模块」**；有 slug 时目前多直接显示 slug，未解析模块**展示名**。 +> - 期望：本机库卡片能看见的业务表，同步后「查看线上表」也应能看见（行数允许短暂滞后；**空表走 Z10**）。 +> - **有数据表对齐**靠 push；**空表对齐**靠 Z10，不单靠 Z8g。 + +| 编号 | 优先级 | 诉求 | 说明 | 负责 | +|------|--------|------|------|------| +| **Z8g-1** | **P1** | **SyncPage 模块列可读** | 「智能体/模块」：有 `app_slug` 时解析已发布模块**展示名**（如「测」），slug 作副文案；仅缺绑定才显示「未绑模块」。`agent_id` 有值但 agents 列表未加载时仍显示 `#id`，勿整列空白 | **智建** `web/src/SyncPage.tsx` | +| **Z8g-2** | **P1** | **通道绑定可被宇恒写入** | 保持 `PUT /api/v1/admin/sync/channels/{id}` 可写 `agent_id`、`app_slug`；列表 API 原样返回。宇恒已在 sync/mode、full-push 调 `ensure_channel_module_link`（环境 `YXD_SYNC_AGENT_ID` / `YXD_SYNC_APP_SLUG` / `YXD_SYNC_MODULE_NAME`） | **智建保持契约**；宇恒已接 | +| **Z8g-3** | **P1** | **查看线上表 = 本机业务表子集** | inspect 列出 remote 全部业务表（含中文/含 `模块·`）；继续默认藏 `_ajz_*`。缺表文案明确：「本机有、这里没有 = 尚未 push」 | **智建**（展示已基本具备；验收与文案强化） | +| **Z8g-4** | **P2** | **白名单列勿误导** | Z4 已允许任意表 push；若通道 `tables` 仍只配 `["orders"]`，UI 勿暗示「只能同步 orders」。可标「策略表（参考）」或「整库同步中」 | **智建** SyncPage/配置抽屉 | + +**验收** + +1. 通道已挂 `agent_id=1`、`app_slug=mismatch_test2` → SyncPage 不再显示「未绑模块」，模块列可见「测」或至少 slug。 +2. 本机 `模块·测·Sheet1` 全量 push 后，「查看线上表」出现同名表且行数对齐（幂等 upsert）。 +3. 本机 `填土高度(6标一工区)` / `orders` 与线上一致。 + +--- + +### 5.6 【已落实 Z9】模块导入默认能力（方案 A · 2026-08-04） + +> **联调现象** +> 宇恒绑定账号已具备 `row.import`（导入数据），智能体调用 +> `POST /api/v1/apps/{slug}/{resource}/import` 仍返回 +> `HTTP 400: operation import not allowed`。 +> 根因（已修）：平台在角色权限之外，还校验蓝图 `apis.resources[].operations` 是否含 `import`； +> 旧生成默认只有 `list/get/create/update/delete`。现已默认带 import，且 publish 兜底补全（Z9）。 +> 编辑补丁时模型仍可能只改部分表；**再发布**或依赖 Z9d/`EnsureDefaultImportExport` 即可。 +> 另：同 page id 编辑曾 400，见 §5.5.1（merge 已改）。 + +**产品结论：采用方案 A** + +绑定宇恒账号解决「谁可以调导入接口」；方案 A 解决「业务表是否声明允许导入」，避免「有权限仍 400」。 + +不采用「仅有 `row.import` 就跳过 `hasOp`」（方案 B）作主路径，以免列表页无导入入口、前后端能力不一致。 + +| 编号 | 优先级 | 诉求 | 说明 | 现状 | +|------|--------|------|------|------| +| **Z9a** | **P0** | **生成默认带 import** | 业务 `apis.resources[].operations` 默认含 `list,get,create,update,delete,**import**,**export**`（除非需求明确禁止） | **已落实**：`generation_rules` + `prompt-contract.md` | +| **Z9b** | **P0** | **列表 actions 对齐** | list 页 `actions` 默认含 **`import`**（及可选 `export`） | **已落实**：发布兜底补 list actions | +| **Z9c** | **P0** | **改 prompt / 文档** | `prompt-contract.md`：由「提到导入再加」改为「默认加；明确不要再去掉」 | **已落实** | +| **Z9d** | **P0** | **publish/merge 兜底** | 发布或编辑合并时：业务 resource 缺 `import` 则自动补上（可配置，默认开） | **已落实**：`EnsureDefaultImportExport`；但**只在 publish 路径触发** | +| **Z9e** | **P0（升格）** | **存量模块一次性扫库补齐** | **禁止**依赖用户「碰巧再发布一次」。平台启动或管理接口：遍历租户已发布蓝图，对可写业务 resource 缺 `import/export` 的执行与 Z9d 相同补齐并落库（可干跑+确认）。目标：**任意旧模块** `POST .../import` 不再因缺 ops 400 | **待智建**；2026-08-05 `coerce_fields/items` 仍只有 list/create/… 无 import，联调第三次踩同一坑 | +| **Z9f** | **P1** | **读路径也兜底（双保险）** | `GetBlueprint` / `ResolveResource` / `ImportRows` 入口：若可写 resource 缺 import，**内存补齐后再校验**（或返回明确引导「请点修复导入能力」）。避免「UI 能点导入、API 仍 400」 | **待智建**；宇恒侧已在导入前自动 replace 补 ops，不能替代平台扫存量 | + +**存量复现（必须消灭）** + +```text +POST /api/v1/apps/coerce_fields/items/import +→ HTTP 400: operation import not allowed +蓝图 ops 实为：list, create, get, update, delete ← 无 import +（Z9d 已合入后新建模块正常；未再发布的旧模块仍坏） +``` + +**智建交付标准（Z9e 验收）** + +1. 不手工编辑、不触发「编辑模块」的情况下，对租户内**全部**已发布可写业务 resource 执行扫库后：`hasOp("import")==true`。 +2. 回归：任取 3 个「合入 Z9 之前发布」的旧 slug，直接 `POST .../import` **不再**出现 `operation import not allowed`。 +3. 控制台可选：「数据同步 / 模块管理 → 一键开启全部业务表导入」按钮，调用同一扫库逻辑。 +4. 文档写明：Z9d≠存量修复；上线 Z9 后**必须跑一次 Z9e**，否则用户会反复以为「绑定/权限坏了」。 + +**分工** + +| 方 | 负责 | +|----|------| +| **智建** | **Z9e 扫存量（P0）**、Z9f 读路径兜底；保持 Z9a–d；错误文案区分「蓝图未开 import」vs「账号无 row.import」 | +| 宇恒 / 智能体 | 导入前尽力自动补齐（已做）；表单选对 slug/resource；**不能**代替平台扫全租户 | + +**验收（含存量）** + +1. 新建模块需求**不提**「导入」→ 发布后 `POST .../import` 不再因 `operation import not allowed` 失败。 +2. **旧模块**（Z9 合入前发布、之后未再发布）在跑完 Z9e 后同样不再 400。 +3. 用户明确写「禁止导入」的表可不含 `import`（`disable_default_import` 或表级标记）。 +4. 400 文案可读：指出是蓝图 operations 缺 import，而不是笼统「操作失败」。 +### 5.7 【智建已落实 Z10】空表也要两侧建齐（2026-08-04） + +> **现象** +> 本机有空表（如 `accounts` 0 行）时，线上「查看线上表」没有该表； +> 仅靠 outbox 行 push 时，**没有行就没有建表事件**。同理，线上空表下行时本机若缺表，旧 pull 只给空 `items`、不给列，本机无法建表。 + +| 编号 | 优先级 | 诉求 | 现状 | +|------|--------|------|------| +| **Z10a** | **P0** | 本机空表 → 线上建空表 | **智建已落实** `POST /api/v1/agent/sync/channels/{id}/schema/ensure` | +| **Z10b** | **P0** | 线上空表 → 本机建空表 | **智建已落实** `POST .../schema`；`pull`/`bootstrap` 响应带 `columns` | +| **Z10c** | **P0** | 宇恒 sync 周期调用上述 API | **待宇恒**：drain **前**对本机业务表（含 0 行）ensure；bootstrap **前** schema + 本地 `CREATE IF NOT EXISTS` | + +**契约摘要** + +```http +POST .../schema/ensure +{ "online_db_id": "...", "tables": [{ "table": "accounts", "pk_column": "id", "columns": ["id","username",...] }] } + +POST .../schema +{ "online_db_id": "..." } +→ result.tables[].name / columns / row_count +``` + +列一律按 TEXT + 指定 PK 建空表；已存在幂等跳过。详例见 `宇恒-松离线数据同步使用文档.md`「表结构同步」。 + +**验收** + +1. 本机空表 ensure 后，智建「查看线上表」可见同名 **0 行**表。 +2. 换机 / 仅线上：schema → 本机建空表 → bootstrap 灌行。 +3. 有数据的表仍走原 push；ensure **不替代** outbox 行同步。 + +### 5.8 【待智建 Z11】蓝图增字段须迁 Postgres（2026-08-04） + +> **联调现象** +> 模块 `record` 蓝图后来带了 `title/content/status`，但 Postgres 表仍只有 `col_0/col_1/col_2`。 +> 导入 Excel 若含「标题/内容/状态」→ `pq: 关系 "record" 的 "title" 字段不存在 (42703)`，整批 skipped。 +> 反过来：蓝图把这些标成非空、Excel 又没有时，会先报 `missing required field: title`。 + +| 编号 | 优先级 | 诉求 | 负责 | +|------|--------|------|------| +| **Z11a** | **P1** | 编辑/发布增字段后 **自动 ALTER** 业务表补列（或发布失败并提示未迁库） | **智建** | +| **Z11b** | **P2** | 导入时：Excel 列映射到蓝图有、库无的字段 → 明确报「请先迁库」，勿只堆 28 条 pq 错误 | **智建** | + +**宇恒临时绕过**：导入 Excel 只保留库里已有列对应表头（填土：`序号/断面里程/填土高度` → col_0/1/2）。 + +--- + +## 6. 联系与附件 + +- 方案：`松离线-dbsync方案-最终版.md`（含 2026-08-01 联调建议落地记录） +- 宇恒使用说明：`宇恒-松离线数据同步使用文档.md`（含 Z10 schema/ensure） +- 开通说明：`docs/数据同步-开通说明.md` +- 本意见如与冻结方案冲突，**以冻结方案为准**；§5.1–5.7 为产品增量与复测记录，不推翻 H1–H6 默认无感约束。