feat: ship loose-offline dbsync (validate, agent push, LWW audit)

Add UUID/FK channel checks, agent whitelist/push APIs, bindings, super-admin LWW audit with rollback, reconcile rate limits, and sync docs. Default customers stay opt-in; company conflict UI is removed.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
whm
2026-07-31 17:54:14 +08:00
parent 632057c857
commit 76cdcd760e
39 changed files with 3302 additions and 199 deletions

View File

@@ -60,10 +60,20 @@ var Catalog = []Entry{
{Method: "PUT", Path: "/api/v1/admin/sync/channels/{id}", OperationID: "updateSyncChannel", Summary: "更新同步通道", Group: "admin"},
{Method: "DELETE", Path: "/api/v1/admin/sync/channels/{id}", OperationID: "deleteSyncChannel", Summary: "删除同步通道", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/test", OperationID: "testSyncEndpoints", Summary: "测试本地/线上库连接", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/prepare", OperationID: "prepareSyncChannel", Summary: "准备同步(建 outbox/触发器)", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/start", OperationID: "startSyncChannel", Summary: "启动近实时同步", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/stop", OperationID: "stopSyncChannel", Summary: "停止同步", Group: "admin"},
{Method: "GET", Path: "/api/v1/admin/sync/conflicts", OperationID: "listSyncConflicts", Summary: "冲突队列", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/conflicts/{id}/resolve", OperationID: "resolveSyncConflict", Summary: "解决冲突", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/reconcile", OperationID: "reconcileSyncChannel", Summary: "主键对账(同步修复,有限流)", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/channels/{id}/ingest", OperationID: "ingestSyncRows", Summary: "外部行写入通道 local", Group: "admin"},
{Method: "GET", Path: "/api/v1/admin/sync/bindings", OperationID: "listSyncBindings", Summary: "列出本机库↔线上库绑定", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/bindings", OperationID: "ensureSyncBinding", Summary: "登记/更新绑定", Group: "admin"},
{Method: "GET", Path: "/api/v1/admin/sync/conflicts", OperationID: "listSyncConflicts", Summary: "已废弃:公司侧 403改用超管 LWW 审计", Group: "admin"},
{Method: "POST", Path: "/api/v1/admin/sync/conflicts/{id}/resolve", OperationID: "resolveSyncConflict", Summary: "已废弃:公司侧 403", Group: "admin"},
{Method: "GET", Path: "/api/v1/agent/sync/channels/{id}/whitelist", OperationID: "agentSyncWhitelist", Summary: "本机 agent 拉取表白名单", Group: "agent"},
{Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push", OperationID: "agentSyncPush", Summary: "本机 agent 推变更到线上 A", Group: "agent"},
{Method: "POST", Path: "/api/v1/agent/sync/channels/{id}/push/batch", OperationID: "agentSyncPushBatch", Summary: "本机 agent 批量推送", Group: "agent"},
{Method: "GET", Path: "/api/v1/platform/dbsync/lww-overrides", OperationID: "platformLwwOverrides", Summary: "超管查看 LWW 覆盖审计", Group: "platform"},
{Method: "POST", Path: "/api/v1/platform/dbsync/lww-overrides/{id}/rollback", OperationID: "platformLwwRollback", Summary: "超管按落败快照回滚线上单行", Group: "platform"},
{Method: "GET", Path: "/api/v1/apps", OperationID: "listApps", Summary: "列出模块:管理账号看本租户全部(含在建);智能体仅已授权", Group: "app"},
{Method: "PUT", Path: "/api/v1/apps/{slug}/draft", OperationID: "saveDraft", Summary: "登记在建模块蓝图(不发布)", Group: "app"},

View File

@@ -39,6 +39,7 @@ var permAlias = map[string]string{
"tenant.invite": Perm邀请成员,
"org.admin": Perm管理组织,
"sync.admin": Perm数据同步,
"sync.push": Perm数据同步,
"tenant.admin": Perm管理租户,
Perm读取模块: Perm读取模块,
Perm写入模块: Perm写入模块,

View File

@@ -68,6 +68,8 @@ type StorageConf struct {
}
type DBSyncConf struct {
Enabled bool `json:",default=true"`
DataDir string `json:",default=./data/dbsync"` // 通道/冲突队列 JSON
Enabled bool `json:",default=true"`
DataDir string `json:",default=./data/dbsync"` // 通道/冲突/LWW 审计 JSON
LwwAuditTTLDays int `json:",default=90"` // 超管 LWW 覆盖日志保留天数
ReconcileMinSec int `json:",default=300"` // 手动对账最小间隔(秒)
}

View File

@@ -0,0 +1,143 @@
package dbsync
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
)
// Binding 本机库 ↔ 线上库映射P1不挡推送供登记/查询)。
type Binding struct {
ID string `json:"id"`
TenantID int64 `json:"tenant_id"`
UserID int64 `json:"user_id,omitempty"`
LocalDatabaseID string `json:"local_database_id"`
OnlineDBID string `json:"online_db_id"`
ChannelID string `json:"channel_id,omitempty"`
Note string `json:"note,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func (s *FileStore) bindingPath() string {
return filepath.Join(s.dir, "bindings.json")
}
func (s *FileStore) EnsureBinding(b Binding) (Binding, error) {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readBindingsUnlocked()
if err != nil {
return b, err
}
b.LocalDatabaseID = strings.TrimSpace(b.LocalDatabaseID)
b.OnlineDBID = strings.TrimSpace(b.OnlineDBID)
if b.TenantID <= 0 {
return b, fmt.Errorf("tenant_id required")
}
if b.LocalDatabaseID == "" || b.OnlineDBID == "" {
return b, fmt.Errorf("local_database_id and online_db_id required")
}
now := time.Now().UTC()
for i := range list {
if list[i].TenantID == b.TenantID && list[i].LocalDatabaseID == b.LocalDatabaseID {
if b.UserID > 0 && list[i].UserID > 0 && list[i].UserID != b.UserID {
continue
}
list[i].OnlineDBID = b.OnlineDBID
if b.ChannelID != "" {
list[i].ChannelID = b.ChannelID
}
if b.Note != "" {
list[i].Note = b.Note
}
if b.UserID > 0 {
list[i].UserID = b.UserID
}
list[i].UpdatedAt = now
if err := s.writeBindingsUnlocked(list); err != nil {
return b, err
}
return list[i], nil
}
}
if b.ID == "" {
b.ID = uuid.NewString()
}
b.CreatedAt = now
b.UpdatedAt = now
list = append(list, b)
if err := s.writeBindingsUnlocked(list); err != nil {
return b, err
}
return b, nil
}
func (s *FileStore) ListBindings(tenantID int64, localDatabaseID string) ([]Binding, error) {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readBindingsUnlocked()
if err != nil {
return nil, err
}
out := make([]Binding, 0)
for _, b := range list {
if b.TenantID != tenantID {
continue
}
if localDatabaseID != "" && b.LocalDatabaseID != localDatabaseID {
continue
}
out = append(out, b)
}
return out, nil
}
func (s *FileStore) GetBinding(tenantID int64, localDatabaseID string) (*Binding, error) {
list, err := s.ListBindings(tenantID, localDatabaseID)
if err != nil {
return nil, err
}
if len(list) == 0 {
return nil, fmt.Errorf("binding not found")
}
cp := list[0]
return &cp, nil
}
func (s *FileStore) readBindingsUnlocked() ([]Binding, error) {
path := s.bindingPath()
b, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return []Binding{}, nil
}
return nil, err
}
if len(b) == 0 {
return []Binding{}, nil
}
var list []Binding
if err := json.Unmarshal(b, &list); err != nil {
return nil, err
}
return list, nil
}
func (s *FileStore) writeBindingsUnlocked(list []Binding) error {
path := s.bindingPath()
raw, err := json.MarshalIndent(list, "", " ")
if err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}

View File

@@ -0,0 +1,35 @@
package dbsync
import (
"path/filepath"
"testing"
)
func TestEnsureBindingUpsert(t *testing.T) {
dir := t.TempDir()
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
if err != nil {
t.Fatal(err)
}
b, err := st.EnsureBinding(Binding{
TenantID: 1,
LocalDatabaseID: "local-a",
OnlineDBID: "online-1",
ChannelID: "ch1",
})
if err != nil || b.ID == "" {
t.Fatalf("ensure: %+v err=%v", b, err)
}
b2, err := st.EnsureBinding(Binding{
TenantID: 1,
LocalDatabaseID: "local-a",
OnlineDBID: "online-2",
})
if err != nil || b2.OnlineDBID != "online-2" || b2.ID != b.ID {
t.Fatalf("upsert: %+v err=%v", b2, err)
}
list, err := st.ListBindings(1, "local-a")
if err != nil || len(list) != 1 {
t.Fatalf("list=%d err=%v", len(list), err)
}
}

View File

@@ -0,0 +1,41 @@
package dbsync
import (
"context"
"database/sql"
"time"
)
const (
EntryDrain = "drain"
EntryAgentPush = "agent_push"
EntryRollback = "rollback"
OutcomeApplied = "applied_source"
OutcomeKept = "kept_target"
OutcomeRolled = "rolled_back"
)
// RecordLwwOverride 在 LWW 覆盖或保留目标时写入超管审计(失败忽略,不挡同步)。
func RecordLwwOverride(store *FileStore, o LwwOverride) {
if store == nil {
return
}
_ = store.AddLwwOverride(o)
}
// SnapshotTargetRow 取目标端当前行 JSON无行则空串
func SnapshotTargetRow(ctx context.Context, db *sql.DB, driver Driver, table, pkCol, rowPK string) string {
if db == nil || rowPK == "" {
return ""
}
js, _, err := FetchRowJSON(ctx, db, driver, table, pkCol, rowPK)
if err != nil {
return ""
}
return js
}
// DefaultLwwAuditTTL 默认 90 天。
func DefaultLwwAuditTTL() time.Duration {
return 90 * 24 * time.Hour
}

View File

@@ -0,0 +1,72 @@
package dbsync
import (
"path/filepath"
"testing"
"time"
)
func TestLwwOverrideStoreAndPurge(t *testing.T) {
dir := t.TempDir()
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
if err != nil {
t.Fatal(err)
}
old := LwwOverride{
TenantID: 1,
ChannelID: "ch1",
Table: "orders",
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
Entry: EntryAgentPush,
Policy: string(PolicyLWWSource),
Outcome: OutcomeApplied,
CreatedAt: time.Now().UTC().Add(-100 * 24 * time.Hour),
}
fresh := LwwOverride{
TenantID: 1,
ChannelID: "ch1",
Table: "orders",
RowPK: "bbbbbbbb-bbbb-cccc-dddd-eeeeeeeeeeee",
Entry: EntryDrain,
Policy: string(PolicyLWWSource),
Outcome: OutcomeApplied,
CreatedAt: time.Now().UTC(),
}
if err := st.AddLwwOverride(old); err != nil {
t.Fatal(err)
}
if err := st.AddLwwOverride(fresh); err != nil {
t.Fatal(err)
}
list, err := st.ListLwwOverrides(1, "ch1", 10)
if err != nil || len(list) != 2 {
t.Fatalf("list=%d err=%v", len(list), err)
}
n, err := st.PurgeLwwOverridesBefore(time.Now().UTC().Add(-90 * 24 * time.Hour))
if err != nil || n != 1 {
t.Fatalf("purge n=%d err=%v", n, err)
}
list, err = st.ListLwwOverrides(0, "", 10)
if err != nil || len(list) != 1 {
t.Fatalf("after purge list=%d err=%v", len(list), err)
}
}
func TestCanReconcile(t *testing.T) {
ok, _ := CanReconcile(nil, time.Minute)
if !ok {
t.Fatal("nil channel should allow")
}
past := time.Now().UTC().Add(-10 * time.Minute)
ch := &Channel{LastReconcileAt: &past}
ok, _ = CanReconcile(ch, 5*time.Minute)
if !ok {
t.Fatal("expected allow")
}
recent := time.Now().UTC()
ch.LastReconcileAt = &recent
ok, wait := CanReconcile(ch, 5*time.Minute)
if ok || wait <= 0 {
t.Fatalf("expected deny wait>0 ok=%v wait=%v", ok, wait)
}
}

View File

@@ -93,10 +93,24 @@ func (m *Manager) loop(ctx context.Context, id string) {
log.Printf("dbsync channel %s: %v", id, err)
}
ticks++
// 约每分钟主键对账一次,补漏(漏投递 / 触发器未装时的存量差
if ticks%120 == 0 && ch.Direction == DirBidirectional {
if _, rerr := ReconcileChannel(ctx, ch); rerr != nil {
log.Printf("dbsync reconcile %s: %v", id, rerr)
// 双向通道自动对账:默认约每 15 分钟一次(限流
autoEvery := 1800 // poll 500ms → ~15min
if ch.PollIntervalMS > 0 {
autoEvery = int((15 * time.Minute) / (time.Duration(ch.PollIntervalMS) * time.Millisecond))
if autoEvery < 60 {
autoEvery = 60
}
}
if ticks%autoEvery == 0 && ch.Direction == DirBidirectional {
if allow, _ := CanReconcile(ch, 15*time.Minute); allow {
if _, rerr := ReconcileChannel(ctx, ch); rerr != nil {
log.Printf("dbsync reconcile %s: %v", id, rerr)
} else {
_ = m.store.PatchStats(id, func(c *Channel) {
now := time.Now().UTC()
c.LastReconcileAt = &now
})
}
}
}
select {
@@ -214,12 +228,63 @@ func (m *Manager) drain(ctx context.Context, ch *Channel, sourceName string, src
continue
}
if has && tgtVer > r.Version {
switch ch.ConflictPolicy {
policy := ch.ConflictPolicy
if policy == "" {
policy = PolicyLWWSource
}
switch policy {
case PolicyLWWTarget:
loser := payload
winner := SnapshotTargetRow(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.RowPK)
RecordLwwOverride(m.store, LwwOverride{
TenantID: ch.TenantID,
ChannelID: ch.ID,
Table: r.TableName,
RowPK: r.RowPK,
Op: r.Op,
Entry: EntryDrain,
Policy: string(PolicyLWWTarget),
Outcome: OutcomeKept,
LoserPayload: loser,
WinnerPayload: winner,
TargetVer: tgtVer,
SourceVer: r.Version,
})
done = append(done, r.ID)
continue
case PolicyLWWSource:
// fallthrough apply
loser := SnapshotTargetRow(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.RowPK)
if err := ApplyChange(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.Op, payload, r.Version); err != nil {
_ = m.store.PatchStats(ch.ID, func(c *Channel) {
c.Stats.Retries++
c.LastError = err.Error()
})
continue
}
RecordLwwOverride(m.store, LwwOverride{
TenantID: ch.TenantID,
ChannelID: ch.ID,
Table: r.TableName,
RowPK: r.RowPK,
Op: r.Op,
Entry: EntryDrain,
Policy: string(PolicyLWWSource),
Outcome: OutcomeApplied,
LoserPayload: loser,
WinnerPayload: payload,
TargetVer: tgtVer,
SourceVer: r.Version,
})
done = append(done, r.ID)
okCount++
_ = m.store.PatchStats(ch.ID, func(c *Channel) {
if sourceName == "local" {
c.Stats.PushedOK++
} else {
c.Stats.PulledOK++
}
})
continue
default:
_ = m.store.AddConflict(Conflict{
TenantID: ch.TenantID,
@@ -264,6 +329,9 @@ func (m *Manager) drain(ctx context.Context, ch *Channel, sourceName string, src
// PrepareChannel 连接两端、建 outbox/触发器,供「测试/启用」调用。
func PrepareChannel(ctx context.Context, ch *Channel) error {
if err := ValidateChannelAgainstDB(ctx, ch); err != nil {
return err
}
for _, ep := range []Endpoint{ch.Local, ch.Remote} {
db, err := Open(ep.Driver, ep.DSN)
if err != nil {

View File

@@ -0,0 +1,256 @@
package dbsync
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
)
// PushItem 本机 agent → 线上 A 的一条变更(形态 B
type PushItem struct {
Table string `json:"table"`
Op string `json:"op"` // insert|update|update_by_id|delete|upsert
RowPK string `json:"row_pk"`
Row map[string]any `json:"row"` // 单行;与 Rows 二选一
Rows []map[string]any `json:"rows"` // 兼容批量 insert
Version int64 `json:"version"` // outbox 单调 version0 则用时间戳
ClientOutboxID string `json:"client_outbox_id"`
OnlineDBID string `json:"online_db_id"`
}
// PushResult 单条推送结果。
type PushResult struct {
OK bool `json:"ok"`
Applied bool `json:"applied"`
Skipped bool `json:"skipped"`
Conflict bool `json:"conflict"`
AppliedVersion int64 `json:"applied_version"`
Message string `json:"message"`
ClientOutboxID string `json:"client_outbox_id,omitempty"`
}
// PushToRemote 仅打开 remote按 LWW/幂等将变更落到线上 A。不连本机 SQLite。
func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushItem) (*PushResult, error) {
if ch == nil {
return nil, fmt.Errorf("channel is nil")
}
table := strings.TrimSpace(item.Table)
if table == "" {
return nil, fmt.Errorf("table required")
}
if !tableInChannel(ch, table) {
return nil, fmt.Errorf("table %s 不在通道白名单", table)
}
pkCol := "id"
if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" {
pkCol = ch.PKColumns[table]
}
op, payload, rowPK, err := normalizePushPayload(item, pkCol)
if err != nil {
return nil, err
}
version := item.Version
if version <= 0 {
version = time.Now().UnixNano()
}
db, err := Open(ch.Remote.Driver, ch.Remote.DSN)
if err != nil {
return nil, fmt.Errorf("open remote: %w", err)
}
defer db.Close()
if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil {
return nil, fmt.Errorf("ensure meta: %w", err)
}
res := &PushResult{
ClientOutboxID: item.ClientOutboxID,
AppliedVersion: version,
}
tgtVer, has, err := GetMetaVersion(ctx, db, ch.Remote.Driver, table, rowPK)
if err != nil {
return nil, err
}
// 幂等:同 version 已落地 → 跳过
if has && tgtVer == version {
res.OK = true
res.Skipped = true
res.Message = "already applied (same version)"
return res, nil
}
if has && tgtVer > version {
policy := ch.ConflictPolicy
if policy == "" {
policy = PolicyLWWSource // B→A 默认偏源
}
switch policy {
case PolicyLWWTarget:
loser := payload
winner := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, rowPK)
RecordLwwOverride(store, LwwOverride{
TenantID: ch.TenantID,
ChannelID: ch.ID,
Table: table,
RowPK: rowPK,
Op: op,
Entry: EntryAgentPush,
Policy: string(PolicyLWWTarget),
Outcome: OutcomeKept,
LoserPayload: loser,
WinnerPayload: winner,
TargetVer: tgtVer,
SourceVer: version,
})
res.OK = true
res.Skipped = true
res.Message = "target newer; kept (lww_target)"
return res, nil
case PolicyLWWSource:
loser := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, rowPK)
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil {
return nil, err
}
RecordLwwOverride(store, LwwOverride{
TenantID: ch.TenantID,
ChannelID: ch.ID,
Table: table,
RowPK: rowPK,
Op: op,
Entry: EntryAgentPush,
Policy: string(PolicyLWWSource),
Outcome: OutcomeApplied,
LoserPayload: loser,
WinnerPayload: payload,
TargetVer: tgtVer,
SourceVer: version,
})
if store != nil {
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ })
}
res.OK = true
res.Applied = true
res.Message = "applied (lww_source override)"
return res, nil
default:
if store != nil {
_ = store.AddConflict(Conflict{
TenantID: ch.TenantID,
ChannelID: ch.ID,
Table: table,
RowPK: rowPK,
Op: op,
Source: "agent",
Payload: payload,
TargetVer: tgtVer,
SourceVer: version,
Message: "target version newer than agent push",
})
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.Conflicts++ })
}
res.OK = true
res.Skipped = true
res.Conflict = true
res.Message = "queued conflict; target newer"
return res, nil
}
}
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil {
return nil, err
}
if store != nil {
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ })
}
res.OK = true
res.Applied = true
res.Message = "applied"
return res, nil
}
// PushBatchToRemote 保序批量;遇错即停,已成功条数在返回切片中。
func PushBatchToRemote(ctx context.Context, ch *Channel, store *FileStore, items []PushItem) ([]PushResult, error) {
out := make([]PushResult, 0, len(items))
for i, it := range items {
r, err := PushToRemote(ctx, ch, store, it)
if err != nil {
fail := PushResult{
OK: false,
Message: err.Error(),
ClientOutboxID: it.ClientOutboxID,
}
out = append(out, fail)
return out, fmt.Errorf("item[%d]: %w", i, err)
}
out = append(out, *r)
}
return out, nil
}
func tableInChannel(ch *Channel, table string) bool {
for _, t := range uniqueTables(ch.Local.Tables, ch.Remote.Tables) {
if strings.EqualFold(strings.TrimSpace(t), table) {
return true
}
}
return false
}
func normalizePushPayload(item PushItem, pkCol string) (op string, payload string, rowPK string, err error) {
rawOp := strings.ToLower(strings.TrimSpace(item.Op))
if rawOp == "" {
rawOp = "upsert"
}
switch rawOp {
case "delete":
op = "delete"
case "insert", "update", "update_by_id", "upsert":
op = "upsert"
default:
return "", "", "", fmt.Errorf("unsupported op: %s", item.Op)
}
row := item.Row
if row == nil && len(item.Rows) > 0 {
row = item.Rows[0]
}
rowPK = strings.TrimSpace(item.RowPK)
if row != nil {
if rowPK == "" {
rowPK = strings.TrimSpace(fmt.Sprint(row[pkCol]))
}
if op == "upsert" {
// 保证 payload 含主键
if _, ok := row[pkCol]; !ok && rowPK != "" {
row = copyMap(row)
row[pkCol] = rowPK
}
}
}
if rowPK == "" || rowPK == "<nil>" {
return "", "", "", fmt.Errorf("row_pk required")
}
if op == "delete" && row == nil {
row = map[string]any{pkCol: rowPK}
}
if row == nil {
return "", "", "", fmt.Errorf("row required for %s", op)
}
b, err := json.Marshal(row)
if err != nil {
return "", "", "", err
}
return op, string(b), rowPK, nil
}
func copyMap(m map[string]any) map[string]any {
out := make(map[string]any, len(m)+1)
for k, v := range m {
out[k] = v
}
return out
}

View File

@@ -0,0 +1,46 @@
package dbsync
import "testing"
func TestNormalizePushPayloadInsert(t *testing.T) {
op, payload, pk, err := normalizePushPayload(PushItem{
Op: "insert",
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
Row: map[string]any{
"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"name": "x",
},
}, "id")
if err != nil {
t.Fatal(err)
}
if op != "upsert" || pk == "" || payload == "" {
t.Fatalf("got op=%s pk=%s payload=%s", op, pk, payload)
}
}
func TestNormalizePushPayloadDelete(t *testing.T) {
op, payload, pk, err := normalizePushPayload(PushItem{
Op: "delete",
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
}, "id")
if err != nil {
t.Fatal(err)
}
if op != "delete" || pk == "" || payload == "" {
t.Fatalf("got op=%s pk=%s payload=%s", op, pk, payload)
}
}
func TestTableInChannel(t *testing.T) {
ch := &Channel{
Local: Endpoint{Tables: []string{"orders"}},
Remote: Endpoint{Tables: []string{"orders"}},
}
if !tableInChannel(ch, "orders") {
t.Fatal("expected in")
}
if tableInChannel(ch, "other") {
t.Fatal("expected out")
}
}

View File

@@ -0,0 +1,30 @@
package dbsync
import (
"fmt"
"time"
)
// DefaultManualReconcileInterval 公司管理员手动对账最小间隔。
const DefaultManualReconcileInterval = 5 * time.Minute
// CanReconcile 限流:距上次对账不足 minInterval 则拒绝。
func CanReconcile(ch *Channel, minInterval time.Duration) (bool, time.Duration) {
if ch == nil || ch.LastReconcileAt == nil || ch.LastReconcileAt.IsZero() {
return true, 0
}
if minInterval <= 0 {
minInterval = DefaultManualReconcileInterval
}
elapsed := time.Since(ch.LastReconcileAt.UTC())
if elapsed >= minInterval {
return true, 0
}
return false, minInterval - elapsed
}
// ReconcileTooSoonError 供 handler 返回 429。
func ReconcileTooSoonError(wait time.Duration) error {
sec := int(wait.Seconds()) + 1
return fmt.Errorf("对账过于频繁,请 %d 秒后再试", sec)
}

View File

@@ -0,0 +1,79 @@
package dbsync
import (
"context"
"fmt"
"strings"
"time"
"github.com/google/uuid"
)
// RollbackLwwOverride 超管按落败快照回滚线上 A 单行。
// 仅 outcome=applied_source 可回滚:把 loser_payload 写回 remote无快照则按 pk 删除。
func RollbackLwwOverride(ctx context.Context, store *FileStore, overrideID string) (*LwwOverride, error) {
if store == nil {
return nil, fmt.Errorf("store is nil")
}
o, err := store.GetLwwOverride(overrideID)
if err != nil {
return nil, err
}
if o.Outcome != OutcomeApplied {
return nil, fmt.Errorf("仅「源端覆盖」记录可回滚(当前 outcome=%s", o.Outcome)
}
if strings.TrimSpace(o.ChannelID) == "" {
return nil, fmt.Errorf("override missing channel_id")
}
ch, err := store.GetChannel(o.ChannelID)
if err != nil {
return nil, fmt.Errorf("channel: %w", err)
}
table := strings.TrimSpace(o.Table)
pkCol := "id"
if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" {
pkCol = ch.PKColumns[table]
}
db, err := Open(ch.Remote.Driver, ch.Remote.DSN)
if err != nil {
return nil, fmt.Errorf("open remote: %w", err)
}
defer db.Close()
if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil {
return nil, err
}
before := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, o.RowPK)
ver := time.Now().UnixNano()
loser := strings.TrimSpace(o.LoserPayload)
op := "upsert"
payload := loser
if loser == "" || loser == "{}" {
op = "delete"
payload = fmt.Sprintf(`{%q:%q}`, pkCol, o.RowPK)
}
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, ver); err != nil {
return nil, fmt.Errorf("apply rollback: %w", err)
}
rec := LwwOverride{
ID: uuid.NewString(),
TenantID: o.TenantID,
ChannelID: o.ChannelID,
Table: o.Table,
RowPK: o.RowPK,
Op: op,
Entry: EntryRollback,
Policy: "manual_rollback",
Outcome: OutcomeRolled,
LoserPayload: before,
WinnerPayload: payload,
TargetVer: o.SourceVer,
SourceVer: ver,
CreatedAt: time.Now().UTC(),
}
if err := store.AddLwwOverride(rec); err != nil {
return nil, err
}
return &rec, nil
}

View File

@@ -0,0 +1,54 @@
package dbsync
import (
"context"
"path/filepath"
"testing"
"time"
)
func TestRollbackRejectsNonApplied(t *testing.T) {
dir := t.TempDir()
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
if err != nil {
t.Fatal(err)
}
o := LwwOverride{
ID: "ov1",
TenantID: 1,
ChannelID: "ch1",
Table: "orders",
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
Outcome: OutcomeKept,
CreatedAt: time.Now().UTC(),
}
if err := st.AddLwwOverride(o); err != nil {
t.Fatal(err)
}
_, err = RollbackLwwOverride(context.Background(), st, "ov1")
if err == nil {
t.Fatal("expected reject kept_target")
}
}
func TestGetLwwOverride(t *testing.T) {
dir := t.TempDir()
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
if err != nil {
t.Fatal(err)
}
o := LwwOverride{
ID: "ov2",
TenantID: 1,
Table: "t",
RowPK: "pk",
Outcome: OutcomeApplied,
}
if err := st.AddLwwOverride(o); err != nil {
t.Fatal(err)
}
got, err := st.GetLwwOverride("ov2")
if err != nil || got.Table != "t" {
t.Fatalf("got=%+v err=%v", got, err)
}
}

View File

@@ -11,12 +11,13 @@ import (
"github.com/google/uuid"
)
// FileStore 持久化通道冲突队列JSON不依赖业务库类型。
// FileStore 持久化通道冲突队列与超管 LWW 审计JSON不依赖业务库类型。
type FileStore struct {
mu sync.Mutex
dir string
chPath string
cfPath string
mu sync.Mutex
dir string
chPath string
cfPath string
lwwPath string
}
func NewFileStore(dir string) (*FileStore, error) {
@@ -27,9 +28,10 @@ func NewFileStore(dir string) (*FileStore, error) {
return nil, err
}
return &FileStore{
dir: dir,
chPath: filepath.Join(dir, "channels.json"),
cfPath: filepath.Join(dir, "conflicts.json"),
dir: dir,
chPath: filepath.Join(dir, "channels.json"),
cfPath: filepath.Join(dir, "conflicts.json"),
lwwPath: filepath.Join(dir, "lww_overrides.json"),
}, nil
}
@@ -103,7 +105,7 @@ func (s *FileStore) SaveChannel(ch Channel) (Channel, error) {
ch.Direction = DirLocalToRemote
}
if ch.ConflictPolicy == "" {
ch.ConflictPolicy = PolicyQueue
ch.ConflictPolicy = PolicyLWWSource
}
if ch.PKColumns == nil {
ch.PKColumns = map[string]string{}
@@ -120,6 +122,7 @@ func (s *FileStore) SaveChannel(ch Channel) (Channel, error) {
}
ch.CreatedAt = list[i].CreatedAt
ch.Stats = list[i].Stats
ch.LastReconcileAt = list[i].LastReconcileAt
list[i] = ch
found = true
break
@@ -325,3 +328,117 @@ func (s *FileStore) writeConflicts(list []Conflict) error {
}
return os.Rename(tmp, s.cfPath)
}
func (s *FileStore) AddLwwOverride(o LwwOverride) error {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readLwwOverrides()
if err != nil {
return err
}
if o.ID == "" {
o.ID = uuid.NewString()
}
if o.CreatedAt.IsZero() {
o.CreatedAt = time.Now().UTC()
}
list = append(list, o)
return s.writeLwwOverrides(list)
}
func (s *FileStore) GetLwwOverride(id string) (*LwwOverride, error) {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readLwwOverrides()
if err != nil {
return nil, err
}
for i := range list {
if list[i].ID == id {
cp := list[i]
return &cp, nil
}
}
return nil, fmt.Errorf("lww override not found")
}
// ListLwwOverrides 超管查询tenantID/channelID 为 0/空 表示不过滤。
func (s *FileStore) ListLwwOverrides(tenantID int64, channelID string, limit int) ([]LwwOverride, error) {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readLwwOverrides()
if err != nil {
return nil, err
}
out := make([]LwwOverride, 0, len(list))
for i := len(list) - 1; i >= 0; i-- { // 新→旧
o := list[i]
if tenantID > 0 && o.TenantID != tenantID {
continue
}
if channelID != "" && o.ChannelID != channelID {
continue
}
out = append(out, o)
if limit > 0 && len(out) >= limit {
break
}
}
return out, nil
}
// PurgeLwwOverridesBefore 删除 created_at 早于 cutoff 的记录,返回删除条数。
func (s *FileStore) PurgeLwwOverridesBefore(cutoff time.Time) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
list, err := s.readLwwOverrides()
if err != nil {
return 0, err
}
keep := make([]LwwOverride, 0, len(list))
removed := 0
for _, o := range list {
if o.CreatedAt.Before(cutoff) {
removed++
continue
}
keep = append(keep, o)
}
if removed == 0 {
return 0, nil
}
if err := s.writeLwwOverrides(keep); err != nil {
return 0, err
}
return removed, nil
}
func (s *FileStore) readLwwOverrides() ([]LwwOverride, error) {
b, err := os.ReadFile(s.lwwPath)
if err != nil {
if os.IsNotExist(err) {
return []LwwOverride{}, nil
}
return nil, err
}
if len(b) == 0 {
return []LwwOverride{}, nil
}
var list []LwwOverride
if err := json.Unmarshal(b, &list); err != nil {
return nil, err
}
return list, nil
}
func (s *FileStore) writeLwwOverrides(list []LwwOverride) error {
b, err := json.MarshalIndent(list, "", " ")
if err != nil {
return err
}
tmp := s.lwwPath + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, s.lwwPath)
}

View File

@@ -52,6 +52,7 @@ type Channel struct {
UpdatedAt time.Time `json:"updated_at"`
LastError string `json:"last_error,omitempty"`
LastSyncAt *time.Time `json:"last_sync_at,omitempty"`
LastReconcileAt *time.Time `json:"last_reconcile_at,omitempty"`
Stats ChannelStats `json:"stats"`
}
@@ -80,6 +81,24 @@ type Conflict struct {
Resolution string `json:"resolution,omitempty"` // apply_source | keep_target | discard
}
// LwwOverride LWW 自动覆盖审计(仅平台超级管理员可见;与租户冲突队列分离)。
type LwwOverride struct {
ID string `json:"id"`
TenantID int64 `json:"tenant_id"`
ChannelID string `json:"channel_id"`
Table string `json:"table"`
RowPK string `json:"row_pk"`
Op string `json:"op"`
Entry string `json:"entry"` // drain | agent_push
Policy string `json:"policy"` // lww_source | lww_target
Outcome string `json:"outcome"` // applied_source | kept_target
LoserPayload string `json:"loser_payload"` // 落败侧
WinnerPayload string `json:"winner_payload"` // 胜出侧
TargetVer int64 `json:"target_ver"`
SourceVer int64 `json:"source_ver"`
CreatedAt time.Time `json:"created_at"`
}
type OutboxRow struct {
ID int64
TableName string
@@ -91,8 +110,8 @@ type OutboxRow struct {
}
type TestResult struct {
OK bool `json:"ok"`
Driver string `json:"driver"`
Message string `json:"message"`
OK bool `json:"ok"`
Driver string `json:"driver"`
Message string `json:"message"`
Tables []string `json:"tables,omitempty"`
}

View File

@@ -0,0 +1,264 @@
package dbsync
import (
"context"
"database/sql"
"fmt"
"strings"
)
// ValidateChannelConfig 静态校验不连库表名单、PK 列名约定。
func ValidateChannelConfig(ch *Channel) error {
if ch == nil {
return fmt.Errorf("channel is nil")
}
tables := uniqueTables(ch.Local.Tables, ch.Remote.Tables)
if len(tables) == 0 {
return fmt.Errorf("同步表白名单为空:请至少在 local 或 remote 填写表名")
}
for _, t := range tables {
t = strings.TrimSpace(t)
if t == "" {
return fmt.Errorf("表名不能为空")
}
if strings.HasPrefix(t, "_ajz_") {
return fmt.Errorf("禁止同步系统表: %s", t)
}
pk := pkColumn(ch, t)
if pk == "" {
return fmt.Errorf("表 %s 主键列名为空", t)
}
}
return nil
}
func pkColumn(ch *Channel, table string) string {
if ch.PKColumns != nil {
if v := strings.TrimSpace(ch.PKColumns[table]); v != "" {
return v
}
}
return "id"
}
// ValidateChannelAgainstDB 对可连接端做TEXT/UUID 主键类型 + FK 闭包。
// 某端连不上时跳过该端(形态 B 下 local 常不可达),但至少一端须校验成功,否则拒绝。
func ValidateChannelAgainstDB(ctx context.Context, ch *Channel) error {
if err := ValidateChannelConfig(ch); err != nil {
return err
}
tables := uniqueTables(ch.Local.Tables, ch.Remote.Tables)
checked := 0
var lastSkip error
for _, ep := range []Endpoint{ch.Remote, ch.Local} {
if strings.TrimSpace(ep.DSN) == "" || ep.Driver == "" {
continue
}
epTables := ep.Tables
if len(epTables) == 0 {
epTables = tables
}
db, err := Open(ep.Driver, ep.DSN)
if err != nil {
lastSkip = fmt.Errorf("%s 无法连接(跳过库内校验): %w", ep.Driver, err)
continue
}
if err := validateEndpointSchema(ctx, db, ep.Driver, ch, epTables, tables); err != nil {
_ = db.Close()
return fmt.Errorf("[%s] %w", ep.Driver, err)
}
_ = db.Close()
checked++
}
if checked == 0 {
if lastSkip != nil {
return fmt.Errorf("无法对任何端做主键/外键校验:%v请保证线上库remoteDSN 可达后再保存", lastSkip)
}
return fmt.Errorf("无法对任何端做主键/外键校验:请配置可达的 remote DSN")
}
return nil
}
func validateEndpointSchema(ctx context.Context, db *sql.DB, driver Driver, ch *Channel, epTables, whitelist []string) error {
wl := map[string]struct{}{}
for _, t := range whitelist {
wl[strings.TrimSpace(t)] = struct{}{}
}
for _, t := range epTables {
t = strings.TrimSpace(t)
if t == "" {
continue
}
pk := pkColumn(ch, t)
typ, err := DescribeColumnType(ctx, db, driver, t, pk)
if err != nil {
return fmt.Errorf("表 %s 主键列 %s: %w", t, pk, err)
}
if !isTextLikePK(typ) {
return fmt.Errorf("表 %s 主键 %s 类型为 %q同步表须为 TEXT/VARCHAR/UUID 类(禁止自增整数作同步键)", t, pk, typ)
}
fks, err := ListForeignKeys(ctx, db, driver, t)
if err != nil {
return fmt.Errorf("表 %s 外键: %w", t, err)
}
for _, fk := range fks {
_, childIn := wl[fk.ChildTable]
_, parentIn := wl[fk.ParentTable]
if childIn != parentIn {
missing := fk.ParentTable
if !childIn {
missing = fk.ChildTable
}
return fmt.Errorf("外键闭包不完整:%s.%s → %s.%s请将 %s 一并加入同步白名单",
fk.ChildTable, fk.ChildColumn, fk.ParentTable, fk.ParentColumn, missing)
}
}
}
return nil
}
func isTextLikePK(typ string) bool {
raw := strings.ToUpper(strings.TrimSpace(typ))
if raw == "" {
return false
}
token := strings.Fields(raw)[0]
token = strings.Split(token, "(")[0]
switch token {
case "TEXT", "VARCHAR", "CHAR", "CHARACTER", "UUID", "NVARCHAR", "NCHAR", "STRING", "CITEXT", "CLOB":
return true
case "INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", "MEDIUMINT",
"SERIAL", "BIGSERIAL", "SMALLSERIAL",
"NUMERIC", "DECIMAL", "NUMBER", "FLOAT", "DOUBLE", "REAL", "BOOLEAN", "BOOL":
return false
default:
if strings.Contains(raw, "CHAR") || strings.Contains(raw, "TEXT") || strings.Contains(raw, "UUID") || strings.Contains(raw, "CLOB") {
return true
}
return false
}
}
// ForeignKey 子表 → 父表
type ForeignKey struct {
ChildTable string
ChildColumn string
ParentTable string
ParentColumn string
}
// DescribeColumnType 返回列类型字符串(驱动相关原文)。
func DescribeColumnType(ctx context.Context, db *sql.DB, driver Driver, table, column string) (string, error) {
table = strings.TrimSpace(table)
column = strings.TrimSpace(column)
switch driver {
case DriverSQLite:
rows, err := db.QueryContext(ctx, fmt.Sprintf(`PRAGMA table_info(%s)`, quoteIdent(driver, table)))
if err != nil {
return "", err
}
defer rows.Close()
for rows.Next() {
var cid int
var name, typ string
var notnull, pk int
var dflt sql.NullString
if err := rows.Scan(&cid, &name, &typ, &notnull, &dflt, &pk); err != nil {
return "", err
}
if strings.EqualFold(name, column) {
if typ == "" {
typ = "TEXT" // sqlite 松类型兜底:无声明时按 TEXT 处理需调用方结合;此处空则拒
return "", fmt.Errorf("列 %s 无类型声明sqlite请显式声明为 TEXT", column)
}
return typ, nil
}
}
return "", fmt.Errorf("列不存在")
case DriverMySQL:
var typ string
err := db.QueryRowContext(ctx, `
SELECT DATA_TYPE FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`, table, column).Scan(&typ)
if err != nil {
return "", err
}
return typ, nil
case DriverPostgres:
var typ string
err := db.QueryRowContext(ctx, `
SELECT data_type FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`, table, column).Scan(&typ)
if err != nil {
return "", err
}
return typ, nil
default:
return "", fmt.Errorf("unsupported driver")
}
}
// ListForeignKeys 列出以 table 为子表的外键。
func ListForeignKeys(ctx context.Context, db *sql.DB, driver Driver, table string) ([]ForeignKey, error) {
table = strings.TrimSpace(table)
switch driver {
case DriverSQLite:
rows, err := db.QueryContext(ctx, fmt.Sprintf(`PRAGMA foreign_key_list(%s)`, quoteIdent(driver, table)))
if err != nil {
return nil, err
}
defer rows.Close()
var out []ForeignKey
for rows.Next() {
var id, seq int
var parent, from, to, onUpdate, onDelete, match string
if err := rows.Scan(&id, &seq, &parent, &from, &to, &onUpdate, &onDelete, &match); err != nil {
return nil, err
}
out = append(out, ForeignKey{
ChildTable: table, ChildColumn: from,
ParentTable: parent, ParentColumn: to,
})
}
return out, rows.Err()
case DriverMySQL:
rows, err := db.QueryContext(ctx, `
SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
FROM information_schema.KEY_COLUMN_USAGE
WHERE table_schema = DATABASE() AND TABLE_NAME = ?
AND REFERENCED_TABLE_NAME IS NOT NULL`, table)
if err != nil {
return nil, err
}
defer rows.Close()
return scanFKRows(rows)
case DriverPostgres:
rows, err := db.QueryContext(ctx, `
SELECT tc.table_name, kcu.column_name, ccu.table_name, ccu.column_name
FROM information_schema.table_constraints AS tc
JOIN information_schema.key_column_usage AS kcu
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
JOIN information_schema.constraint_column_usage AS ccu
ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public' AND tc.table_name = $1`, table)
if err != nil {
return nil, err
}
defer rows.Close()
return scanFKRows(rows)
default:
return nil, fmt.Errorf("unsupported driver")
}
}
func scanFKRows(rows *sql.Rows) ([]ForeignKey, error) {
var out []ForeignKey
for rows.Next() {
var ctab, ccol, ptab, pcol string
if err := rows.Scan(&ctab, &ccol, &ptab, &pcol); err != nil {
return nil, err
}
out = append(out, ForeignKey{ChildTable: ctab, ChildColumn: ccol, ParentTable: ptab, ParentColumn: pcol})
}
return out, rows.Err()
}

View File

@@ -0,0 +1,65 @@
package dbsync
import (
"context"
"path/filepath"
"testing"
)
func TestIsTextLikePK(t *testing.T) {
good := []string{"TEXT", "VARCHAR(36)", "uuid", "CHARACTER VARYING", "NVARCHAR(64)", "char(36)"}
for _, g := range good {
if !isTextLikePK(g) {
t.Fatalf("expected text-like: %s", g)
}
}
bad := []string{"INTEGER", "INT", "BIGINT", "SERIAL", "BIGSERIAL", "int(11)", "NUMERIC", "DECIMAL(10,2)"}
for _, b := range bad {
if isTextLikePK(b) {
t.Fatalf("expected reject: %s", b)
}
}
}
func TestValidateChannelConfigEmpty(t *testing.T) {
ch := &Channel{}
if err := ValidateChannelConfig(ch); err == nil {
t.Fatal("expected error for empty tables")
}
ch.Local.Tables = []string{"orders"}
if err := ValidateChannelConfig(ch); err != nil {
t.Fatal(err)
}
}
func TestValidateChannelAgainstDBRejectsIntegerPK(t *testing.T) {
dir := t.TempDir()
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "t.db")) + "?_pragma=foreign_keys(1)"
db, err := Open(DriverSQLite, dsn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, title TEXT)`); err != nil {
t.Fatal(err)
}
if _, err = db.Exec(`CREATE TABLE orders_uuid (id TEXT PRIMARY KEY, title TEXT)`); err != nil {
t.Fatal(err)
}
bad := &Channel{
Local: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders"}},
Remote: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders"}},
}
if err := ValidateChannelAgainstDB(context.Background(), bad); err == nil {
t.Fatal("expected reject INTEGER PK")
}
good := &Channel{
Local: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders_uuid"}},
Remote: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders_uuid"}},
}
if err := ValidateChannelAgainstDB(context.Background(), good); err != nil {
t.Fatal(err)
}
}

View File

@@ -0,0 +1,119 @@
package handler
import (
"encoding/json"
"net/http"
"aijianzhan/platform/internal/authx"
"aijianzhan/platform/internal/dbsync"
"aijianzhan/platform/internal/svc"
"github.com/zeromicro/go-zero/rest/httpx"
"github.com/zeromicro/go-zero/rest/pathvar"
)
// agent 同步只读白名单 + 推远程 A形态 B需 JWT 含「数据同步」权限(人类管理员或智能体均可)。
func agentSyncWhitelistHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r))
if err != nil {
authx.WriteError(w, http.StatusNotFound, err.Error())
return
}
tables := uniqueStringSlice(ch.Local.Tables, ch.Remote.Tables)
httpx.OkJson(w, map[string]any{
"channel_id": ch.ID,
"name": ch.Name,
"enabled": ch.Enabled,
"direction": ch.Direction,
"conflict_policy": ch.ConflictPolicy,
"tables": tables,
"pk_columns": ch.PKColumns,
"hint": "本机 agent 缓存此表白名单;仅白名单表走 local_dbsync",
})
}
}
func agentSyncPushHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r))
if err != nil {
authx.WriteError(w, http.StatusNotFound, err.Error())
return
}
var item dbsync.PushItem
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
res, err := dbsync.PushToRemote(r.Context(), ch, svcCtx.DBSync.Store(), item)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{"success": true, "result": res})
}
}
func agentSyncPushBatchHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
ch, err := svcCtx.DBSync.Store().GetChannelForTenant(pathvar.Vars(r)["id"], syncTenantID(r))
if err != nil {
authx.WriteError(w, http.StatusNotFound, err.Error())
return
}
var body struct {
Items []dbsync.PushItem `json:"items"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
if len(body.Items) == 0 {
authx.WriteError(w, http.StatusBadRequest, "items required")
return
}
if len(body.Items) > 100 {
authx.WriteError(w, http.StatusBadRequest, "items limit 100")
return
}
results, err := dbsync.PushBatchToRemote(r.Context(), ch, svcCtx.DBSync.Store(), body.Items)
if err != nil {
httpx.OkJson(w, map[string]any{
"success": false,
"error": err.Error(),
"results": results,
})
return
}
httpx.OkJson(w, map[string]any{"success": true, "results": results})
}
}
func uniqueStringSlice(a, b []string) []string {
seen := map[string]struct{}{}
var out []string
for _, xs := range [][]string{a, b} {
for _, s := range xs {
if s == "" {
continue
}
if _, ok := seen[s]; ok {
continue
}
seen[s] = struct{}{}
out = append(out, s)
}
}
return out
}

View File

@@ -281,6 +281,141 @@ paths:
responses:
"200": { description: OK }
/api/v1/admin/sync/channels:
get:
operationId: listSyncChannels
summary: 列出同步通道
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK } }
post:
operationId: createSyncChannel
summary: 创建同步通道UUID PK + FK 闭包校验)
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK }, "400": { description: Bad Request } }
/api/v1/admin/sync/channels/{id}:
get:
operationId: getSyncChannel
summary: 获取同步通道
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
put:
operationId: updateSyncChannel
summary: 更新同步通道
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
delete:
operationId: deleteSyncChannel
summary: 删除同步通道
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/admin/sync/test:
post:
operationId: testSyncEndpoints
summary: 测试本地/线上库连接
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK } }
/api/v1/admin/sync/channels/{id}/prepare:
post:
operationId: prepareSyncChannel
summary: 准备同步outbox/触发器)
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/admin/sync/channels/{id}/start:
post:
operationId: startSyncChannel
summary: 启动同步
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/admin/sync/channels/{id}/stop:
post:
operationId: stopSyncChannel
summary: 停止同步
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/admin/sync/channels/{id}/reconcile:
post:
operationId: reconcileSyncChannel
summary: 同步修复(对账,有限流)
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK }, "429": { description: Too Many Requests } }
/api/v1/admin/sync/channels/{id}/ingest:
post:
operationId: ingestSyncRows
summary: 外部行写入通道 local
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/admin/sync/bindings:
get:
operationId: listSyncBindings
summary: 列出本机库↔线上库绑定
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK } }
post:
operationId: ensureSyncBinding
summary: 登记/更新绑定
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK } }
/api/v1/admin/sync/conflicts:
get:
operationId: listSyncConflicts
summary: 已废弃(公司侧 403
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "403": { description: Forbidden } }
/api/v1/agent/sync/channels/{id}/whitelist:
get:
operationId: agentSyncWhitelist
summary: 本机 agent 拉取表白名单
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/agent/sync/channels/{id}/push:
post:
operationId: agentSyncPush
summary: 本机 agent 推变更到线上 A
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/agent/sync/channels/{id}/push/batch:
post:
operationId: agentSyncPushBatch
summary: 本机 agent 批量推送
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK } }
/api/v1/platform/dbsync/lww-overrides:
get:
operationId: platformLwwOverrides
summary: 超管 LWW 覆盖审计
parameters: [{ $ref: "#/components/parameters/Authorization" }]
responses: { "200": { description: OK }, "403": { description: Forbidden } }
/api/v1/platform/dbsync/lww-overrides/{id}/rollback:
post:
operationId: platformLwwRollback
summary: 超管按落败快照回滚线上单行
parameters:
- { $ref: "#/components/parameters/Authorization" }
- { $ref: "#/components/parameters/Id" }
responses: { "200": { description: OK }, "400": { description: Bad Request } }
components:
parameters:
Authorization:

View File

@@ -0,0 +1,68 @@
package handler
import (
"net/http"
"strconv"
"aijianzhan/platform/internal/authx"
"aijianzhan/platform/internal/dbsync"
"aijianzhan/platform/internal/svc"
"github.com/zeromicro/go-zero/rest/httpx"
"github.com/zeromicro/go-zero/rest/pathvar"
)
// GET /api/v1/platform/dbsync/lww-overrides — 仅平台超级管理员。
func platformLwwOverridesHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if svcCtx.DBSync == nil {
authx.WriteError(w, http.StatusServiceUnavailable, "dbsync not enabled")
return
}
q := r.URL.Query()
var tenantID int64
if s := q.Get("tenant_id"); s != "" {
tenantID, _ = strconv.ParseInt(s, 10, 64)
}
channelID := q.Get("channel_id")
limit := 200
if s := q.Get("limit"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
limit = n
if limit > 1000 {
limit = 1000
}
}
}
list, err := svcCtx.DBSync.Store().ListLwwOverrides(tenantID, channelID, limit)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{
"items": list,
"hint": "LWW 自动覆盖审计;公司管理员不可见;默认 TTL 90 天;可对 applied_source 回滚单行",
})
}
}
// POST /api/v1/platform/dbsync/lww-overrides/:id/rollback — 按落败快照回滚线上单行。
func platformLwwRollbackHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if svcCtx.DBSync == nil {
authx.WriteError(w, http.StatusServiceUnavailable, "dbsync not enabled")
return
}
id := pathvar.Vars(r)["id"]
rec, err := dbsync.RollbackLwwOverride(r.Context(), svcCtx.DBSync.Store(), id)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{
"ok": true,
"record": rec,
"hint": "已按落败快照写回线上 A并追加一条 rollback 审计",
})
}
}

View File

@@ -80,6 +80,8 @@ func RegisterHandlers(server *rest.Server, svcCtx *svc.ServiceContext) {
{Method: http.MethodGet, Path: "/api/v1/platform/perm-modules", Handler: chain(platformPermModulesHandler(svcCtx), rl, authMW, platformAdmin)},
{Method: http.MethodGet, Path: "/api/v1/platform/tenants/:id/permissions", Handler: chain(platformGetTenantPermsHandler(svcCtx), rl, authMW, platformAdmin)},
{Method: http.MethodPut, Path: "/api/v1/platform/tenants/:id/permissions", Handler: chain(platformSetTenantPermsHandler(svcCtx), rl, authMW, platformAdmin)},
{Method: http.MethodGet, Path: "/api/v1/platform/dbsync/lww-overrides", Handler: chain(platformLwwOverridesHandler(svcCtx), rl, authMW, platformAdmin)},
{Method: http.MethodPost, Path: "/api/v1/platform/dbsync/lww-overrides/:id/rollback", Handler: chain(platformLwwRollbackHandler(svcCtx), rl, authMW, platformAdmin)},
})
// —— 鉴权:需已加入租户 ——
@@ -131,6 +133,13 @@ func RegisterHandlers(server *rest.Server, svcCtx *svc.ServiceContext) {
{Method: http.MethodPost, Path: "/api/v1/admin/sync/conflicts/:id/resolve", Handler: chain(syncResolveConflictHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/reconcile", Handler: chain(syncReconcileHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodPost, Path: "/api/v1/admin/sync/channels/:id/ingest", Handler: chain(syncIngestHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodGet, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsListHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodPost, Path: "/api/v1/admin/sync/bindings", Handler: chain(syncBindingsEnsureHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
// —— 本机 sync agent形态 B白名单拉取 + 推线上 A需「数据同步」权限 ——
{Method: http.MethodGet, Path: "/api/v1/agent/sync/channels/:id/whitelist", Handler: chain(agentSyncWhitelistHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push", Handler: chain(agentSyncPushHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
{Method: http.MethodPost, Path: "/api/v1/agent/sync/channels/:id/push/batch", Handler: chain(agentSyncPushBatchHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm数据同步))},
// 存储POST 创建对象GET 读取(无 /upload 动词路径;旧路径保留别名防断裂)
{Method: http.MethodPost, Path: "/api/v1/storage", Handler: chain(uploadHandler(svcCtx), rl, authMW, tenant, perm(authx.Perm上传文件))},

View File

@@ -3,6 +3,7 @@ package handler
import (
"encoding/json"
"net/http"
"time"
"aijianzhan/platform/internal/authx"
"aijianzhan/platform/internal/dbsync"
@@ -20,6 +21,14 @@ func requireDBSync(svcCtx *svc.ServiceContext, w http.ResponseWriter) bool {
return true
}
func reconcileMinInterval(svcCtx *svc.ServiceContext) time.Duration {
sec := 300
if svcCtx != nil && svcCtx.Config.DBSync.ReconcileMinSec > 0 {
sec = svcCtx.Config.DBSync.ReconcileMinSec
}
return time.Duration(sec) * time.Second
}
// sync 仅公司顶级权限(管理员 /「数据同步」);智能体与编辑不可配。
func syncTenantID(r *http.Request) int64 {
return authx.TenantID(r.Context())
@@ -74,6 +83,15 @@ func syncSaveHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
}
// 强制归属当前公司,禁止客户端伪造 tenant_id
ch.TenantID = syncTenantID(r)
if err := dbsync.ValidateChannelConfig(&ch); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
// 对可达端做 UUID 主键 + 外键闭包校验remote 通常为线上库)
if err := dbsync.ValidateChannelAgainstDB(r.Context(), &ch); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
saved, err := svcCtx.DBSync.Store().SaveChannel(ch)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
@@ -189,37 +207,14 @@ func syncStopHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
func syncConflictsHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
only := r.URL.Query().Get("unresolved") != "0"
list, err := svcCtx.DBSync.Store().ListConflictsByTenant(syncTenantID(r), only)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{"items": list})
// M3LWW/冲突追溯仅平台超级管理员;公司 top 403
authx.WriteError(w, http.StatusForbidden, "冲突/LWW 覆盖日志仅平台超级管理员可查")
}
}
func syncResolveConflictHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
var body struct {
Resolution string `json:"resolution"` // apply_source | keep_target | discard
}
_ = json.NewDecoder(r.Body).Decode(&body)
if body.Resolution == "" {
body.Resolution = "discard"
}
id := pathvar.Vars(r)["id"]
if err := svcCtx.DBSync.Store().ResolveConflictForTenant(id, syncTenantID(r), body.Resolution); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{"ok": true, "resolution": body.Resolution})
authx.WriteError(w, http.StatusForbidden, "冲突/LWW 覆盖日志仅平台超级管理员可操作")
}
}
@@ -233,11 +228,19 @@ func syncReconcileHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
authx.WriteError(w, http.StatusNotFound, err.Error())
return
}
if ok, wait := dbsync.CanReconcile(ch, reconcileMinInterval(svcCtx)); !ok {
authx.WriteError(w, http.StatusTooManyRequests, dbsync.ReconcileTooSoonError(wait).Error())
return
}
res, err := dbsync.ReconcileChannel(r.Context(), ch)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
now := time.Now().UTC()
_ = svcCtx.DBSync.Store().PatchStats(ch.ID, func(c *dbsync.Channel) {
c.LastReconcileAt = &now
})
httpx.OkJson(w, res)
}
}

View File

@@ -0,0 +1,50 @@
package handler
import (
"encoding/json"
"net/http"
"aijianzhan/platform/internal/authx"
"aijianzhan/platform/internal/dbsync"
"aijianzhan/platform/internal/svc"
"github.com/zeromicro/go-zero/rest/httpx"
)
func syncBindingsListHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
localID := r.URL.Query().Get("local_database_id")
list, err := svcCtx.DBSync.Store().ListBindings(syncTenantID(r), localID)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, map[string]any{"items": list})
}
}
func syncBindingsEnsureHandler(svcCtx *svc.ServiceContext) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !requireDBSync(svcCtx, w) {
return
}
var body dbsync.Binding
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
body.TenantID = syncTenantID(r)
if body.UserID == 0 {
body.UserID = authx.UserID(r.Context())
}
saved, err := svcCtx.DBSync.Store().EnsureBinding(body)
if err != nil {
authx.WriteError(w, http.StatusBadRequest, err.Error())
return
}
httpx.OkJson(w, saved)
}
}