feat: ship loose-offline dbsync (validate, agent push, LWW audit)
Add UUID/FK channel checks, agent whitelist/push APIs, bindings, super-admin LWW audit with rollback, reconcile rate limits, and sync docs. Default customers stay opt-in; company conflict UI is removed. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
143
platform/internal/dbsync/binding.go
Normal file
143
platform/internal/dbsync/binding.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Binding 本机库 ↔ 线上库映射（P1；不挡推送，供登记/查询）。
|
||||
type Binding struct {
|
||||
ID string `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
UserID int64 `json:"user_id,omitempty"`
|
||||
LocalDatabaseID string `json:"local_database_id"`
|
||||
OnlineDBID string `json:"online_db_id"`
|
||||
ChannelID string `json:"channel_id,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (s *FileStore) bindingPath() string {
|
||||
return filepath.Join(s.dir, "bindings.json")
|
||||
}
|
||||
|
||||
func (s *FileStore) EnsureBinding(b Binding) (Binding, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readBindingsUnlocked()
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
b.LocalDatabaseID = strings.TrimSpace(b.LocalDatabaseID)
|
||||
b.OnlineDBID = strings.TrimSpace(b.OnlineDBID)
|
||||
if b.TenantID <= 0 {
|
||||
return b, fmt.Errorf("tenant_id required")
|
||||
}
|
||||
if b.LocalDatabaseID == "" || b.OnlineDBID == "" {
|
||||
return b, fmt.Errorf("local_database_id and online_db_id required")
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for i := range list {
|
||||
if list[i].TenantID == b.TenantID && list[i].LocalDatabaseID == b.LocalDatabaseID {
|
||||
if b.UserID > 0 && list[i].UserID > 0 && list[i].UserID != b.UserID {
|
||||
continue
|
||||
}
|
||||
list[i].OnlineDBID = b.OnlineDBID
|
||||
if b.ChannelID != "" {
|
||||
list[i].ChannelID = b.ChannelID
|
||||
}
|
||||
if b.Note != "" {
|
||||
list[i].Note = b.Note
|
||||
}
|
||||
if b.UserID > 0 {
|
||||
list[i].UserID = b.UserID
|
||||
}
|
||||
list[i].UpdatedAt = now
|
||||
if err := s.writeBindingsUnlocked(list); err != nil {
|
||||
return b, err
|
||||
}
|
||||
return list[i], nil
|
||||
}
|
||||
}
|
||||
if b.ID == "" {
|
||||
b.ID = uuid.NewString()
|
||||
}
|
||||
b.CreatedAt = now
|
||||
b.UpdatedAt = now
|
||||
list = append(list, b)
|
||||
if err := s.writeBindingsUnlocked(list); err != nil {
|
||||
return b, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) ListBindings(tenantID int64, localDatabaseID string) ([]Binding, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readBindingsUnlocked()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Binding, 0)
|
||||
for _, b := range list {
|
||||
if b.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if localDatabaseID != "" && b.LocalDatabaseID != localDatabaseID {
|
||||
continue
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) GetBinding(tenantID int64, localDatabaseID string) (*Binding, error) {
|
||||
list, err := s.ListBindings(tenantID, localDatabaseID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return nil, fmt.Errorf("binding not found")
|
||||
}
|
||||
cp := list[0]
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) readBindingsUnlocked() ([]Binding, error) {
|
||||
path := s.bindingPath()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []Binding{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if len(b) == 0 {
|
||||
return []Binding{}, nil
|
||||
}
|
||||
var list []Binding
|
||||
if err := json.Unmarshal(b, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) writeBindingsUnlocked(list []Binding) error {
|
||||
path := s.bindingPath()
|
||||
raw, err := json.MarshalIndent(list, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
35
platform/internal/dbsync/binding_test.go
Normal file
35
platform/internal/dbsync/binding_test.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEnsureBindingUpsert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := st.EnsureBinding(Binding{
|
||||
TenantID: 1,
|
||||
LocalDatabaseID: "local-a",
|
||||
OnlineDBID: "online-1",
|
||||
ChannelID: "ch1",
|
||||
})
|
||||
if err != nil || b.ID == "" {
|
||||
t.Fatalf("ensure: %+v err=%v", b, err)
|
||||
}
|
||||
b2, err := st.EnsureBinding(Binding{
|
||||
TenantID: 1,
|
||||
LocalDatabaseID: "local-a",
|
||||
OnlineDBID: "online-2",
|
||||
})
|
||||
if err != nil || b2.OnlineDBID != "online-2" || b2.ID != b.ID {
|
||||
t.Fatalf("upsert: %+v err=%v", b2, err)
|
||||
}
|
||||
list, err := st.ListBindings(1, "local-a")
|
||||
if err != nil || len(list) != 1 {
|
||||
t.Fatalf("list=%d err=%v", len(list), err)
|
||||
}
|
||||
}
|
||||
41
platform/internal/dbsync/lww_audit.go
Normal file
41
platform/internal/dbsync/lww_audit.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
EntryDrain = "drain"
|
||||
EntryAgentPush = "agent_push"
|
||||
EntryRollback = "rollback"
|
||||
OutcomeApplied = "applied_source"
|
||||
OutcomeKept = "kept_target"
|
||||
OutcomeRolled = "rolled_back"
|
||||
)
|
||||
|
||||
// RecordLwwOverride 在 LWW 覆盖或保留目标时写入超管审计（失败忽略，不挡同步）。
|
||||
func RecordLwwOverride(store *FileStore, o LwwOverride) {
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
_ = store.AddLwwOverride(o)
|
||||
}
|
||||
|
||||
// SnapshotTargetRow 取目标端当前行 JSON（无行则空串）。
|
||||
func SnapshotTargetRow(ctx context.Context, db *sql.DB, driver Driver, table, pkCol, rowPK string) string {
|
||||
if db == nil || rowPK == "" {
|
||||
return ""
|
||||
}
|
||||
js, _, err := FetchRowJSON(ctx, db, driver, table, pkCol, rowPK)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return js
|
||||
}
|
||||
|
||||
// DefaultLwwAuditTTL 默认 90 天。
|
||||
func DefaultLwwAuditTTL() time.Duration {
|
||||
return 90 * 24 * time.Hour
|
||||
}
|
||||
72
platform/internal/dbsync/lww_audit_test.go
Normal file
72
platform/internal/dbsync/lww_audit_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLwwOverrideStoreAndPurge(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := LwwOverride{
|
||||
TenantID: 1,
|
||||
ChannelID: "ch1",
|
||||
Table: "orders",
|
||||
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
Entry: EntryAgentPush,
|
||||
Policy: string(PolicyLWWSource),
|
||||
Outcome: OutcomeApplied,
|
||||
CreatedAt: time.Now().UTC().Add(-100 * 24 * time.Hour),
|
||||
}
|
||||
fresh := LwwOverride{
|
||||
TenantID: 1,
|
||||
ChannelID: "ch1",
|
||||
Table: "orders",
|
||||
RowPK: "bbbbbbbb-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
Entry: EntryDrain,
|
||||
Policy: string(PolicyLWWSource),
|
||||
Outcome: OutcomeApplied,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := st.AddLwwOverride(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.AddLwwOverride(fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, err := st.ListLwwOverrides(1, "ch1", 10)
|
||||
if err != nil || len(list) != 2 {
|
||||
t.Fatalf("list=%d err=%v", len(list), err)
|
||||
}
|
||||
n, err := st.PurgeLwwOverridesBefore(time.Now().UTC().Add(-90 * 24 * time.Hour))
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("purge n=%d err=%v", n, err)
|
||||
}
|
||||
list, err = st.ListLwwOverrides(0, "", 10)
|
||||
if err != nil || len(list) != 1 {
|
||||
t.Fatalf("after purge list=%d err=%v", len(list), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanReconcile(t *testing.T) {
|
||||
ok, _ := CanReconcile(nil, time.Minute)
|
||||
if !ok {
|
||||
t.Fatal("nil channel should allow")
|
||||
}
|
||||
past := time.Now().UTC().Add(-10 * time.Minute)
|
||||
ch := &Channel{LastReconcileAt: &past}
|
||||
ok, _ = CanReconcile(ch, 5*time.Minute)
|
||||
if !ok {
|
||||
t.Fatal("expected allow")
|
||||
}
|
||||
recent := time.Now().UTC()
|
||||
ch.LastReconcileAt = &recent
|
||||
ok, wait := CanReconcile(ch, 5*time.Minute)
|
||||
if ok || wait <= 0 {
|
||||
t.Fatalf("expected deny wait>0 ok=%v wait=%v", ok, wait)
|
||||
}
|
||||
}
|
||||
@@ -93,10 +93,24 @@ func (m *Manager) loop(ctx context.Context, id string) {
|
||||
log.Printf("dbsync channel %s: %v", id, err)
|
||||
}
|
||||
ticks++
|
||||
// 约每分钟主键对账一次，补漏（漏投递 / 触发器未装时的存量差）
|
||||
if ticks%120 == 0 && ch.Direction == DirBidirectional {
|
||||
if _, rerr := ReconcileChannel(ctx, ch); rerr != nil {
|
||||
log.Printf("dbsync reconcile %s: %v", id, rerr)
|
||||
// 双向通道自动对账：默认约每 15 分钟一次（限流）
|
||||
autoEvery := 1800 // poll 500ms → ~15min
|
||||
if ch.PollIntervalMS > 0 {
|
||||
autoEvery = int((15 * time.Minute) / (time.Duration(ch.PollIntervalMS) * time.Millisecond))
|
||||
if autoEvery < 60 {
|
||||
autoEvery = 60
|
||||
}
|
||||
}
|
||||
if ticks%autoEvery == 0 && ch.Direction == DirBidirectional {
|
||||
if allow, _ := CanReconcile(ch, 15*time.Minute); allow {
|
||||
if _, rerr := ReconcileChannel(ctx, ch); rerr != nil {
|
||||
log.Printf("dbsync reconcile %s: %v", id, rerr)
|
||||
} else {
|
||||
_ = m.store.PatchStats(id, func(c *Channel) {
|
||||
now := time.Now().UTC()
|
||||
c.LastReconcileAt = &now
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
@@ -214,12 +228,63 @@ func (m *Manager) drain(ctx context.Context, ch *Channel, sourceName string, src
|
||||
continue
|
||||
}
|
||||
if has && tgtVer > r.Version {
|
||||
switch ch.ConflictPolicy {
|
||||
policy := ch.ConflictPolicy
|
||||
if policy == "" {
|
||||
policy = PolicyLWWSource
|
||||
}
|
||||
switch policy {
|
||||
case PolicyLWWTarget:
|
||||
loser := payload
|
||||
winner := SnapshotTargetRow(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.RowPK)
|
||||
RecordLwwOverride(m.store, LwwOverride{
|
||||
TenantID: ch.TenantID,
|
||||
ChannelID: ch.ID,
|
||||
Table: r.TableName,
|
||||
RowPK: r.RowPK,
|
||||
Op: r.Op,
|
||||
Entry: EntryDrain,
|
||||
Policy: string(PolicyLWWTarget),
|
||||
Outcome: OutcomeKept,
|
||||
LoserPayload: loser,
|
||||
WinnerPayload: winner,
|
||||
TargetVer: tgtVer,
|
||||
SourceVer: r.Version,
|
||||
})
|
||||
done = append(done, r.ID)
|
||||
continue
|
||||
case PolicyLWWSource:
|
||||
// fallthrough apply
|
||||
loser := SnapshotTargetRow(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.RowPK)
|
||||
if err := ApplyChange(ctx, dst, dstEp.Driver, r.TableName, pkCol, r.Op, payload, r.Version); err != nil {
|
||||
_ = m.store.PatchStats(ch.ID, func(c *Channel) {
|
||||
c.Stats.Retries++
|
||||
c.LastError = err.Error()
|
||||
})
|
||||
continue
|
||||
}
|
||||
RecordLwwOverride(m.store, LwwOverride{
|
||||
TenantID: ch.TenantID,
|
||||
ChannelID: ch.ID,
|
||||
Table: r.TableName,
|
||||
RowPK: r.RowPK,
|
||||
Op: r.Op,
|
||||
Entry: EntryDrain,
|
||||
Policy: string(PolicyLWWSource),
|
||||
Outcome: OutcomeApplied,
|
||||
LoserPayload: loser,
|
||||
WinnerPayload: payload,
|
||||
TargetVer: tgtVer,
|
||||
SourceVer: r.Version,
|
||||
})
|
||||
done = append(done, r.ID)
|
||||
okCount++
|
||||
_ = m.store.PatchStats(ch.ID, func(c *Channel) {
|
||||
if sourceName == "local" {
|
||||
c.Stats.PushedOK++
|
||||
} else {
|
||||
c.Stats.PulledOK++
|
||||
}
|
||||
})
|
||||
continue
|
||||
default:
|
||||
_ = m.store.AddConflict(Conflict{
|
||||
TenantID: ch.TenantID,
|
||||
@@ -264,6 +329,9 @@ func (m *Manager) drain(ctx context.Context, ch *Channel, sourceName string, src
|
||||
|
||||
// PrepareChannel 连接两端、建 outbox/触发器，供「测试/启用」调用。
|
||||
func PrepareChannel(ctx context.Context, ch *Channel) error {
|
||||
if err := ValidateChannelAgainstDB(ctx, ch); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ep := range []Endpoint{ch.Local, ch.Remote} {
|
||||
db, err := Open(ep.Driver, ep.DSN)
|
||||
if err != nil {
|
||||
|
||||
256
platform/internal/dbsync/push.go
Normal file
256
platform/internal/dbsync/push.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PushItem 本机 agent → 线上 A 的一条变更（形态 B）。
|
||||
type PushItem struct {
|
||||
Table string `json:"table"`
|
||||
Op string `json:"op"` // insert|update|update_by_id|delete|upsert
|
||||
RowPK string `json:"row_pk"`
|
||||
Row map[string]any `json:"row"` // 单行；与 Rows 二选一
|
||||
Rows []map[string]any `json:"rows"` // 兼容批量 insert
|
||||
Version int64 `json:"version"` // outbox 单调 version；0 则用时间戳
|
||||
ClientOutboxID string `json:"client_outbox_id"`
|
||||
OnlineDBID string `json:"online_db_id"`
|
||||
}
|
||||
|
||||
// PushResult 单条推送结果。
|
||||
type PushResult struct {
|
||||
OK bool `json:"ok"`
|
||||
Applied bool `json:"applied"`
|
||||
Skipped bool `json:"skipped"`
|
||||
Conflict bool `json:"conflict"`
|
||||
AppliedVersion int64 `json:"applied_version"`
|
||||
Message string `json:"message"`
|
||||
ClientOutboxID string `json:"client_outbox_id,omitempty"`
|
||||
}
|
||||
|
||||
// PushToRemote 仅打开 remote，按 LWW/幂等将变更落到线上 A。不连本机 SQLite。
|
||||
func PushToRemote(ctx context.Context, ch *Channel, store *FileStore, item PushItem) (*PushResult, error) {
|
||||
if ch == nil {
|
||||
return nil, fmt.Errorf("channel is nil")
|
||||
}
|
||||
table := strings.TrimSpace(item.Table)
|
||||
if table == "" {
|
||||
return nil, fmt.Errorf("table required")
|
||||
}
|
||||
if !tableInChannel(ch, table) {
|
||||
return nil, fmt.Errorf("table %s 不在通道白名单", table)
|
||||
}
|
||||
pkCol := "id"
|
||||
if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" {
|
||||
pkCol = ch.PKColumns[table]
|
||||
}
|
||||
|
||||
op, payload, rowPK, err := normalizePushPayload(item, pkCol)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
version := item.Version
|
||||
if version <= 0 {
|
||||
version = time.Now().UnixNano()
|
||||
}
|
||||
|
||||
db, err := Open(ch.Remote.Driver, ch.Remote.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open remote: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil {
|
||||
return nil, fmt.Errorf("ensure meta: %w", err)
|
||||
}
|
||||
|
||||
res := &PushResult{
|
||||
ClientOutboxID: item.ClientOutboxID,
|
||||
AppliedVersion: version,
|
||||
}
|
||||
|
||||
tgtVer, has, err := GetMetaVersion(ctx, db, ch.Remote.Driver, table, rowPK)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 幂等：同 version 已落地 → 跳过
|
||||
if has && tgtVer == version {
|
||||
res.OK = true
|
||||
res.Skipped = true
|
||||
res.Message = "already applied (same version)"
|
||||
return res, nil
|
||||
}
|
||||
if has && tgtVer > version {
|
||||
policy := ch.ConflictPolicy
|
||||
if policy == "" {
|
||||
policy = PolicyLWWSource // B→A 默认偏源
|
||||
}
|
||||
switch policy {
|
||||
case PolicyLWWTarget:
|
||||
loser := payload
|
||||
winner := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, rowPK)
|
||||
RecordLwwOverride(store, LwwOverride{
|
||||
TenantID: ch.TenantID,
|
||||
ChannelID: ch.ID,
|
||||
Table: table,
|
||||
RowPK: rowPK,
|
||||
Op: op,
|
||||
Entry: EntryAgentPush,
|
||||
Policy: string(PolicyLWWTarget),
|
||||
Outcome: OutcomeKept,
|
||||
LoserPayload: loser,
|
||||
WinnerPayload: winner,
|
||||
TargetVer: tgtVer,
|
||||
SourceVer: version,
|
||||
})
|
||||
res.OK = true
|
||||
res.Skipped = true
|
||||
res.Message = "target newer; kept (lww_target)"
|
||||
return res, nil
|
||||
case PolicyLWWSource:
|
||||
loser := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, rowPK)
|
||||
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
RecordLwwOverride(store, LwwOverride{
|
||||
TenantID: ch.TenantID,
|
||||
ChannelID: ch.ID,
|
||||
Table: table,
|
||||
RowPK: rowPK,
|
||||
Op: op,
|
||||
Entry: EntryAgentPush,
|
||||
Policy: string(PolicyLWWSource),
|
||||
Outcome: OutcomeApplied,
|
||||
LoserPayload: loser,
|
||||
WinnerPayload: payload,
|
||||
TargetVer: tgtVer,
|
||||
SourceVer: version,
|
||||
})
|
||||
if store != nil {
|
||||
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ })
|
||||
}
|
||||
res.OK = true
|
||||
res.Applied = true
|
||||
res.Message = "applied (lww_source override)"
|
||||
return res, nil
|
||||
default:
|
||||
if store != nil {
|
||||
_ = store.AddConflict(Conflict{
|
||||
TenantID: ch.TenantID,
|
||||
ChannelID: ch.ID,
|
||||
Table: table,
|
||||
RowPK: rowPK,
|
||||
Op: op,
|
||||
Source: "agent",
|
||||
Payload: payload,
|
||||
TargetVer: tgtVer,
|
||||
SourceVer: version,
|
||||
Message: "target version newer than agent push",
|
||||
})
|
||||
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.Conflicts++ })
|
||||
}
|
||||
res.OK = true
|
||||
res.Skipped = true
|
||||
res.Conflict = true
|
||||
res.Message = "queued conflict; target newer"
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, version); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store != nil {
|
||||
_ = store.PatchStats(ch.ID, func(c *Channel) { c.Stats.PushedOK++ })
|
||||
}
|
||||
res.OK = true
|
||||
res.Applied = true
|
||||
res.Message = "applied"
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// PushBatchToRemote 保序批量；遇错即停，已成功条数在返回切片中。
|
||||
func PushBatchToRemote(ctx context.Context, ch *Channel, store *FileStore, items []PushItem) ([]PushResult, error) {
|
||||
out := make([]PushResult, 0, len(items))
|
||||
for i, it := range items {
|
||||
r, err := PushToRemote(ctx, ch, store, it)
|
||||
if err != nil {
|
||||
fail := PushResult{
|
||||
OK: false,
|
||||
Message: err.Error(),
|
||||
ClientOutboxID: it.ClientOutboxID,
|
||||
}
|
||||
out = append(out, fail)
|
||||
return out, fmt.Errorf("item[%d]: %w", i, err)
|
||||
}
|
||||
out = append(out, *r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func tableInChannel(ch *Channel, table string) bool {
|
||||
for _, t := range uniqueTables(ch.Local.Tables, ch.Remote.Tables) {
|
||||
if strings.EqualFold(strings.TrimSpace(t), table) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizePushPayload(item PushItem, pkCol string) (op string, payload string, rowPK string, err error) {
|
||||
rawOp := strings.ToLower(strings.TrimSpace(item.Op))
|
||||
if rawOp == "" {
|
||||
rawOp = "upsert"
|
||||
}
|
||||
switch rawOp {
|
||||
case "delete":
|
||||
op = "delete"
|
||||
case "insert", "update", "update_by_id", "upsert":
|
||||
op = "upsert"
|
||||
default:
|
||||
return "", "", "", fmt.Errorf("unsupported op: %s", item.Op)
|
||||
}
|
||||
|
||||
row := item.Row
|
||||
if row == nil && len(item.Rows) > 0 {
|
||||
row = item.Rows[0]
|
||||
}
|
||||
rowPK = strings.TrimSpace(item.RowPK)
|
||||
if row != nil {
|
||||
if rowPK == "" {
|
||||
rowPK = strings.TrimSpace(fmt.Sprint(row[pkCol]))
|
||||
}
|
||||
if op == "upsert" {
|
||||
// 保证 payload 含主键
|
||||
if _, ok := row[pkCol]; !ok && rowPK != "" {
|
||||
row = copyMap(row)
|
||||
row[pkCol] = rowPK
|
||||
}
|
||||
}
|
||||
}
|
||||
if rowPK == "" || rowPK == "<nil>" {
|
||||
return "", "", "", fmt.Errorf("row_pk required")
|
||||
}
|
||||
if op == "delete" && row == nil {
|
||||
row = map[string]any{pkCol: rowPK}
|
||||
}
|
||||
if row == nil {
|
||||
return "", "", "", fmt.Errorf("row required for %s", op)
|
||||
}
|
||||
b, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
return op, string(b), rowPK, nil
|
||||
}
|
||||
|
||||
func copyMap(m map[string]any) map[string]any {
|
||||
out := make(map[string]any, len(m)+1)
|
||||
for k, v := range m {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
46
platform/internal/dbsync/push_test.go
Normal file
46
platform/internal/dbsync/push_test.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package dbsync
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizePushPayloadInsert(t *testing.T) {
|
||||
op, payload, pk, err := normalizePushPayload(PushItem{
|
||||
Op: "insert",
|
||||
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
Row: map[string]any{
|
||||
"id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
"name": "x",
|
||||
},
|
||||
}, "id")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if op != "upsert" || pk == "" || payload == "" {
|
||||
t.Fatalf("got op=%s pk=%s payload=%s", op, pk, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePushPayloadDelete(t *testing.T) {
|
||||
op, payload, pk, err := normalizePushPayload(PushItem{
|
||||
Op: "delete",
|
||||
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
}, "id")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if op != "delete" || pk == "" || payload == "" {
|
||||
t.Fatalf("got op=%s pk=%s payload=%s", op, pk, payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableInChannel(t *testing.T) {
|
||||
ch := &Channel{
|
||||
Local: Endpoint{Tables: []string{"orders"}},
|
||||
Remote: Endpoint{Tables: []string{"orders"}},
|
||||
}
|
||||
if !tableInChannel(ch, "orders") {
|
||||
t.Fatal("expected in")
|
||||
}
|
||||
if tableInChannel(ch, "other") {
|
||||
t.Fatal("expected out")
|
||||
}
|
||||
}
|
||||
30
platform/internal/dbsync/reconcile_limit.go
Normal file
30
platform/internal/dbsync/reconcile_limit.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultManualReconcileInterval 公司管理员手动对账最小间隔。
|
||||
const DefaultManualReconcileInterval = 5 * time.Minute
|
||||
|
||||
// CanReconcile 限流：距上次对账不足 minInterval 则拒绝。
|
||||
func CanReconcile(ch *Channel, minInterval time.Duration) (bool, time.Duration) {
|
||||
if ch == nil || ch.LastReconcileAt == nil || ch.LastReconcileAt.IsZero() {
|
||||
return true, 0
|
||||
}
|
||||
if minInterval <= 0 {
|
||||
minInterval = DefaultManualReconcileInterval
|
||||
}
|
||||
elapsed := time.Since(ch.LastReconcileAt.UTC())
|
||||
if elapsed >= minInterval {
|
||||
return true, 0
|
||||
}
|
||||
return false, minInterval - elapsed
|
||||
}
|
||||
|
||||
// ReconcileTooSoonError 供 handler 返回 429。
|
||||
func ReconcileTooSoonError(wait time.Duration) error {
|
||||
sec := int(wait.Seconds()) + 1
|
||||
return fmt.Errorf("对账过于频繁，请 %d 秒后再试", sec)
|
||||
}
|
||||
79
platform/internal/dbsync/rollback.go
Normal file
79
platform/internal/dbsync/rollback.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// RollbackLwwOverride 超管按落败快照回滚线上 A 单行。
|
||||
// 仅 outcome=applied_source 可回滚：把 loser_payload 写回 remote；无快照则按 pk 删除。
|
||||
func RollbackLwwOverride(ctx context.Context, store *FileStore, overrideID string) (*LwwOverride, error) {
|
||||
if store == nil {
|
||||
return nil, fmt.Errorf("store is nil")
|
||||
}
|
||||
o, err := store.GetLwwOverride(overrideID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o.Outcome != OutcomeApplied {
|
||||
return nil, fmt.Errorf("仅「源端覆盖」记录可回滚（当前 outcome=%s）", o.Outcome)
|
||||
}
|
||||
if strings.TrimSpace(o.ChannelID) == "" {
|
||||
return nil, fmt.Errorf("override missing channel_id")
|
||||
}
|
||||
ch, err := store.GetChannel(o.ChannelID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("channel: %w", err)
|
||||
}
|
||||
table := strings.TrimSpace(o.Table)
|
||||
pkCol := "id"
|
||||
if ch.PKColumns != nil && strings.TrimSpace(ch.PKColumns[table]) != "" {
|
||||
pkCol = ch.PKColumns[table]
|
||||
}
|
||||
db, err := Open(ch.Remote.Driver, ch.Remote.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open remote: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if err := EnsureMeta(ctx, db, ch.Remote.Driver); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
before := SnapshotTargetRow(ctx, db, ch.Remote.Driver, table, pkCol, o.RowPK)
|
||||
ver := time.Now().UnixNano()
|
||||
loser := strings.TrimSpace(o.LoserPayload)
|
||||
op := "upsert"
|
||||
payload := loser
|
||||
if loser == "" || loser == "{}" {
|
||||
op = "delete"
|
||||
payload = fmt.Sprintf(`{%q:%q}`, pkCol, o.RowPK)
|
||||
}
|
||||
if err := ApplyChange(ctx, db, ch.Remote.Driver, table, pkCol, op, payload, ver); err != nil {
|
||||
return nil, fmt.Errorf("apply rollback: %w", err)
|
||||
}
|
||||
|
||||
rec := LwwOverride{
|
||||
ID: uuid.NewString(),
|
||||
TenantID: o.TenantID,
|
||||
ChannelID: o.ChannelID,
|
||||
Table: o.Table,
|
||||
RowPK: o.RowPK,
|
||||
Op: op,
|
||||
Entry: EntryRollback,
|
||||
Policy: "manual_rollback",
|
||||
Outcome: OutcomeRolled,
|
||||
LoserPayload: before,
|
||||
WinnerPayload: payload,
|
||||
TargetVer: o.SourceVer,
|
||||
SourceVer: ver,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := store.AddLwwOverride(rec); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &rec, nil
|
||||
}
|
||||
54
platform/internal/dbsync/rollback_test.go
Normal file
54
platform/internal/dbsync/rollback_test.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRollbackRejectsNonApplied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := LwwOverride{
|
||||
ID: "ov1",
|
||||
TenantID: 1,
|
||||
ChannelID: "ch1",
|
||||
Table: "orders",
|
||||
RowPK: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
Outcome: OutcomeKept,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
if err := st.AddLwwOverride(o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = RollbackLwwOverride(context.Background(), st, "ov1")
|
||||
if err == nil {
|
||||
t.Fatal("expected reject kept_target")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLwwOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
st, err := NewFileStore(filepath.Join(dir, "dbsync"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o := LwwOverride{
|
||||
ID: "ov2",
|
||||
TenantID: 1,
|
||||
Table: "t",
|
||||
RowPK: "pk",
|
||||
Outcome: OutcomeApplied,
|
||||
}
|
||||
if err := st.AddLwwOverride(o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := st.GetLwwOverride("ov2")
|
||||
if err != nil || got.Table != "t" {
|
||||
t.Fatalf("got=%+v err=%v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -11,12 +11,13 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// FileStore 持久化通道与冲突队列（JSON），不依赖业务库类型。
|
||||
// FileStore 持久化通道、冲突队列与超管 LWW 审计（JSON），不依赖业务库类型。
|
||||
type FileStore struct {
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
chPath string
|
||||
cfPath string
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
chPath string
|
||||
cfPath string
|
||||
lwwPath string
|
||||
}
|
||||
|
||||
func NewFileStore(dir string) (*FileStore, error) {
|
||||
@@ -27,9 +28,10 @@ func NewFileStore(dir string) (*FileStore, error) {
|
||||
return nil, err
|
||||
}
|
||||
return &FileStore{
|
||||
dir: dir,
|
||||
chPath: filepath.Join(dir, "channels.json"),
|
||||
cfPath: filepath.Join(dir, "conflicts.json"),
|
||||
dir: dir,
|
||||
chPath: filepath.Join(dir, "channels.json"),
|
||||
cfPath: filepath.Join(dir, "conflicts.json"),
|
||||
lwwPath: filepath.Join(dir, "lww_overrides.json"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -103,7 +105,7 @@ func (s *FileStore) SaveChannel(ch Channel) (Channel, error) {
|
||||
ch.Direction = DirLocalToRemote
|
||||
}
|
||||
if ch.ConflictPolicy == "" {
|
||||
ch.ConflictPolicy = PolicyQueue
|
||||
ch.ConflictPolicy = PolicyLWWSource
|
||||
}
|
||||
if ch.PKColumns == nil {
|
||||
ch.PKColumns = map[string]string{}
|
||||
@@ -120,6 +122,7 @@ func (s *FileStore) SaveChannel(ch Channel) (Channel, error) {
|
||||
}
|
||||
ch.CreatedAt = list[i].CreatedAt
|
||||
ch.Stats = list[i].Stats
|
||||
ch.LastReconcileAt = list[i].LastReconcileAt
|
||||
list[i] = ch
|
||||
found = true
|
||||
break
|
||||
@@ -325,3 +328,117 @@ func (s *FileStore) writeConflicts(list []Conflict) error {
|
||||
}
|
||||
return os.Rename(tmp, s.cfPath)
|
||||
}
|
||||
|
||||
func (s *FileStore) AddLwwOverride(o LwwOverride) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readLwwOverrides()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.ID == "" {
|
||||
o.ID = uuid.NewString()
|
||||
}
|
||||
if o.CreatedAt.IsZero() {
|
||||
o.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
list = append(list, o)
|
||||
return s.writeLwwOverrides(list)
|
||||
}
|
||||
|
||||
func (s *FileStore) GetLwwOverride(id string) (*LwwOverride, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readLwwOverrides()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range list {
|
||||
if list[i].ID == id {
|
||||
cp := list[i]
|
||||
return &cp, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("lww override not found")
|
||||
}
|
||||
|
||||
// ListLwwOverrides 超管查询；tenantID/channelID 为 0/空 表示不过滤。
|
||||
func (s *FileStore) ListLwwOverrides(tenantID int64, channelID string, limit int) ([]LwwOverride, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readLwwOverrides()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]LwwOverride, 0, len(list))
|
||||
for i := len(list) - 1; i >= 0; i-- { // 新→旧
|
||||
o := list[i]
|
||||
if tenantID > 0 && o.TenantID != tenantID {
|
||||
continue
|
||||
}
|
||||
if channelID != "" && o.ChannelID != channelID {
|
||||
continue
|
||||
}
|
||||
out = append(out, o)
|
||||
if limit > 0 && len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PurgeLwwOverridesBefore 删除 created_at 早于 cutoff 的记录，返回删除条数。
|
||||
func (s *FileStore) PurgeLwwOverridesBefore(cutoff time.Time) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
list, err := s.readLwwOverrides()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
keep := make([]LwwOverride, 0, len(list))
|
||||
removed := 0
|
||||
for _, o := range list {
|
||||
if o.CreatedAt.Before(cutoff) {
|
||||
removed++
|
||||
continue
|
||||
}
|
||||
keep = append(keep, o)
|
||||
}
|
||||
if removed == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := s.writeLwwOverrides(keep); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) readLwwOverrides() ([]LwwOverride, error) {
|
||||
b, err := os.ReadFile(s.lwwPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []LwwOverride{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if len(b) == 0 {
|
||||
return []LwwOverride{}, nil
|
||||
}
|
||||
var list []LwwOverride
|
||||
if err := json.Unmarshal(b, &list); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) writeLwwOverrides(list []LwwOverride) error {
|
||||
b, err := json.MarshalIndent(list, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.lwwPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.lwwPath)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ type Channel struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
LastSyncAt *time.Time `json:"last_sync_at,omitempty"`
|
||||
LastReconcileAt *time.Time `json:"last_reconcile_at,omitempty"`
|
||||
Stats ChannelStats `json:"stats"`
|
||||
}
|
||||
|
||||
@@ -80,6 +81,24 @@ type Conflict struct {
|
||||
Resolution string `json:"resolution,omitempty"` // apply_source | keep_target | discard
|
||||
}
|
||||
|
||||
// LwwOverride LWW 自动覆盖审计（仅平台超级管理员可见；与租户冲突队列分离）。
|
||||
type LwwOverride struct {
|
||||
ID string `json:"id"`
|
||||
TenantID int64 `json:"tenant_id"`
|
||||
ChannelID string `json:"channel_id"`
|
||||
Table string `json:"table"`
|
||||
RowPK string `json:"row_pk"`
|
||||
Op string `json:"op"`
|
||||
Entry string `json:"entry"` // drain | agent_push
|
||||
Policy string `json:"policy"` // lww_source | lww_target
|
||||
Outcome string `json:"outcome"` // applied_source | kept_target
|
||||
LoserPayload string `json:"loser_payload"` // 落败侧
|
||||
WinnerPayload string `json:"winner_payload"` // 胜出侧
|
||||
TargetVer int64 `json:"target_ver"`
|
||||
SourceVer int64 `json:"source_ver"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type OutboxRow struct {
|
||||
ID int64
|
||||
TableName string
|
||||
@@ -91,8 +110,8 @@ type OutboxRow struct {
|
||||
}
|
||||
|
||||
type TestResult struct {
|
||||
OK bool `json:"ok"`
|
||||
Driver string `json:"driver"`
|
||||
Message string `json:"message"`
|
||||
OK bool `json:"ok"`
|
||||
Driver string `json:"driver"`
|
||||
Message string `json:"message"`
|
||||
Tables []string `json:"tables,omitempty"`
|
||||
}
|
||||
|
||||
264
platform/internal/dbsync/validate.go
Normal file
264
platform/internal/dbsync/validate.go
Normal file
@@ -0,0 +1,264 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ValidateChannelConfig 静态校验（不连库）：表名单、PK 列名约定。
|
||||
func ValidateChannelConfig(ch *Channel) error {
|
||||
if ch == nil {
|
||||
return fmt.Errorf("channel is nil")
|
||||
}
|
||||
tables := uniqueTables(ch.Local.Tables, ch.Remote.Tables)
|
||||
if len(tables) == 0 {
|
||||
return fmt.Errorf("同步表白名单为空：请至少在 local 或 remote 填写表名")
|
||||
}
|
||||
for _, t := range tables {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" {
|
||||
return fmt.Errorf("表名不能为空")
|
||||
}
|
||||
if strings.HasPrefix(t, "_ajz_") {
|
||||
return fmt.Errorf("禁止同步系统表: %s", t)
|
||||
}
|
||||
pk := pkColumn(ch, t)
|
||||
if pk == "" {
|
||||
return fmt.Errorf("表 %s 主键列名为空", t)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pkColumn(ch *Channel, table string) string {
|
||||
if ch.PKColumns != nil {
|
||||
if v := strings.TrimSpace(ch.PKColumns[table]); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return "id"
|
||||
}
|
||||
|
||||
// ValidateChannelAgainstDB 对可连接端做：TEXT/UUID 主键类型 + FK 闭包。
|
||||
// 某端连不上时跳过该端（形态 B 下 local 常不可达），但至少一端须校验成功，否则拒绝。
|
||||
func ValidateChannelAgainstDB(ctx context.Context, ch *Channel) error {
|
||||
if err := ValidateChannelConfig(ch); err != nil {
|
||||
return err
|
||||
}
|
||||
tables := uniqueTables(ch.Local.Tables, ch.Remote.Tables)
|
||||
checked := 0
|
||||
var lastSkip error
|
||||
for _, ep := range []Endpoint{ch.Remote, ch.Local} {
|
||||
if strings.TrimSpace(ep.DSN) == "" || ep.Driver == "" {
|
||||
continue
|
||||
}
|
||||
epTables := ep.Tables
|
||||
if len(epTables) == 0 {
|
||||
epTables = tables
|
||||
}
|
||||
db, err := Open(ep.Driver, ep.DSN)
|
||||
if err != nil {
|
||||
lastSkip = fmt.Errorf("%s 无法连接（跳过库内校验）: %w", ep.Driver, err)
|
||||
continue
|
||||
}
|
||||
if err := validateEndpointSchema(ctx, db, ep.Driver, ch, epTables, tables); err != nil {
|
||||
_ = db.Close()
|
||||
return fmt.Errorf("[%s] %w", ep.Driver, err)
|
||||
}
|
||||
_ = db.Close()
|
||||
checked++
|
||||
}
|
||||
if checked == 0 {
|
||||
if lastSkip != nil {
|
||||
return fmt.Errorf("无法对任何端做主键/外键校验：%v；请保证线上库（remote）DSN 可达后再保存", lastSkip)
|
||||
}
|
||||
return fmt.Errorf("无法对任何端做主键/外键校验：请配置可达的 remote DSN")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateEndpointSchema(ctx context.Context, db *sql.DB, driver Driver, ch *Channel, epTables, whitelist []string) error {
|
||||
wl := map[string]struct{}{}
|
||||
for _, t := range whitelist {
|
||||
wl[strings.TrimSpace(t)] = struct{}{}
|
||||
}
|
||||
for _, t := range epTables {
|
||||
t = strings.TrimSpace(t)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
pk := pkColumn(ch, t)
|
||||
typ, err := DescribeColumnType(ctx, db, driver, t, pk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("表 %s 主键列 %s: %w", t, pk, err)
|
||||
}
|
||||
if !isTextLikePK(typ) {
|
||||
return fmt.Errorf("表 %s 主键 %s 类型为 %q，同步表须为 TEXT/VARCHAR/UUID 类（禁止自增整数作同步键）", t, pk, typ)
|
||||
}
|
||||
fks, err := ListForeignKeys(ctx, db, driver, t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("表 %s 外键: %w", t, err)
|
||||
}
|
||||
for _, fk := range fks {
|
||||
_, childIn := wl[fk.ChildTable]
|
||||
_, parentIn := wl[fk.ParentTable]
|
||||
if childIn != parentIn {
|
||||
missing := fk.ParentTable
|
||||
if !childIn {
|
||||
missing = fk.ChildTable
|
||||
}
|
||||
return fmt.Errorf("外键闭包不完整：%s.%s → %s.%s，请将 %s 一并加入同步白名单",
|
||||
fk.ChildTable, fk.ChildColumn, fk.ParentTable, fk.ParentColumn, missing)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isTextLikePK(typ string) bool {
|
||||
raw := strings.ToUpper(strings.TrimSpace(typ))
|
||||
if raw == "" {
|
||||
return false
|
||||
}
|
||||
token := strings.Fields(raw)[0]
|
||||
token = strings.Split(token, "(")[0]
|
||||
switch token {
|
||||
case "TEXT", "VARCHAR", "CHAR", "CHARACTER", "UUID", "NVARCHAR", "NCHAR", "STRING", "CITEXT", "CLOB":
|
||||
return true
|
||||
case "INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", "MEDIUMINT",
|
||||
"SERIAL", "BIGSERIAL", "SMALLSERIAL",
|
||||
"NUMERIC", "DECIMAL", "NUMBER", "FLOAT", "DOUBLE", "REAL", "BOOLEAN", "BOOL":
|
||||
return false
|
||||
default:
|
||||
if strings.Contains(raw, "CHAR") || strings.Contains(raw, "TEXT") || strings.Contains(raw, "UUID") || strings.Contains(raw, "CLOB") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ForeignKey 子表 → 父表
|
||||
type ForeignKey struct {
|
||||
ChildTable string
|
||||
ChildColumn string
|
||||
ParentTable string
|
||||
ParentColumn string
|
||||
}
|
||||
|
||||
// DescribeColumnType 返回列类型字符串（驱动相关原文）。
|
||||
func DescribeColumnType(ctx context.Context, db *sql.DB, driver Driver, table, column string) (string, error) {
|
||||
table = strings.TrimSpace(table)
|
||||
column = strings.TrimSpace(column)
|
||||
switch driver {
|
||||
case DriverSQLite:
|
||||
rows, err := db.QueryContext(ctx, fmt.Sprintf(`PRAGMA table_info(%s)`, quoteIdent(driver, table)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, typ string
|
||||
var notnull, pk int
|
||||
var dflt sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &typ, ¬null, &dflt, &pk); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if strings.EqualFold(name, column) {
|
||||
if typ == "" {
|
||||
typ = "TEXT" // sqlite 松类型兜底：无声明时按 TEXT 处理需调用方结合；此处空则拒
|
||||
return "", fmt.Errorf("列 %s 无类型声明（sqlite）；请显式声明为 TEXT", column)
|
||||
}
|
||||
return typ, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("列不存在")
|
||||
case DriverMySQL:
|
||||
var typ string
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT DATA_TYPE FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`, table, column).Scan(&typ)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return typ, nil
|
||||
case DriverPostgres:
|
||||
var typ string
|
||||
err := db.QueryRowContext(ctx, `
|
||||
SELECT data_type FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`, table, column).Scan(&typ)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return typ, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported driver")
|
||||
}
|
||||
}
|
||||
|
||||
// ListForeignKeys 列出以 table 为子表的外键。
|
||||
func ListForeignKeys(ctx context.Context, db *sql.DB, driver Driver, table string) ([]ForeignKey, error) {
|
||||
table = strings.TrimSpace(table)
|
||||
switch driver {
|
||||
case DriverSQLite:
|
||||
rows, err := db.QueryContext(ctx, fmt.Sprintf(`PRAGMA foreign_key_list(%s)`, quoteIdent(driver, table)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ForeignKey
|
||||
for rows.Next() {
|
||||
var id, seq int
|
||||
var parent, from, to, onUpdate, onDelete, match string
|
||||
if err := rows.Scan(&id, &seq, &parent, &from, &to, &onUpdate, &onDelete, &match); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, ForeignKey{
|
||||
ChildTable: table, ChildColumn: from,
|
||||
ParentTable: parent, ParentColumn: to,
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
case DriverMySQL:
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT TABLE_NAME, COLUMN_NAME, REFERENCED_TABLE_NAME, REFERENCED_COLUMN_NAME
|
||||
FROM information_schema.KEY_COLUMN_USAGE
|
||||
WHERE table_schema = DATABASE() AND TABLE_NAME = ?
|
||||
AND REFERENCED_TABLE_NAME IS NOT NULL`, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanFKRows(rows)
|
||||
case DriverPostgres:
|
||||
rows, err := db.QueryContext(ctx, `
|
||||
SELECT tc.table_name, kcu.column_name, ccu.table_name, ccu.column_name
|
||||
FROM information_schema.table_constraints AS tc
|
||||
JOIN information_schema.key_column_usage AS kcu
|
||||
ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema
|
||||
JOIN information_schema.constraint_column_usage AS ccu
|
||||
ON ccu.constraint_name = tc.constraint_name AND ccu.table_schema = tc.table_schema
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public' AND tc.table_name = $1`, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanFKRows(rows)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported driver")
|
||||
}
|
||||
}
|
||||
|
||||
func scanFKRows(rows *sql.Rows) ([]ForeignKey, error) {
|
||||
var out []ForeignKey
|
||||
for rows.Next() {
|
||||
var ctab, ccol, ptab, pcol string
|
||||
if err := rows.Scan(&ctab, &ccol, &ptab, &pcol); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, ForeignKey{ChildTable: ctab, ChildColumn: ccol, ParentTable: ptab, ParentColumn: pcol})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
65
platform/internal/dbsync/validate_test.go
Normal file
65
platform/internal/dbsync/validate_test.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package dbsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsTextLikePK(t *testing.T) {
|
||||
good := []string{"TEXT", "VARCHAR(36)", "uuid", "CHARACTER VARYING", "NVARCHAR(64)", "char(36)"}
|
||||
for _, g := range good {
|
||||
if !isTextLikePK(g) {
|
||||
t.Fatalf("expected text-like: %s", g)
|
||||
}
|
||||
}
|
||||
bad := []string{"INTEGER", "INT", "BIGINT", "SERIAL", "BIGSERIAL", "int(11)", "NUMERIC", "DECIMAL(10,2)"}
|
||||
for _, b := range bad {
|
||||
if isTextLikePK(b) {
|
||||
t.Fatalf("expected reject: %s", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChannelConfigEmpty(t *testing.T) {
|
||||
ch := &Channel{}
|
||||
if err := ValidateChannelConfig(ch); err == nil {
|
||||
t.Fatal("expected error for empty tables")
|
||||
}
|
||||
ch.Local.Tables = []string{"orders"}
|
||||
if err := ValidateChannelConfig(ch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateChannelAgainstDBRejectsIntegerPK(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dsn := "file:" + filepath.ToSlash(filepath.Join(dir, "t.db")) + "?_pragma=foreign_keys(1)"
|
||||
db, err := Open(DriverSQLite, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err = db.Exec(`CREATE TABLE orders (id INTEGER PRIMARY KEY, title TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Exec(`CREATE TABLE orders_uuid (id TEXT PRIMARY KEY, title TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bad := &Channel{
|
||||
Local: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders"}},
|
||||
Remote: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders"}},
|
||||
}
|
||||
if err := ValidateChannelAgainstDB(context.Background(), bad); err == nil {
|
||||
t.Fatal("expected reject INTEGER PK")
|
||||
}
|
||||
|
||||
good := &Channel{
|
||||
Local: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders_uuid"}},
|
||||
Remote: Endpoint{Driver: DriverSQLite, DSN: dsn, Tables: []string{"orders_uuid"}},
|
||||
}
|
||||
if err := ValidateChannelAgainstDB(context.Background(), good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user