被代理的 WeKnora 收到 SIGTERM 会先关监听再 exit(1),在途请求被切断。 排空改到真正的连接入口:闸门停止接客、等在途连接归零、写 drain.json 并 exit 0;协调器只认这条证据,写者退出码只作记录。 顺带修掉维护占用的一条死锁:释放后的窗口此前不能被新作业接管, 等于第一次同步之后谁也再进不去;持有中的窗口仍只允许原作业重入。
This commit is contained in:
@@ -16,6 +16,13 @@
|
||||
- 迁移 `000071_library_maintenance_level`:维护记录新增 `level`(instance/library)、
|
||||
`target_fingerprint`、`services`、`ingress_state`,页面/审计能看清这次到底动的是实例还是只锁了一个库。
|
||||
|
||||
- 入口闸门(`LIBRARY_TARGET_INGRESS_SERVICE` + `INGRESS_DRAIN_FILE`):被代理的 WeKnora 收到 SIGTERM
|
||||
会先关监听再因 `use of closed network connection` 直接 exit(1),在途请求会被切断。
|
||||
因此排空放在真正的连接入口上做:闸门先停止接受新连接、等在途连接归零、写 `drain.json`
|
||||
并 exit 0;协调器只认「闸门 exit 0 + phase=drained」这一条证据,写者的退出码只作为记录。
|
||||
- 维护占用释放后可被新作业接管(此前 `ON CONFLICT ... WHERE owner_job_id = ...` 会让
|
||||
"第一次同步之后谁也再进不去");仍在持有的窗口依然只允许原作业重入。
|
||||
|
||||
### Fixed
|
||||
|
||||
- 进程被杀/授权过期不再误放行半写目标:journal 里的 `content_risk` 一旦置位,
|
||||
|
||||
@@ -73,15 +73,17 @@ func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manage
|
||||
return nil, err
|
||||
}
|
||||
return librarymaintenance.New(librarymaintenance.Config{
|
||||
Instance: config.LibraryTargetInstance,
|
||||
Project: config.LibraryTargetProject,
|
||||
Writers: config.LibraryTargetWriters,
|
||||
NonWriters: config.LibraryTargetNonWriters,
|
||||
DockerSocket: config.LibraryTargetDockerSocket,
|
||||
IngressURL: config.LibraryTargetIngressURL,
|
||||
Database: database,
|
||||
Index: config.LibraryProductionIndex,
|
||||
Bucket: config.LibraryProductionS3Bucket,
|
||||
Instance: config.LibraryTargetInstance,
|
||||
Project: config.LibraryTargetProject,
|
||||
Writers: config.LibraryTargetWriters,
|
||||
NonWriters: config.LibraryTargetNonWriters,
|
||||
DockerSocket: config.LibraryTargetDockerSocket,
|
||||
IngressURL: config.LibraryTargetIngressURL,
|
||||
IngressService: config.LibraryTargetIngressService,
|
||||
IngressDrainFile: config.LibraryTargetIngressDrainFile,
|
||||
Database: database,
|
||||
Index: config.LibraryProductionIndex,
|
||||
Bucket: config.LibraryProductionS3Bucket,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -102,7 +102,10 @@ type Config struct {
|
||||
LibraryTargetNonWriters map[string]string
|
||||
LibraryTargetDockerSocket string
|
||||
LibraryTargetIngressURL string
|
||||
LibraryTargetJournalRoot string
|
||||
// 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。
|
||||
LibraryTargetIngressService string
|
||||
LibraryTargetIngressDrainFile string
|
||||
LibraryTargetJournalRoot string
|
||||
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。
|
||||
// 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。
|
||||
LibraryAgentSocket string
|
||||
@@ -170,42 +173,44 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"),
|
||||
TestAutoReviewPolicy: valueOrDefault(getenv("TEST_AUTO_REVIEW_POLICY"), string(governance.AutoReviewInternal)),
|
||||
// 整库逻辑备份的目标主机:三项同时给出才接线(脚本目录/备份根有默认值)。
|
||||
BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")),
|
||||
BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")),
|
||||
BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"),
|
||||
BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"),
|
||||
VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")),
|
||||
VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"),
|
||||
VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")),
|
||||
LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")),
|
||||
LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")),
|
||||
LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")),
|
||||
LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")),
|
||||
LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"),
|
||||
LibraryProductionTenantID: uint64(tenantID),
|
||||
LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")),
|
||||
LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")),
|
||||
LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"),
|
||||
LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")),
|
||||
LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"),
|
||||
LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")),
|
||||
LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")),
|
||||
LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")),
|
||||
LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")),
|
||||
LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")),
|
||||
LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")),
|
||||
LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")),
|
||||
LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"),
|
||||
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
|
||||
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
|
||||
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
|
||||
LibraryEmbeddingDimension: embeddingDimension,
|
||||
LibraryCapacityBudgetBytes: capacityBudget,
|
||||
LibraryRetentionVersions: retentionVersions,
|
||||
LibraryRetentionDays: retentionDays,
|
||||
VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")),
|
||||
VersionHistoryHTTPTimeout: 15 * time.Second,
|
||||
VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200),
|
||||
BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")),
|
||||
BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")),
|
||||
BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"),
|
||||
BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"),
|
||||
VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")),
|
||||
VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"),
|
||||
VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")),
|
||||
LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")),
|
||||
LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")),
|
||||
LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")),
|
||||
LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")),
|
||||
LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"),
|
||||
LibraryProductionTenantID: uint64(tenantID),
|
||||
LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")),
|
||||
LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")),
|
||||
LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"),
|
||||
LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")),
|
||||
LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"),
|
||||
LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")),
|
||||
LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")),
|
||||
LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")),
|
||||
LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")),
|
||||
LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")),
|
||||
LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")),
|
||||
LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")),
|
||||
LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"),
|
||||
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
|
||||
LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")),
|
||||
LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")),
|
||||
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
|
||||
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
|
||||
LibraryEmbeddingDimension: embeddingDimension,
|
||||
LibraryCapacityBudgetBytes: capacityBudget,
|
||||
LibraryRetentionVersions: retentionVersions,
|
||||
LibraryRetentionDays: retentionDays,
|
||||
VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")),
|
||||
VersionHistoryHTTPTimeout: 15 * time.Second,
|
||||
VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200),
|
||||
}
|
||||
if config.DatabaseURL == "" {
|
||||
return Config{}, errors.New("DATABASE_URL is required")
|
||||
@@ -338,6 +343,12 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
if config.LibraryTargetJournalRoot == "" {
|
||||
missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT")
|
||||
}
|
||||
if config.LibraryTargetIngressService == "" {
|
||||
missing = append(missing, "LIBRARY_TARGET_INGRESS_SERVICE")
|
||||
}
|
||||
if config.LibraryTargetIngressDrainFile == "" {
|
||||
missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、"))
|
||||
}
|
||||
@@ -347,6 +358,9 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
if !filepath.IsAbs(config.LibraryTargetDockerSocket) {
|
||||
return Config{}, errors.New("LIBRARY_TARGET_DOCKER_SOCKET 必须是绝对路径")
|
||||
}
|
||||
if !filepath.IsAbs(config.LibraryTargetIngressDrainFile) {
|
||||
return Config{}, errors.New("LIBRARY_TARGET_INGRESS_DRAIN_FILE 必须是绝对路径")
|
||||
}
|
||||
if len(config.LibraryTargetNonWriters) == 0 {
|
||||
return Config{}, errors.New("LIBRARY_TARGET_NON_WRITERS 必须逐项说明项目内非写入服务")
|
||||
}
|
||||
|
||||
@@ -29,6 +29,24 @@ type Service struct {
|
||||
ExitCode *int `json:"exit_code,omitempty"`
|
||||
}
|
||||
|
||||
// GateEvidence 是"入口真的关了、在途真的排空了"的证据。
|
||||
// 只有被代理的服务自己不会优雅退出时,才知道为什么要在入口这一层排空。
|
||||
type GateEvidence struct {
|
||||
Closed bool `json:"closed"`
|
||||
Drained bool `json:"drained"`
|
||||
ActiveAtClose int `json:"active_at_close"`
|
||||
Served int `json:"served"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Gate 是目标的唯一对外入口:维护窗口先关它、等在途请求结束,再停写入者。
|
||||
type Gate interface {
|
||||
// CloseIngress 必须等到在途请求结束才返回成功;返回的不是"已发信号"而是"已排空"。
|
||||
CloseIngress(context.Context) (GateEvidence, error)
|
||||
// OpenIngress 恢复入口,并确认它真的在服务。
|
||||
OpenIngress(context.Context) error
|
||||
}
|
||||
|
||||
// Inventory must enumerate the complete, independently scoped test writer set,
|
||||
// using immutable instance IDs. Never implement it as only the currently running
|
||||
// subset. Start must wait for readiness; callers still read back the result.
|
||||
@@ -54,16 +72,21 @@ type journal struct {
|
||||
Result
|
||||
// Level 区分"实例级维护"(写入者/入口被真的关掉)与"仅治理库互斥"。
|
||||
// 没有 Level 的旧 journal 来自封存链路,按实例级处理但不冒充已验证。
|
||||
Level string `json:"level,omitempty"`
|
||||
Ingress string `json:"ingress,omitempty"`
|
||||
IngressClosed bool `json:"ingress_closed,omitempty"`
|
||||
DrainVerified bool `json:"drain_verified,omitempty"`
|
||||
ContentRisk bool `json:"content_risk,omitempty"`
|
||||
RestoreBlocked bool `json:"restore_blocked,omitempty"`
|
||||
Services []serviceRecord `json:"services"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Ingress string `json:"ingress,omitempty"`
|
||||
IngressClosed bool `json:"ingress_closed,omitempty"`
|
||||
DrainVerified bool `json:"drain_verified,omitempty"`
|
||||
ContentRisk bool `json:"content_risk,omitempty"`
|
||||
RestoreBlocked bool `json:"restore_blocked,omitempty"`
|
||||
// Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。
|
||||
// 入口恢复后证据仍保留(GateReopened=true),否则事后查不到
|
||||
// "关的时候还有几个在途请求"。
|
||||
Gate *GateEvidence `json:"gate,omitempty"`
|
||||
GateReopened bool `json:"gate_reopened,omitempty"`
|
||||
Services []serviceRecord `json:"services"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
}
|
||||
type Coordinator struct {
|
||||
root *os.Root
|
||||
@@ -75,6 +98,7 @@ type Coordinator struct {
|
||||
// 证明入口真的关了、真的回来了,而不是"停了容器就算关"。
|
||||
ingressClosed func(context.Context) error
|
||||
ingressReady func(context.Context) error
|
||||
gate Gate
|
||||
}
|
||||
|
||||
func Open(root, source string, c Controller) (*Coordinator, error) {
|
||||
@@ -116,6 +140,15 @@ func (c *Coordinator) SetTarget(level, ingress string, closed, ready func(contex
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetGate 声明目标入口闸门:实例级维护里"关入口"就是关它、并从它拿到排空证据。
|
||||
func (c *Coordinator) SetGate(g Gate) error {
|
||||
if g == nil {
|
||||
return errors.New("invalid ingress gate")
|
||||
}
|
||||
c.gate = g
|
||||
return nil
|
||||
}
|
||||
|
||||
func validIngress(raw string) bool {
|
||||
parsed, err := url.Parse(raw)
|
||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
|
||||
@@ -312,6 +345,12 @@ func (c *Coordinator) load(job string) (journal, error) {
|
||||
if j.RestoreBlocked && !j.ContentRisk {
|
||||
return journal{}, errors.New("invalid freeze restore block")
|
||||
}
|
||||
if j.Gate != nil && (!j.Gate.Closed || j.Gate.ActiveAtClose < 0 || j.Gate.Served < 0) {
|
||||
return journal{}, errors.New("invalid freeze ingress evidence")
|
||||
}
|
||||
if j.GateReopened && j.Gate == nil {
|
||||
return journal{}, errors.New("ingress reopened without close evidence")
|
||||
}
|
||||
if len(j.Services) == 0 || len(j.Services) > 256 {
|
||||
return journal{}, errors.New("invalid freeze journal services")
|
||||
}
|
||||
@@ -444,5 +483,18 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error {
|
||||
failures = errors.Join(failures, errors.New("original test service missing; replacement not started"))
|
||||
}
|
||||
}
|
||||
// 服务都回来了才开放入口:先开门再启动服务,等于把流量打向没准备好的目标。
|
||||
if j.Gate != nil {
|
||||
if c.gate == nil {
|
||||
failures = errors.Join(failures, errors.New("入口闸门缺失:不能确认目标已恢复"))
|
||||
} else if err := c.gate.OpenIngress(ctx); err != nil {
|
||||
failures = errors.Join(failures, fmt.Errorf("目标入口恢复失败:%w", err))
|
||||
} else {
|
||||
j.GateReopened = true
|
||||
if err := c.save(*j); err != nil {
|
||||
failures = errors.Join(failures, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return failures
|
||||
}
|
||||
|
||||
@@ -125,7 +125,23 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
|
||||
}
|
||||
}
|
||||
|
||||
// 停写入者 + 排空 + 关入口:任何一步不成立都不许往下写。
|
||||
// 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。
|
||||
// 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。
|
||||
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
|
||||
evidence, gateErr := c.gate.CloseIngress(ctx)
|
||||
if gateErr != nil {
|
||||
return c.abandonWindow(ctx, &j, fmt.Errorf("关闭目标入口失败:%w", gateErr), "gate_failed")
|
||||
}
|
||||
if !evidence.Closed || !evidence.Drained {
|
||||
return c.abandonWindow(ctx, &j, errors.New("目标入口未确认排空(仍有在途请求)"), "gate_not_drained")
|
||||
}
|
||||
j.Gate = &evidence
|
||||
if err = c.save(j); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// 停写入者:任何一步不成立都不许往下写。
|
||||
if stopErr := c.stopAll(ctx, &j); stopErr != nil {
|
||||
return c.abandonWindow(ctx, &j, stopErr, "stop_failed")
|
||||
}
|
||||
@@ -284,6 +300,10 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
|
||||
if needed == 0 {
|
||||
return nil
|
||||
}
|
||||
// 入口层已经证明"在途请求全部结束"时,写者的退出码只作为记录:
|
||||
// 被代理的 WeKnora 收到 SIGTERM 会自己 exit(1)(upstream bug),
|
||||
// 那不代表请求被切断——请求早在入口关闭时就已经跑完了。
|
||||
gateDrained := j.Gate != nil && j.Gate.Closed && j.Gate.Drained
|
||||
items, err := inventory(ctx, c.control)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -303,18 +323,18 @@ func (c *Coordinator) verifyDrain(ctx context.Context, j *journal) error {
|
||||
return errors.New("test writer still running")
|
||||
}
|
||||
if now.ExitCode == nil {
|
||||
if c.level == "instance" {
|
||||
if c.level == "instance" && !gateDrained {
|
||||
return fmt.Errorf("无法证明 %s 的在途请求已排空(控制器没有退出码)", s.ID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
s.ExitCode = now.ExitCode
|
||||
if *now.ExitCode != 0 {
|
||||
if *now.ExitCode != 0 && !gateDrained {
|
||||
return fmt.Errorf("写入者 %s 未优雅退出(exit=%d):在途请求没有完成排空", s.ID, *now.ExitCode)
|
||||
}
|
||||
verified++
|
||||
}
|
||||
j.DrainVerified = verified == needed
|
||||
j.DrainVerified = gateDrained || verified == needed
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -251,3 +251,64 @@ func TestLibraryLevelRejectsIngressClaim(t *testing.T) {
|
||||
t.Fatal("empty level accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeGate 记录"入口先关、写者后停、恢复时先起服务再开入口"的顺序。
|
||||
type fakeGate struct {
|
||||
calls *[]string
|
||||
drain bool
|
||||
active int
|
||||
}
|
||||
|
||||
func (g *fakeGate) CloseIngress(context.Context) (GateEvidence, error) {
|
||||
*g.calls = append(*g.calls, "gate:close")
|
||||
return GateEvidence{Closed: true, Drained: g.drain, ActiveAtClose: g.active, Served: 3}, nil
|
||||
}
|
||||
func (g *fakeGate) OpenIngress(context.Context) error {
|
||||
*g.calls = append(*g.calls, "gate:open")
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestWindowClosesGateBeforeStoppingWriters(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
e.ingressClosed = func(context.Context) error { return nil }
|
||||
gate := &fakeGate{calls: &c.calls, drain: true, active: 1}
|
||||
if err := e.SetGate(gate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := e.RunWindow(context.Background(), "job-gate", func(context.Context) error { return nil },
|
||||
func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "gate:close,stop:app,stop:store,start:app,start:store,gate:open"
|
||||
if strings.Join(c.calls, ",") != want {
|
||||
t.Fatalf("calls=%v want=%s", c.calls, want)
|
||||
}
|
||||
// 写者退出码非 0 也不影响结论:排空是在入口层证明的(被代理服务自己不优雅退出)。
|
||||
j, err := e.load("job-gate")
|
||||
if err != nil || !j.DrainVerified {
|
||||
t.Fatalf("journal=%+v err=%v", j, err)
|
||||
}
|
||||
// 入口恢复后证据仍留着:报告要能说清"关的时候还有几个在途请求"。
|
||||
if j.Gate == nil || !j.GateReopened {
|
||||
t.Fatalf("gate evidence lost after reopening: %+v reopened=%v", j.Gate, j.GateReopened)
|
||||
}
|
||||
}
|
||||
|
||||
// 入口没排空(还有在途请求)时不许停写者、更不许写目标。
|
||||
func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
gate := &fakeGate{calls: &c.calls, drain: false, active: 2}
|
||||
if err := e.SetGate(gate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
work := 0
|
||||
_, err := e.RunWindow(context.Background(), "job-gate-bad", func(context.Context) error { return nil },
|
||||
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
||||
if err == nil || work != 0 {
|
||||
t.Fatalf("err=%v work=%d", err, work)
|
||||
}
|
||||
if strings.Join(c.calls, ",") != "gate:close" {
|
||||
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package librarymaintenance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
)
|
||||
|
||||
// dockerGate 是目标入口闸门的真实实现:它只认"容器已经优雅退出"+
|
||||
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
|
||||
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
|
||||
type dockerGate struct {
|
||||
client *http.Client
|
||||
socket string
|
||||
project string
|
||||
service string
|
||||
drainFile string
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
|
||||
if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) {
|
||||
return nil, errors.New("入口闸门配置不完整")
|
||||
}
|
||||
info, err := os.Stat(socket)
|
||||
if err != nil || info.Mode()&os.ModeSocket == 0 {
|
||||
return nil, errors.New("local Docker socket unavailable")
|
||||
}
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
|
||||
}}
|
||||
return &dockerGate{
|
||||
client: &http.Client{Transport: transport, Timeout: timeout,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }},
|
||||
socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *dockerGate) request(ctx context.Context, method, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
|
||||
if err != nil {
|
||||
return errors.New("invalid docker request")
|
||||
}
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return errors.New("docker request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
|
||||
}
|
||||
if out != nil {
|
||||
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if readErr != nil || json.Unmarshal(data, out) != nil {
|
||||
return errors.New("invalid docker response")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type gateInspection struct {
|
||||
ID string `json:"Id"`
|
||||
Config struct {
|
||||
Labels map[string]string `json:"Labels"`
|
||||
} `json:"Config"`
|
||||
State struct {
|
||||
Running bool
|
||||
Status string
|
||||
ExitCode int
|
||||
Health *struct{ Status string }
|
||||
} `json:"State"`
|
||||
}
|
||||
|
||||
func (g *dockerGate) container(ctx context.Context) (gateInspection, error) {
|
||||
filters, _ := json.Marshal(map[string][]string{"label": {
|
||||
"com.docker.compose.project=" + g.project,
|
||||
"com.docker.compose.service=" + g.service,
|
||||
}})
|
||||
var listed []struct {
|
||||
ID string `json:"Id"`
|
||||
}
|
||||
if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil {
|
||||
return gateInspection{}, err
|
||||
}
|
||||
if len(listed) != 1 {
|
||||
return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed))
|
||||
}
|
||||
var data gateInspection
|
||||
if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil {
|
||||
return gateInspection{}, err
|
||||
}
|
||||
if data.Config.Labels["com.docker.compose.project"] != g.project ||
|
||||
data.Config.Labels["com.docker.compose.service"] != g.service {
|
||||
return gateInspection{}, errors.New("入口闸门身份不匹配")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type drainRecord struct {
|
||||
Phase string `json:"phase"`
|
||||
Active int `json:"active"`
|
||||
Served int `json:"served"`
|
||||
WaitedMS int64 `json:"waited_ms"`
|
||||
}
|
||||
|
||||
func (g *dockerGate) readDrain() (drainRecord, error) {
|
||||
file, err := os.Open(g.drainFile)
|
||||
if err != nil {
|
||||
return drainRecord{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
var record drainRecord
|
||||
decoder := json.NewDecoder(io.LimitReader(file, 64<<10))
|
||||
if err := decoder.Decode(&record); err != nil {
|
||||
return drainRecord{}, err
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// CloseIngress:先删掉上一次的 drain 记录(不能用旧证据冒充这次),再发 SIGTERM,
|
||||
// 等到容器自己退出,最后只认 exit 0 + phase=drained。
|
||||
func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEvidence, error) {
|
||||
bounded, cancel := context.WithTimeout(ctx, g.timeout)
|
||||
defer cancel()
|
||||
data, err := g.container(bounded)
|
||||
if err != nil {
|
||||
return libraryfreeze.GateEvidence{}, err
|
||||
}
|
||||
if data.State.Running {
|
||||
if err := os.Remove(g.drainFile); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法清理上次的排空记录:%w", err)
|
||||
}
|
||||
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
|
||||
return libraryfreeze.GateEvidence{}, err
|
||||
}
|
||||
deadline := time.Now().Add(g.timeout)
|
||||
for {
|
||||
state, err := g.container(bounded)
|
||||
if err != nil {
|
||||
return libraryfreeze.GateEvidence{}, err
|
||||
}
|
||||
if !state.State.Running {
|
||||
data = state
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return libraryfreeze.GateEvidence{}, errors.New("入口闸门没有在期限内退出(在途请求可能还没结束)")
|
||||
}
|
||||
select {
|
||||
case <-bounded.Done():
|
||||
return libraryfreeze.GateEvidence{}, errors.New("入口闸门退出等待被中断")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
record, err := g.readDrain()
|
||||
if err != nil {
|
||||
return libraryfreeze.GateEvidence{}, fmt.Errorf("读不到入口排空记录:%w", err)
|
||||
}
|
||||
if record.Phase != "drained" || data.State.ExitCode != 0 {
|
||||
return libraryfreeze.GateEvidence{}, fmt.Errorf("入口排空未确认(phase=%s exit=%d active=%d)",
|
||||
record.Phase, data.State.ExitCode, record.Active)
|
||||
}
|
||||
return libraryfreeze.GateEvidence{Closed: true, Drained: true, ActiveAtClose: record.Active,
|
||||
Served: record.Served, Detail: fmt.Sprintf("gate exit=0 phase=%s waited_ms=%d", record.Phase, record.WaitedMS)}, nil
|
||||
}
|
||||
|
||||
// OpenIngress:把闸门拉起来并等健康检查通过;能不能真的服务由入口探针再确认一次。
|
||||
func (g *dockerGate) OpenIngress(ctx context.Context) error {
|
||||
bounded, cancel := context.WithTimeout(ctx, g.timeout)
|
||||
defer cancel()
|
||||
data, err := g.container(bounded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !data.State.Running {
|
||||
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(g.timeout)
|
||||
for {
|
||||
state, err := g.container(bounded)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
healthy := state.State.Running && state.State.Health != nil && state.State.Health.Status == "healthy"
|
||||
if healthy {
|
||||
// 记下新的 serving 记录:下一次关闭要用它自己的证据。
|
||||
if _, err := g.readDrain(); err == nil {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return errors.New("入口闸门没有在期限内恢复健康")
|
||||
}
|
||||
select {
|
||||
case <-bounded.Done():
|
||||
return errors.New("入口闸门恢复等待被中断")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,10 +49,14 @@ type Config struct {
|
||||
NonWriters map[string]string
|
||||
DockerSocket string
|
||||
IngressURL string
|
||||
Database string
|
||||
Index string
|
||||
Bucket string
|
||||
Timeout time.Duration
|
||||
// IngressService 是入口闸门在 compose 项目里的服务名;DrainFile 是它写
|
||||
// 排空证据的宿主机文件。两者缺一,就只能说"停了写入者",不能说"排空了"。
|
||||
IngressService string
|
||||
IngressDrainFile string
|
||||
Database string
|
||||
Index string
|
||||
Bucket string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
@@ -63,6 +67,7 @@ type Manager struct {
|
||||
nonWriters map[string]string
|
||||
timeout time.Duration
|
||||
client *http.Client
|
||||
gate *dockerGate
|
||||
}
|
||||
|
||||
func New(cfg Config) (*Manager, error) {
|
||||
@@ -97,9 +102,17 @@ func New(cfg Config) (*Manager, error) {
|
||||
return nil, err
|
||||
}
|
||||
target.Fingerprint = fingerprint
|
||||
return &Manager{target: target, source: project, docker: cfg.DockerSocket,
|
||||
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
|
||||
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
|
||||
client: &http.Client{Timeout: 5 * time.Second}}, nil
|
||||
client: &http.Client{Timeout: 5 * time.Second}}
|
||||
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
|
||||
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manager.gate = gate
|
||||
}
|
||||
return manager, nil
|
||||
}
|
||||
|
||||
func fingerprintOf(t Target) (string, error) {
|
||||
@@ -146,6 +159,13 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
|
||||
control.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if m.gate != nil {
|
||||
if err := coordinator.SetGate(m.gate); err != nil {
|
||||
coordinator.Close()
|
||||
control.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return coordinator, control, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -72,7 +72,10 @@ ON CONFLICT (scope) DO UPDATE SET
|
||||
note = EXCLUDED.note,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
released_at = NULL
|
||||
WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id
|
||||
-- 已经释放的窗口允许被新作业接管(否则第一个同步之后谁也再进不去);
|
||||
-- 仍在持有的窗口只允许原作业重入。
|
||||
WHERE library_maintenance.released_at IS NOT NULL
|
||||
OR library_maintenance.owner_job_id = EXCLUDED.owner_job_id
|
||||
RETURNING `+maintenanceColumns,
|
||||
item.Scope, item.OwnerJobID, item.TargetIdentity, level, item.TargetFingerprint, string(raw),
|
||||
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// 维护占用的生命周期必须只有一条规则:
|
||||
// - 正在持有的窗口只允许原作业重入,别的作业一律拒绝;
|
||||
// - 已经释放的窗口必须能被新作业接管,否则"第一次同步之后谁也再进不去"。
|
||||
func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Now().UTC()
|
||||
|
||||
first, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
|
||||
Scope: "internal", OwnerJobID: "job-1", TargetIdentity: "instance=kgmaint",
|
||||
Level: "instance", TargetFingerprint: "fp", Services: []string{"app"},
|
||||
IngressState: "unknown", State: "acquired", OriginalServiceState: "app=running",
|
||||
Note: "test",
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("first acquire: %v", err)
|
||||
}
|
||||
if first.Level != "instance" || len(first.Services) != 1 || first.Services[0] != "app" {
|
||||
t.Fatalf("acquired row lost identity: %+v", first)
|
||||
}
|
||||
|
||||
// 同一作业重入(接管)必须成功。
|
||||
if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
|
||||
Scope: "internal", OwnerJobID: "job-1", State: "applying", Level: "instance",
|
||||
}, now.Add(time.Second)); err != nil {
|
||||
t.Fatalf("same owner re-entry rejected: %v", err)
|
||||
}
|
||||
// 另一个作业在持有期间必须被拒。
|
||||
if _, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
|
||||
Scope: "internal", OwnerJobID: "job-2", State: "acquired", Level: "instance",
|
||||
}, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) {
|
||||
t.Fatalf("second owner during hold: err=%v", err)
|
||||
}
|
||||
|
||||
if err := store.ReleaseLibraryMaintenance(ctx, "internal", "job-1", now.Add(2*time.Second)); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
second, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{
|
||||
Scope: "internal", OwnerJobID: "job-2", TargetIdentity: "instance=kgmaint",
|
||||
Level: "instance", TargetFingerprint: "fp2", Services: []string{"app"},
|
||||
IngressState: "unknown", State: "acquired", Note: "second",
|
||||
}, now.Add(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("acquire after release: %v", err)
|
||||
}
|
||||
if second.OwnerJobID != "job-2" || second.ReleasedAt != nil || second.TargetFingerprint != "fp2" {
|
||||
t.Fatalf("released window not re-acquired cleanly: %+v", second)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user