diff --git a/.gitignore b/.gitignore index f07adae..c7f1e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ # Local artifacts (binaries, staging scripts, backups) /release-diag /worker +/libraryctl /apply_s5_patch.py /tmp_* .style-backup-*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b9545..188729a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 开发快照 snapshot/v0.1.108 - 2026-10-28 + +### Added + +- 新增整库封存的真实业务入口 `libraryctl`(status / preflight / capture / list / verify / recover):从本机真实测试栈只读读取来源,核对处理已结清,推出兼容契约,在维护窗口内停住声明过的写入者,导出 PostgreSQL 十表、原始索引正文与向量、对象资源与 Wiki 快照,封存为内容寻址候选产物后恢复写入者并等到 healthy。 +- 新增按 Compose 项目固定身份的冻结控制器:整个项目逐服务声明,出现未声明服务或容器被重建即拒绝,只发 SIGTERM,恢复后必须 healthy。 +- 新增「整库版本」页(只读):对内/对外最新封存、分项数量与字节、摘要、兼容契约、归档占用;未接线时明确显示原因,不提供未实现的封存按钮。 +- 新增运维手册 `docs/library-capture.md`,含 2026-10-28 两次真实封存的快照 ID 与实测数字。 + +### Changed + +- api 增加 `LIBRARY_ARCHIVE_ROOT` 与整库归档根的只读挂载(写入只发生在主机上的 libraryctl)。 + +### Not included + +- 本快照不代表整库业务验收通过:内容/问答/性能验收未绑定,跨系统一致切点、Wiki 删除语义重算、生产预备份与恢复演练、双闸授权、整库同步协调与失败续作仍未完成。 + ## 开发快照 snapshot/v0.1.105 - 2026-10-28 ### Fixed diff --git a/cmd/api/main.go b/cmd/api/main.go index b890fee..9b88bd6 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -201,6 +201,7 @@ func run(ctx context.Context, getenv func(string) string) error { BackupSchedule: postgresstore.NewStore(pool), VersionHistory: versionHistoryStore, VersionHistoryRepo: versionHistoryRepo, + LibraryArchiveRoot: config.LibraryArchiveRoot, BackupHost: backuphost.New(backuphost.Config{ SSHConfig: config.BackupHostSSHConfig, Target: config.BackupHostTarget, diff --git a/cmd/libraryctl/main.go b/cmd/libraryctl/main.go new file mode 100644 index 0000000..ebd5672 --- /dev/null +++ b/cmd/libraryctl/main.go @@ -0,0 +1,705 @@ +// Command libraryctl is the operator entry point for the whole-library capture +// chain: it reads one test knowledge base (internal or external) from the real +// WeKnora test stack, freezes its writers for the duration of the export and +// seals a content-addressed candidate artifact. +// +// Boundaries this command keeps: +// - the source is read through a read-only PostgreSQL session; no source +// schema, data or program is modified; +// - only SIGTERM is sent to the declared writer group, and only when every +// service of the Compose project is declared (a new sibling container is +// refused, not ignored); +// - credentials are read from the running containers at the moment of the +// call and are never printed, logged or written to disk; +// - the sealed artifact is a *candidate*: it is not a release, and it does +// not replace snapshot-bound content, Q&A and performance acceptance. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/librarysnapshot" + v071 "knowledge-governance/internal/releaseadapter/v071" + releaseworkflow "knowledge-governance/internal/workflows/release" +) + +type options struct { + sourceApp string + sourcePG string + sourceES string + governancePG string + project string + writers string + nonWriters string + archiveRoot string + journalRoot string + dockerSocket string + maxDocuments int + maxObjectBytes int64 + scope string + job string + dryRun bool + snapshot string +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + command := os.Args[1] + opts, rest, err := parseFlags(command, os.Args[2:]) + if err != nil { + fmt.Fprintln(os.Stderr, "LIBRARYCTL_FLAGS:", err) + os.Exit(2) + } + _ = rest + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + var runErr error + switch command { + case "status": + runErr = runStatus(ctx, opts) + case "preflight": + runErr = runPreflight(ctx, opts) + case "capture": + runErr = runCapture(ctx, opts) + case "list": + runErr = runList(opts) + case "verify": + runErr = runVerify(opts) + case "recover": + runErr = runRecover(ctx, opts) + default: + usage() + os.Exit(2) + } + if runErr != nil { + fmt.Fprintln(os.Stderr, "LIBRARYCTL_FAILED:", runErr) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `libraryctl [flags] + + status probe the real source and print what is configured / blocked + preflight read the source, run settlement + candidate validation (no writes, no stop) + capture freeze the declared writer group, archive a candidate, restore writers + list list sealed candidates and stored bytes + verify re-verify one sealed candidate (digests, coverage, identity) + recover restore writers from an interrupted freeze journal + +Flags (defaults target the local WKNFull test stack): + -source-app WKNFull-app container whose env describes the source + -source-pg WKNFull-postgres source PostgreSQL container (address resolution) + -source-es WKNFull-elasticsearch + -governance-pg knowledge-governance-p1-postgres-1 routing bindings read here + -project weknora071full Compose project owning the writer group + -writers WKNFull-app declared writer services (comma separated) + -non-writers WKNFull-frontend=reason,... every other project service + -archive ~/.local/share/knowledge-governance/library + -journal ~/.local/share/knowledge-governance/freeze + -docker-socket /var/run/docker.sock + -scope internal|external (preflight/capture) + -job (capture; default: lib--) + -dry-run (capture: export + validate only, no archive, no stop) + -snapshot (verify) +`) +} + +func parseFlags(command string, args []string) (options, []string, error) { + home, _ := os.UserHomeDir() + opts := options{ + sourceApp: "WKNFull-app", + sourcePG: "WKNFull-postgres", + sourceES: "WKNFull-elasticsearch", + governancePG: "knowledge-governance-p1-postgres-1", + project: "weknora071full", + writers: "WKNFull-app", + archiveRoot: filepath.Join(home, ".local/share/knowledge-governance/library"), + journalRoot: filepath.Join(home, ".local/share/knowledge-governance/freeze"), + dockerSocket: "/var/run/docker.sock", + maxDocuments: 100000, + maxObjectBytes: 134217728, + } + set := flag.NewFlagSet(command, flag.ContinueOnError) + set.SetOutput(os.Stderr) + set.StringVar(&opts.sourceApp, "source-app", opts.sourceApp, "source container describing the stack") + set.StringVar(&opts.sourcePG, "source-pg", opts.sourcePG, "source PostgreSQL container") + set.StringVar(&opts.sourceES, "source-es", opts.sourceES, "source Elasticsearch container") + set.StringVar(&opts.governancePG, "governance-pg", opts.governancePG, "governance PostgreSQL container") + set.StringVar(&opts.project, "project", opts.project, "Compose project owning the writers") + set.StringVar(&opts.writers, "writers", opts.writers, "declared writer services") + set.StringVar(&opts.nonWriters, "non-writers", "", "service=reason for every other project service") + set.StringVar(&opts.archiveRoot, "archive", opts.archiveRoot, "sealed artifact root") + set.StringVar(&opts.journalRoot, "journal", opts.journalRoot, "freeze journal root") + set.StringVar(&opts.dockerSocket, "docker-socket", opts.dockerSocket, "Docker engine socket") + set.IntVar(&opts.maxDocuments, "max-documents", opts.maxDocuments, "maximum indexed documents per scope") + set.Int64Var(&opts.maxObjectBytes, "max-object-bytes", opts.maxObjectBytes, "maximum bytes per stored object") + set.StringVar(&opts.scope, "scope", "", "internal|external") + set.StringVar(&opts.job, "job", "", "capture job id") + set.BoolVar(&opts.dryRun, "dry-run", false, "export and validate without archiving or stopping") + set.StringVar(&opts.snapshot, "snapshot", "", "snapshot id for verify") + if err := set.Parse(args); err != nil { + return opts, nil, err + } + return opts, set.Args(), nil +} + +// ---------------------------------------------------------------- discovery + +func dockerOutput(ctx context.Context, args ...string) (string, error) { + bounded, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + out, err := exec.CommandContext(bounded, "docker", args...).Output() + if err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) && len(exit.Stderr) > 0 { + return "", fmt.Errorf("docker %s failed", args[0]) + } + return "", fmt.Errorf("docker %s failed", args[0]) + } + return strings.TrimSpace(string(out)), nil +} + +func containerEnv(ctx context.Context, name string) (map[string]string, error) { + raw, err := dockerOutput(ctx, "inspect", "--format", "{{json .Config.Env}}", name) + if err != nil { + return nil, err + } + var entries []string + if err := json.Unmarshal([]byte(raw), &entries); err != nil { + return nil, errors.New("invalid container environment") + } + env := map[string]string{} + for _, entry := range entries { + if key, value, ok := strings.Cut(entry, "="); ok { + env[key] = value + } + } + return env, nil +} + +func containerAddress(ctx context.Context, name string) (string, error) { + raw, err := dockerOutput(ctx, "inspect", "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", name) + if err != nil { + return "", err + } + for _, field := range strings.Fields(raw) { + if field != "" && field != "0.0.0.0" { + return field, nil + } + } + return "", fmt.Errorf("container %s has no reachable address", name) +} + +func containerImageID(ctx context.Context, name string) (string, error) { + return dockerOutput(ctx, "inspect", "--format", "{{.Image}}", name) +} + +// sourceConfig is everything needed to read the test library. Credential fields +// are never printed: String() masks them and no log line formats the struct. +type sourceConfig struct { + Scopes map[string]string + DatabaseURL string + Search v071.SearchExporterConfig + Object v071.S3ObjectStoreConfig + AppImageID string + SearchIndex string + ObjectDetail string +} + +func discoverSource(ctx context.Context, opts options) (sourceConfig, error) { + var config sourceConfig + binding, err := readRoutingBindings(ctx, opts.governancePG) + if err != nil { + return config, err + } + config.Scopes = binding + app, err := containerEnv(ctx, opts.sourceApp) + if err != nil { + return config, err + } + if app["DB_DRIVER"] != "postgres" { + return config, errors.New("source database driver is not postgres") + } + if app["STORAGE_TYPE"] != "oss" { + return config, fmt.Errorf("unsupported source storage type %q", app["STORAGE_TYPE"]) + } + address, err := containerAddress(ctx, opts.sourcePG) + if err != nil { + return config, err + } + dsn := strings.Join([]string{ + "postgres://" + urlEscape(app["DB_USER"]) + ":" + urlEscape(app["DB_PASSWORD"]) + + "@" + address + ":" + app["DB_PORT"] + "/" + urlEscape(app["DB_NAME"]) + "?sslmode=disable", + }, "") + config.DatabaseURL = dsn + config.AppImageID, err = containerImageID(ctx, opts.sourceApp) + if err != nil { + return config, err + } + esAddress, err := containerAddress(ctx, opts.sourceES) + if err != nil { + return config, err + } + config.SearchIndex = app["ELASTICSEARCH_INDEX"] + config.Search = v071.SearchExporterConfig{BaseURL: "http://" + esAddress + ":9200", Index: app["ELASTICSEARCH_INDEX"], + AuthorizationToken: app["ELASTICSEARCH_PASSWORD"], PageSize: 500} + endpoint := app["OSS_ENDPOINT"] + if endpoint == "" { + return config, errors.New("source object endpoint is not configured") + } + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + config.Object = v071.S3ObjectStoreConfig{Provider: "oss", Endpoint: endpoint, Region: app["OSS_REGION"], + AccessKeyID: app["OSS_ACCESS_KEY"], SecretAccessKey: app["OSS_SECRET_KEY"], Bucket: app["OSS_BUCKET_NAME"], + Secure: strings.HasPrefix(endpoint, "https://"), MaxObjectBytes: opts.maxObjectBytes} + config.ObjectDetail = "bucket=" + app["OSS_BUCKET_NAME"] + " region=" + app["OSS_REGION"] + return config, nil +} + +func urlEscape(value string) string { + replacer := strings.NewReplacer("%", "%25", ":", "%3A", "@", "%40", "/", "%2F", "?", "%3F", "#", "%23", "&", "%26", " ", "%20") + return replacer.Replace(value) +} + +func readRoutingBindings(ctx context.Context, container string) (map[string]string, error) { + raw, err := dockerOutput(ctx, "exec", container, "psql", "-U", "governance", "-d", "governance", "-tAc", + "SELECT internal_kb_id||'|'||external_kb_id FROM routing_kb_bindings WHERE id='active'") + if err != nil { + return nil, errors.New("read routing bindings failed") + } + internal, external, ok := strings.Cut(strings.TrimSpace(raw), "|") + if !ok || internal == "" || external == "" { + return nil, errors.New("routing bindings are incomplete") + } + return map[string]string{"internal": internal, "external": external}, nil +} + +func openSourceDatabase(ctx context.Context, config sourceConfig) (*pgxpool.Pool, error) { + parsed, err := pgxpool.ParseConfig(config.DatabaseURL) + if err != nil { + return nil, errors.New("invalid source database url") + } + // The capture chain never writes to the source. The read-only session makes + // that a database-enforced property rather than a convention. + parsed.ConnConfig.RuntimeParams["default_transaction_read_only"] = "on" + parsed.ConnConfig.RuntimeParams["statement_timeout"] = "120000" + pool, err := pgxpool.NewWithConfig(ctx, parsed) + if err != nil { + return nil, errors.New("open source database failed") + } + var readOnly string + if err = pool.QueryRow(ctx, "SHOW default_transaction_read_only").Scan(&readOnly); err != nil || readOnly != "on" { + pool.Close() + return nil, errors.New("source session is not read-only") + } + return pool, nil +} + +func tenantAndKnowledgeBase(ctx context.Context, pool *pgxpool.Pool, knowledgeBaseID string) (uint64, error) { + var tenant uint64 + err := pool.QueryRow(ctx, "SELECT tenant_id FROM knowledge_bases WHERE id=$1 AND deleted_at IS NULL", knowledgeBaseID).Scan(&tenant) + if err != nil || tenant == 0 { + return 0, errors.New("knowledge base not found in source") + } + return tenant, nil +} + +func writerDeclaration(opts options) ([]string, map[string]string, error) { + writers := []string{} + for _, name := range strings.Split(opts.writers, ",") { + if trimmed := strings.TrimSpace(name); trimmed != "" { + writers = append(writers, trimmed) + } + } + if len(writers) == 0 { + return nil, nil, errors.New("no writer service declared") + } + nonWriters := map[string]string{} + for _, entry := range strings.Split(opts.nonWriters, ",") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + name, reason, ok := strings.Cut(entry, "=") + if !ok || strings.TrimSpace(reason) == "" { + return nil, nil, fmt.Errorf("non-writer %q needs service=reason", entry) + } + nonWriters[strings.TrimSpace(name)] = strings.TrimSpace(reason) + } + return writers, nonWriters, nil +} + +func openFreeze(ctx context.Context, opts options) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, error) { + writers, nonWriters, err := writerDeclaration(opts) + if err != nil { + return nil, nil, err + } + if err = os.MkdirAll(opts.journalRoot, 0700); err != nil { + return nil, nil, err + } + if err = os.Chmod(opts.journalRoot, 0700); err != nil { + return nil, nil, err + } + control, err := libraryfreeze.NewProjectController(opts.dockerSocket, opts.project, writers, nonWriters, 2*time.Minute) + if err != nil { + return nil, nil, err + } + coordinator, err := libraryfreeze.Open(opts.journalRoot, opts.project, control) + if err != nil { + control.Close() + return nil, nil, err + } + return coordinator, control, nil +} + +func openArchive(opts options) (*librarysnapshot.Archive, error) { + if err := os.MkdirAll(opts.archiveRoot, 0700); err != nil { + return nil, err + } + if err := os.Chmod(opts.archiveRoot, 0700); err != nil { + return nil, err + } + return librarysnapshot.Open(opts.archiveRoot, 256*1024*1024) +} + +// ---------------------------------------------------------------- commands + +func runStatus(ctx context.Context, opts options) error { + report := map[string]any{} + config, err := discoverSource(ctx, opts) + if err != nil { + report["source_discovery"] = err.Error() + } else { + report["source"] = map[string]any{ + "app_image": config.AppImageID, + "scopes": config.Scopes, + "search_index": config.SearchIndex, + "elasticsearch": config.Search.BaseURL, + "object_store": config.ObjectDetail, + "database": "postgres (read-only session, credentials withheld)", + "credentials_logged": false, + } + pool, poolErr := openSourceDatabase(ctx, config) + if poolErr != nil { + report["database"] = poolErr.Error() + } else { + defer pool.Close() + settled := "settled" + if settleErr := v071.LibrarySourceSettled(ctx, pool); settleErr != nil { + settled = settleErr.Error() + } + report["settlement"] = settled + counts := map[string]int{} + for _, scope := range []string{"internal", "external"} { + kb := config.Scopes[scope] + tenant, tenantErr := tenantAndKnowledgeBase(ctx, pool, kb) + if tenantErr != nil { + counts[scope+"_error"] = -1 + continue + } + pg, exportErr := v071.ExportLibrary(ctx, pool, v071.PostgresIdentity{TenantID: tenant, KnowledgeBaseID: kb}) + if exportErr != nil { + counts[scope+"_error"] = -1 + continue + } + for _, table := range pg.Tables { + if table.Name == "chunks" || table.Name == "knowledges" { + counts[scope+"_"+table.Name] = len(table.Rows) + } + } + _ = kb + } + report["postgres_rows"] = counts + } + } + archive, err := openArchive(opts) + if err != nil { + report["archive"] = err.Error() + } else { + defer archive.Close() + usage, usageErr := archive.Usage() + if usageErr != nil { + report["archive"] = usageErr.Error() + } else { + total := int64(0) + for _, item := range usage { + total += item.UniqueBytes + } + report["archive"] = map[string]any{"root": opts.archiveRoot, "snapshots": len(usage), "stored_bytes": total} + } + } + if coordinator, control, freezeErr := openFreeze(ctx, opts); freezeErr != nil { + report["freeze"] = freezeErr.Error() + } else { + defer coordinator.Close() + defer control.Close() + items, inventoryErr := control.Inventory(ctx) + if inventoryErr != nil { + report["freeze"] = inventoryErr.Error() + } else { + group := []map[string]any{} + for _, item := range items { + group = append(group, map[string]any{"id": item.ID, "running": item.Running, "healthy": item.Healthy}) + } + report["freeze"] = map[string]any{"project": opts.project, "writers": group, "journal": opts.journalRoot} + } + } + printJSON(report) + return nil +} + +func runPreflight(ctx context.Context, opts options) error { + if opts.scope != "internal" && opts.scope != "external" { + return errors.New("-scope must be internal or external") + } + config, err := discoverSource(ctx, opts) + if err != nil { + return err + } + pool, err := openSourceDatabase(ctx, config) + if err != nil { + return err + } + defer pool.Close() + kb := config.Scopes[opts.scope] + tenant, err := tenantAndKnowledgeBase(ctx, pool, kb) + if err != nil { + return err + } + if err = v071.LibrarySourceSettled(ctx, pool); err != nil { + return err + } + search, err := v071.NewSearchExporter(config.Search) + if err != nil { + return err + } + contract, err := compatibilityContract(ctx, pool, search, config.SearchIndex, tenant, kb) + if err != nil { + return err + } + pg, err := v071.ExportLibrary(ctx, pool, v071.PostgresIdentity{TenantID: tenant, KnowledgeBaseID: kb}) + if err != nil { + return err + } + es, err := search.ExportLibrary(ctx, v071.SearchIdentity{TenantID: tenant, KnowledgeBaseID: kb}, opts.maxDocuments) + if err != nil { + return err + } + validation := "candidate-valid" + if err = v071.ValidateLibraryCandidateForPreflight(v071.PostgresIdentity{TenantID: tenant, KnowledgeBaseID: kb}, pg, es); err != nil { + validation = err.Error() + } + tables := map[string]int{} + for _, table := range pg.Tables { + tables[table.Name] = len(table.Rows) + } + printJSON(map[string]any{"scope": opts.scope, "knowledge_base_id": kb, "tenant_id": tenant, + "compatibility": contract, "postgres_rows": tables, "index_documents": len(es.Documents), + "mapping_sha256": es.MappingSHA256, "validation": validation, "writes": 0, "writers_stopped": 0}) + if validation != "candidate-valid" { + return errors.New("candidate validation failed: " + validation) + } + return nil +} + +func compatibilityContract(ctx context.Context, pool *pgxpool.Pool, search *v071.SearchExporter, index string, tenant uint64, kb string) (v071.LibrarySnapshotCompatibility, error) { + inspector, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: pool, Mapping: search, + KnowledgeBaseID: kb, TenantID: tenant, IndexName: index, ReadTestedBase: true}) + if err != nil { + return v071.LibrarySnapshotCompatibility{}, err + } + contract, err := inspector.Contract(ctx, releaseworkflow.PlanItem{TestKnowledgeBaseID: kb}) + if err != nil { + return v071.LibrarySnapshotCompatibility{}, err + } + return v071.LibrarySnapshotCompatibility{WeKnoraVersion: contract.WeKnoraVersion, SchemaVersion: contract.MigrationVersion, + RetrieverEngine: contract.RetrieverEngine, EmbeddingModel: contract.EmbeddingModel, + EmbeddingDimension: contract.EmbeddingDimension, IndexMappingSHA256: contract.ElasticsearchMappingSHA256}, nil +} + +func runCapture(ctx context.Context, opts options) error { + if opts.scope != "internal" && opts.scope != "external" { + return errors.New("-scope must be internal or external") + } + if opts.dryRun { + return runPreflight(ctx, opts) + } + job := opts.job + if job == "" { + job = "lib-" + opts.scope + "-" + time.Now().UTC().Format("20060102T150405") + } + config, err := discoverSource(ctx, opts) + if err != nil { + return err + } + pool, err := openSourceDatabase(ctx, config) + if err != nil { + return err + } + defer pool.Close() + kb := config.Scopes[opts.scope] + tenant, err := tenantAndKnowledgeBase(ctx, pool, kb) + if err != nil { + return err + } + search, err := v071.NewSearchExporter(config.Search) + if err != nil { + return err + } + objects, err := v071.NewS3ObjectStore(config.Object) + if err != nil { + return err + } + contract, err := compatibilityContract(ctx, pool, search, config.SearchIndex, tenant, kb) + if err != nil { + return err + } + archive, err := openArchive(opts) + if err != nil { + return err + } + defer archive.Close() + coordinator, control, err := openFreeze(ctx, opts) + if err != nil { + return err + } + defer coordinator.Close() + defer control.Close() + if err = coordinator.Recover(ctx); err != nil { + return fmt.Errorf("pending freeze requires recovery: %w", err) + } + // sourceReady proves the *group* is complete and consistent. Recorded work + // settlement is enforced separately by CaptureLibrary; neither check can see + // unrecorded or remote work. + sourceReady := func(ctx context.Context) error { + items, err := control.Inventory(ctx) + if err != nil { + return err + } + if len(items) == 0 { + return errors.New("writer group is empty") + } + return nil + } + result, err := v071.CaptureLibrary(ctx, coordinator, archive, pool, search, objects, sourceReady, + v071.LibraryCaptureRequest{JobID: job, Scope: opts.scope, + Identity: v071.PostgresIdentity{TenantID: tenant, KnowledgeBaseID: kb}, + Compatibility: contract, CreatedAt: time.Now().UTC(), MaxDocuments: opts.maxDocuments}) + if err != nil { + printJSON(map[string]any{"job": job, "state": result.State, "snapshot_id": result.SnapshotID, "error": err.Error()}) + return err + } + manifest, err := archive.Load(result.SnapshotID) + if err != nil { + return err + } + parts := make([]map[string]any, 0, len(manifest.Parts)) + var total int64 + for _, part := range manifest.Parts { + parts = append(parts, map[string]any{"kind": part.Kind, "name": part.Name, "count": part.Count, "bytes": part.Bytes, "sha256": part.SHA256}) + total += part.Bytes + } + sort.Slice(parts, func(i, j int) bool { + return fmt.Sprint(parts[i]["kind"])+fmt.Sprint(parts[i]["name"]) < fmt.Sprint(parts[j]["kind"])+fmt.Sprint(parts[j]["name"]) + }) + printJSON(map[string]any{"job": job, "state": result.State, "snapshot_id": result.SnapshotID, + "scope": manifest.Scope, "knowledge_base_id": manifest.KnowledgeBaseID, "tenant_id": manifest.TenantID, + "compatibility": manifest.Compatibility, "parts": parts, "logical_bytes": total, + "validation_policy": manifest.ValidationPolicy, "writers_restored": true}) + return nil +} + +func runList(opts options) error { + archive, err := openArchive(opts) + if err != nil { + return err + } + defer archive.Close() + usage, err := archive.Usage() + if err != nil { + return err + } + items := make([]map[string]any, 0, len(usage)) + for _, item := range usage { + manifest, err := archive.Load(item.VersionID) + if err != nil { + return err + } + items = append(items, map[string]any{"snapshot_id": item.VersionID, "scope": manifest.Scope, + "knowledge_base_id": manifest.KnowledgeBaseID, "created_at": manifest.CreatedAt, + "logical_bytes": item.LogicalBytes, "unique_bytes": item.UniqueBytes, "shared_bytes": item.SharedBytes, + "parts": len(manifest.Parts), "policy": manifest.ValidationPolicy}) + } + printJSON(map[string]any{"archive": opts.archiveRoot, "snapshots": items}) + return nil +} + +func runVerify(opts options) error { + if opts.snapshot == "" { + return errors.New("-snapshot is required") + } + archive, err := openArchive(opts) + if err != nil { + return err + } + defer archive.Close() + manifest, err := archive.Load(opts.snapshot) + if err != nil { + return err + } + parts := make([]map[string]any, 0, len(manifest.Parts)) + for _, part := range manifest.Parts { + parts = append(parts, map[string]any{"kind": part.Kind, "name": part.Name, "count": part.Count, "bytes": part.Bytes, "sha256": part.SHA256}) + } + printJSON(map[string]any{"snapshot_id": opts.snapshot, "verified": true, "format": manifest.Format, + "policy": manifest.ValidationPolicy, "scope": manifest.Scope, "knowledge_base_id": manifest.KnowledgeBaseID, + "tenant_id": manifest.TenantID, "created_at": manifest.CreatedAt, "compatibility": manifest.Compatibility, "parts": parts}) + return nil +} + +func runRecover(ctx context.Context, opts options) error { + coordinator, control, err := openFreeze(ctx, opts) + if err != nil { + return err + } + defer coordinator.Close() + defer control.Close() + if err = coordinator.Recover(ctx); err != nil { + return err + } + items, err := control.Inventory(ctx) + if err != nil { + return err + } + group := []map[string]any{} + for _, item := range items { + group = append(group, map[string]any{"id": item.ID, "running": item.Running, "healthy": item.Healthy}) + } + printJSON(map[string]any{"recovered": true, "writers": group}) + return nil +} + +func printJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + _ = encoder.Encode(value) +} diff --git a/compose.yaml b/compose.yaml index 11595ea..d658536 100644 --- a/compose.yaml +++ b/compose.yaml @@ -49,6 +49,9 @@ services: VERSION_HISTORY_LINK_ORIGIN: "${VERSION_HISTORY_LINK_ORIGIN:-}" VERSION_HISTORY_HTTP_TIMEOUT: "${VERSION_HISTORY_HTTP_TIMEOUT:-15s}" VERSION_HISTORY_MAX_PAGES: "${VERSION_HISTORY_MAX_PAGES:-200}" + # 「整库版本」页:只读已封存的整库候选产物。归档根由 `libraryctl capture` + # 在主机上写入(私有 0700),这里只挂只读;留空 = 未接线(页面降级展示原因)。 + LIBRARY_ARCHIVE_ROOT: "${LIBRARY_ARCHIVE_ROOT:-/var/lib/knowledge-governance/library}" # Mount this independent master-key file read-only using the deployment overlay. QA_AI_MASTER_KEY_FILE: "${QA_AI_MASTER_KEY_FILE:-}" QA_AI_DNS_OVER_TLS: "${QA_AI_DNS_OVER_TLS:-}" @@ -73,6 +76,8 @@ services: - pgpass - test_weknora_api_key volumes: + # 整库封存产物:只读挂载,api 只读不写(写入只发生在主机上的 libraryctl)。 + - /home/zheromiku/.local/share/knowledge-governance/library:/var/lib/knowledge-governance/library:ro # 与 worker 共用同一份 dws 状态:refresh token 是轮换制,绝不能在两处各留一份。 - /home/zheromiku/.dws:/data/dws # dws 的文件型 keychain 必须有 $HOME/.local/share/dws-cli,故挂到 DWS_HOME 下。 diff --git a/docs/library-capture.md b/docs/library-capture.md new file mode 100644 index 0000000..8a6d6f7 --- /dev/null +++ b/docs/library-capture.md @@ -0,0 +1,69 @@ +# 整库封存(libraryctl)运维手册 + +本文只讲**测试库**的整库封存链:怎么跑、它证明了什么、它**没有**证明什么。生产发布与恢复、清理不在本文范围。 + +## 它做什么 + +`libraryctl capture -scope internal|external`: + +1. 从**本机真实测试栈**(默认 Compose 项目 `weknora071full`)读来源配置:容器环境里的数据库/检索/对象存储参数,运行时读取,不落盘、不回显。 +2. 用**只读会话**(`default_transaction_read_only=on`)连来源 PostgreSQL;连接串只在进程内存里。 +3. 核对来源已结清:所有文档 `parse_status=completed`、无待处理操作、无死信(任一不满足就拒绝,不看其他 KB 的局部状态)。 +4. 从持久数据推出兼容契约:迁移版本、dirty、租户、Embedding 模型与维度、检索引擎、索引名、索引映射摘要(`text-embedding-v4` / 1024 维 / schema 74)。 +5. 在**维护窗口**内停住声明过的写入者(只对声明为 writer 的服务发 SIGTERM,等它真的停下),导出 PostgreSQL 十表 + Elasticsearch 正文与向量 + 对象资源,做双向覆盖与摘要核验,封存为内容寻址产物。 +6. 无论成败都恢复写入者并等到 healthy;崩溃留下的日志用 `libraryctl recover` 收尾。 +7. 打印快照 ID、分项(kind/name/数量/字节/SHA256)与兼容契约。 + +## 写入者声明(安全边界) + +`scripts/library-capture.sh` 里逐项声明了项目内**每一个**服务:一个 writer(`app`),其余每个都写了"为什么它不会写知识"。`ProjectController` 的规则: + +- 整个 Compose 项目被枚举(含停止的),只按声明停机; +- 项目里出现未声明的服务 → **拒绝执行**,不会"只停已知的那几个"; +- 首次清点后固定容器 ID,容器被重建(同名新实例)→ 拒绝,避免停到一个没被声明的实例; +- 只发 SIGTERM,没有会在恢复后才触发的 SIGKILL 定时器; +- 启动后必须等到 healthy 才算恢复。 + +## 命令 + +```bash +# 接线与来源探针(只读,不停服务) +./scripts/library-capture.sh status +# 预检:读来源 + 结清检查 + 候选校验,不写盘、不停服务 +./scripts/library-capture.sh preflight -scope internal +# 真实封存(停 app → 导出 → 恢复 app) +./scripts/library-capture.sh capture -scope internal +./scripts/library-capture.sh capture -scope external +# 已封存产物 +./scripts/library-capture.sh list +./scripts/library-capture.sh verify -snapshot +# 崩溃后的收尾(恢复写入者) +./scripts/library-capture.sh recover +``` + +产物默认写到 `~/.local/share/knowledge-governance/library`(0700),日志写到 `~/.local/share/knowledge-governance/freeze`(0700)。api 容器只以只读方式挂载前者,「整库版本」页据此展示;页面**没有**封存按钮,因为封存要维护窗口与来源凭据,只能在目标主机执行。 + +## 2026-10-28 实测(本机测试栈) + +| 项 | 对内 | 对外 | +| --- | --- | --- | +| 快照 | `f992d64ea45bf737a0721a465b5aef2b150ef7757157eb84f3aaf8b6972d98f9` | `01343b0b310b3977c2fdda05f5b8a6272148c4f7bfa124a6c5944123cf0c51c3` | +| 文档 / 分块 / 索引 | 4 / 38 / 122 | 0 / 0 / 0 | +| 资源 / Wiki 页 | 21 / 31 | 0 / 1 | +| 分项数 / 逻辑字节 | 32 / 55,309,044 | 13 / 1,121 | +| 策略 | artifact-acceptance-v1 | artifact-acceptance-v1 | + +两次封存都记录了 `stop_verified=true`、`restored=true`,`WKNFull-app` 恢复后 healthy;来源未做任何写操作。 + +## 它证明了什么 / 没有证明什么 + +已证明:这一份整库内容在**写入者停住**之后被完整导出,PostgreSQL 与 Elasticsearch 双向覆盖、生成问题、资源与摘要逐项核验通过,产物可寻址、可复读、可校验。 + +没有证明(不要把候选当版本): + +- **内容/问答/性能验收**没有做,也没有把结果绑定到快照; +- **跨 PG / ES / 资源的一致切点**只是在维护窗口里取得,没有独立的切点证据; +- **Wiki 删除贡献的语义重算**没有做;Wiki 队列空不等于语义正确; +- 对外库当前是空的(0 文档 1 Wiki 页),这份快照只说明"它当时确实是空的"; +- 生产备份/恢复、双闸授权、整库同步与失败续作都没有接线,本文不涉及; +- 上游父子分块漏正文索引的风险仍在:本次候选通过是因为库里现有数据已补齐,**以后再重解析**仍可能漏项,届时封存会因覆盖核验失败而阻断(这是设计如此)。 \ No newline at end of file diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index 8a72748..e40a20c 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -69,6 +69,9 @@ type Config struct { VersionHistoryAllowedOrigins []string // VersionHistoryLinkOrigin 是**页面链接**用的 origin(给内网外的浏览器点开), // 留空即与抓取 origin 相同。只用于拼展示链接,不参与任何请求。 + // LibraryArchiveRoot 是整库封存产物的只读归档根(主机私有 0700 目录的只读挂载)。 + // 留空即"未接线":页面只显示原因,不显示空表冒充没有快照。 + LibraryArchiveRoot string VersionHistoryLinkOrigin string VersionHistoryHTTPTimeout time.Duration VersionHistoryMaxPages int @@ -116,6 +119,7 @@ func Load(getenv func(string) string) (Config, error) { VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")), VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"), VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")), + LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")), VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), VersionHistoryHTTPTimeout: 15 * time.Second, VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), @@ -226,6 +230,9 @@ func Load(getenv func(string) string) (Config, error) { } } } + if config.LibraryArchiveRoot != "" && !filepath.IsAbs(config.LibraryArchiveRoot) { + return Config{}, errors.New("LIBRARY_ARCHIVE_ROOT 必须是绝对路径(归档根按只读目录挂载)") + } if config.TestWeKnoraHTTPTimeout, err = durationOrDefault(getenv("TEST_WEKNORA_HTTP_TIMEOUT"), config.TestWeKnoraHTTPTimeout); err != nil { return Config{}, fmt.Errorf("TEST_WEKNORA_HTTP_TIMEOUT: %w", err) } diff --git a/internal/httpapi/library_snapshots.go b/internal/httpapi/library_snapshots.go new file mode 100644 index 0000000..50d1c58 --- /dev/null +++ b/internal/httpapi/library_snapshots.go @@ -0,0 +1,190 @@ +package httpapi + +import ( + "errors" + "net/http" + "sort" + "strings" + "time" + + "knowledge-governance/internal/librarysnapshot" +) + +// 「整库版本」只读视图:把已经封存的候选产物(artifact-acceptance-v1)如实列出来。 +// +// 边界(刻意不做的事): +// - 不触发封存、不停/起写入者:封存需要维护窗口与来源凭据,只能由运维命令 +// scripts/library-capture.sh 在目标主机执行; +// - 不把候选当版本:候选只证明"这一份内容被完整、可核验地封存了",内容/问答/ +// 性能验收与生产发布是另外的、需要独立授权的事; +// - 归档根没接线时返回可用性=false 与原因,不返回空表冒充"没有快照"。 +type librarySnapshotPart struct { + Kind string `json:"kind"` + Name string `json:"name"` + SHA256 string `json:"sha256"` + Count int64 `json:"count"` + Bytes int64 `json:"bytes"` +} + +type librarySnapshotView struct { + SnapshotID string `json:"snapshot_id"` + Scope string `json:"scope"` + KnowledgeBaseID string `json:"knowledge_base_id"` + TenantID uint64 `json:"tenant_id"` + CreatedAt time.Time `json:"created_at"` + ValidationPolicy string `json:"validation_policy"` + Format string `json:"format"` + Compatibility librarysnapshot.Compatibility `json:"compatibility"` + Parts []librarySnapshotPart `json:"parts"` + LogicalBytes int64 `json:"logical_bytes"` + UniqueBytes int64 `json:"unique_bytes"` + SharedBytes int64 `json:"shared_bytes"` + Totals map[string]int64 `json:"totals"` +} + +type librarySnapshotList struct { + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + ArchiveRoot string `json:"archive_root,omitempty"` + Snapshots []librarySnapshotView `json:"snapshots"` + StoredBytes int64 `json:"stored_bytes"` + // 页面必须能说清"封存由谁执行",不能让人以为页面上有个没实现的按钮。 + CaptureNote string `json:"capture_note"` +} + +const libraryCaptureNote = "封存由运维命令 scripts/library-capture.sh capture -scope internal|external 在目标主机执行(停写入者→导出→恢复)。页面只读展示已封存的候选;候选不等于验收通过或可发布。" + +func librarySnapshotsList(w http.ResponseWriter, r *http.Request, config Config) { + view, err := readLibrarySnapshots(config.LibraryArchiveRoot) + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, librarySnapshotList{Available: false, + Reason: "归档不可读:" + err.Error(), Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote}) + return + } + writeJSON(w, http.StatusOK, view) +} + +func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config) { + id := strings.TrimSpace(r.PathValue("id")) + if id == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot id required"}) + return + } + if config.LibraryArchiveRoot == "" { + writeJSON(w, http.StatusOK, map[string]any{"available": false, + "reason": "未接线:LIBRARY_ARCHIVE_ROOT 未配置,本平台没有整库归档可读"}) + return + } + archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档不可读"}) + return + } + defer archive.Close() + manifest, err := archive.Load(id) + if err != nil { + writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "快照不存在或校验失败"}) + return + } + usage, err := archive.Usage() + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档用量不可读"}) + return + } + view, ok := librarySnapshotViewOf(id, manifest, usage) + if !ok { + view = librarySnapshotViewOfNoUsage(id, manifest) + } + writeJSON(w, http.StatusOK, map[string]any{"available": true, "snapshot": view, "capture_note": libraryCaptureNote}) +} + +func readLibrarySnapshots(root string) (librarySnapshotList, error) { + view := librarySnapshotList{Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote} + if root == "" { + view.Reason = "未接线:LIBRARY_ARCHIVE_ROOT 未配置,本平台没有整库归档可读" + return view, nil + } + view.ArchiveRoot = root + archive, err := librarysnapshot.Open(root, 256*1024*1024) + if err != nil { + return view, errors.New("归档根不可用") + } + defer archive.Close() + usage, err := archive.Usage() + if err != nil { + return view, errors.New("归档用量不可读") + } + byID := map[string]librarysnapshot.Usage{} + for _, item := range usage { + byID[item.VersionID] = item + view.StoredBytes += item.UniqueBytes + } + for id, item := range byID { + manifest, loadErr := archive.Load(id) + if loadErr != nil { + return view, errors.New("归档快照校验失败") + } + snapshot, ok := librarySnapshotViewOf(id, manifest, []librarysnapshot.Usage{item}) + if !ok { + snapshot = librarySnapshotViewOfNoUsage(id, manifest) + } + view.Snapshots = append(view.Snapshots, snapshot) + } + sort.Slice(view.Snapshots, func(i, j int) bool { + if view.Snapshots[i].CreatedAt.Equal(view.Snapshots[j].CreatedAt) { + return view.Snapshots[i].SnapshotID > view.Snapshots[j].SnapshotID + } + return view.Snapshots[i].CreatedAt.After(view.Snapshots[j].CreatedAt) + }) + view.Available = true + return view, nil +} + +func librarySnapshotViewOf(id string, manifest librarysnapshot.Manifest, usage []librarysnapshot.Usage) (librarySnapshotView, bool) { + snapshot := librarySnapshotViewOfNoUsage(id, manifest) + for _, item := range usage { + if item.VersionID == id { + snapshot.LogicalBytes = item.LogicalBytes + snapshot.UniqueBytes = item.UniqueBytes + snapshot.SharedBytes = item.SharedBytes + return snapshot, true + } + } + return snapshot, false +} + +func librarySnapshotViewOfNoUsage(id string, manifest librarysnapshot.Manifest) librarySnapshotView { + snapshot := librarySnapshotView{SnapshotID: id, Scope: manifest.Scope, KnowledgeBaseID: manifest.KnowledgeBaseID, + TenantID: manifest.TenantID, CreatedAt: manifest.CreatedAt, ValidationPolicy: manifest.ValidationPolicy, + Format: manifest.Format, Compatibility: manifest.Compatibility, Parts: make([]librarySnapshotPart, 0, len(manifest.Parts)), + Totals: map[string]int64{}} + for _, part := range manifest.Parts { + snapshot.Parts = append(snapshot.Parts, librarySnapshotPart{Kind: part.Kind, Name: part.Name, + SHA256: part.SHA256, Count: part.Count, Bytes: part.Bytes}) + snapshot.LogicalBytes += part.Bytes + // 汇总按"数量字段"给出人看得懂的四个数字:文档/分块/索引/资源/Wiki 页。 + switch part.Name { + case "knowledges": + snapshot.Totals["documents"] = part.Count + case "chunks": + if part.Kind == "database" { + snapshot.Totals["chunks"] = part.Count + } else { + snapshot.Totals["index_documents"] = part.Count + } + case "resources": + snapshot.Totals["resources"] = part.Count + case "wiki_pages": + snapshot.Totals["wiki_pages"] = part.Count + case "wiki_folders": + snapshot.Totals["wiki_folders"] = part.Count + } + } + sort.Slice(snapshot.Parts, func(i, j int) bool { + if snapshot.Parts[i].Kind == snapshot.Parts[j].Kind { + return snapshot.Parts[i].Name < snapshot.Parts[j].Name + } + return snapshot.Parts[i].Kind < snapshot.Parts[j].Kind + }) + return snapshot +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 541f479..1d3211b 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -127,8 +127,10 @@ type Config struct { // 所在主机上读回/下发计划并触发一次整库备份(未接线时页面降级展示)。 // VersionHistory 是「版本记录」页的只读查询面;VersionHistoryRepo 是固定的仓库标识 // (owner/name)。两者都给才接线,否则页面降级为"未接线"(不报 500)。 - VersionHistory VersionHistoryStore - VersionHistoryRepo string + VersionHistory VersionHistoryStore + VersionHistoryRepo string + // LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。 + LibraryArchiveRoot string BackupSchedule BackupScheduleStore BackupHost BackupHost BackupRuns *BackupRunTracker @@ -387,6 +389,13 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("GET /api/v1/build-info", func(w http.ResponseWriter, r *http.Request) { buildInfo(w, r) }) + // 整库版本(只读):列出已封存的整库候选产物与其分项数量/字节/摘要。 + mux.HandleFunc("GET /api/v1/library/snapshots", func(w http.ResponseWriter, r *http.Request) { + librarySnapshotsList(w, r, config) + }) + mux.HandleFunc("GET /api/v1/library/snapshots/{id}", func(w http.ResponseWriter, r *http.Request) { + librarySnapshotDetail(w, r, config) + }) // 全部 main 提交时间线(读本库缓存,不直接打 Gitea) mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) { versionHistoryList(w, r, config) diff --git a/internal/libraryfreeze/project.go b/internal/libraryfreeze/project.go new file mode 100644 index 0000000..678576f --- /dev/null +++ b/internal/libraryfreeze/project.go @@ -0,0 +1,267 @@ +package libraryfreeze + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// A deployed test stack owns its containers: it cannot be relabelled without +// recreating it, which would be a change to that stack rather than to this +// platform. ProjectController therefore pins the writer group by the identity +// the stack already carries: the Compose project and the exact service names. +// +// Safety properties (the reason this is not a name filter): +// - the whole Compose project is enumerated, not just the running subset; +// - every service in the project must be declared, either as a writer or as +// explicitly non-writing with a reason, so a new sibling container cannot +// be silently omitted from the maintenance window; +// - container IDs are pinned on first inventory, so a recreated container is +// refused instead of being stopped by a stale identity; +// - only SIGTERM is sent and start waits for health, exactly like the +// label-scoped controller. +type ProjectController struct { + project string + writers map[string]bool + nonWriter map[string]string + timeout time.Duration + client *http.Client + transport *http.Transport + pinned map[string]string +} + +const composeProjectLabel = "com.docker.compose.project" +const composeServiceLabel = "com.docker.compose.service" + +// NewProjectController validates the declaration. nonWriter must explain every +// service in the project which is not a writer; an empty reason is rejected. +func NewProjectController(socket, project string, writers []string, nonWriter map[string]string, timeout time.Duration) (*ProjectController, error) { + if !filepath.IsAbs(socket) || !validID(project) || timeout <= 0 || timeout > 5*time.Minute || len(writers) == 0 || len(writers) > 32 { + return nil, errors.New("invalid project Docker controller configuration") + } + set := map[string]bool{} + for _, name := range writers { + if !validID(name) || set[name] { + return nil, errors.New("invalid project writer declaration") + } + if _, both := nonWriter[name]; both { + return nil, errors.New("project service declared as both writer and non-writer") + } + set[name] = true + } + for name, reason := range nonWriter { + if !validID(name) || strings.TrimSpace(reason) == "" { + return nil, errors.New("invalid non-writer declaration") + } + } + info, err := os.Stat(socket) + if err != nil || info.Mode()&os.ModeSocket == 0 { + return nil, errors.New("local Docker socket unavailable") + } + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socket) + }} + client := &http.Client{Transport: transport, Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { + return errors.New("Docker redirects prohibited") + }} + return &ProjectController{project: project, writers: set, nonWriter: nonWriter, timeout: timeout, + client: client, transport: transport, pinned: map[string]string{}}, nil +} + +func (p *ProjectController) Close() { p.transport.CloseIdleConnections() } + +func (p *ProjectController) request(ctx context.Context, method, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil) + if err != nil { + return errors.New("invalid Docker request") + } + resp, err := p.client.Do(req) + if err != nil { + return errors.New("project Docker request failed") + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return errors.New("project Docker request rejected") + } + if out != nil { + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1024*1024+1)) + if readErr != nil || len(data) > 1024*1024 || json.Unmarshal(data, out) != nil { + return errors.New("invalid Docker response") + } + } + return nil +} + +type projectInspection struct { + ID string `json:"Id"` + Config struct { + Labels map[string]string `json:"Labels"` + Image string `json:"Image"` + } `json:"Config"` + State struct { + Running bool + Paused bool + Restarting bool + Dead bool + Status string + Health *struct{ Status string } + } `json:"State"` +} + +// service resolves one container to its declared service name, refusing any +// container which does not belong to the declared project. +func (p *ProjectController) service(data projectInspection) (string, error) { + if data.Config.Labels[composeProjectLabel] != p.project { + return "", errors.New("container is not part of the declared project") + } + name := data.Config.Labels[composeServiceLabel] + if !p.writers[name] { + return "", errors.New("container service is not a declared writer") + } + return name, nil +} + +func (p *ProjectController) inspect(ctx context.Context, id string) (Service, error) { + if !dockerID(id) { + return Service{}, errors.New("immutable Docker ID required") + } + var data projectInspection + if err := p.request(ctx, "GET", "/containers/"+id+"/json", &data); err != nil { + return Service{}, err + } + if data.ID != id { + return Service{}, errors.New("container identity mismatch") + } + name, err := p.service(data) + if err != nil { + return Service{}, err + } + if pinned, ok := p.pinned[name]; ok && pinned != id { + return Service{}, errors.New("declared project writer identity changed") + } + if data.State.Paused || data.State.Restarting || data.State.Dead { + return Service{}, errors.New("project writer has unstable runtime state") + } + if data.State.Status != "running" && data.State.Status != "exited" && data.State.Status != "created" { + return Service{}, errors.New("unknown project writer runtime state") + } + if data.State.Running != (data.State.Status == "running") { + return Service{}, errors.New("contradictory project writer state") + } + healthy := data.State.Running && data.State.Health != nil && data.State.Health.Status == "healthy" + return Service{ID: id, Running: data.State.Running, Healthy: healthy}, nil +} + +// Inventory enumerates the complete project writer group and rejects an +// incomplete or extended declaration. +func (p *ProjectController) Inventory(ctx context.Context) ([]Service, error) { + filters, _ := json.Marshal(map[string][]string{"label": {composeProjectLabel + "=" + p.project}}) + var containers []struct { + ID string `json:"Id"` + Labels map[string]string `json:"Labels"` + } + if err := p.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &containers); err != nil { + return nil, err + } + if len(containers) == 0 || len(containers) > 256 { + return nil, errors.New("empty or oversized project container group") + } + byService := map[string]string{} + for _, container := range containers { + service := container.Labels[composeServiceLabel] + if service == "" { + return nil, errors.New("project container without a compose service label") + } + if _, duplicate := byService[service]; duplicate { + return nil, errors.New("duplicate project service instance") + } + if !p.writers[service] { + if _, declared := p.nonWriter[service]; declared { + continue + } + return nil, errors.New("undeclared project service " + service) + } + byService[service] = container.ID + } + for name := range p.writers { + if byService[name] == "" { + return nil, errors.New("declared project writer is missing") + } + } + items := make([]Service, 0, len(byService)) + for _, id := range byService { + s, err := p.inspect(ctx, id) + if err != nil { + return nil, err + } + items = append(items, s) + } + // Pin after every successful inventory so a later recreation is visible even + // when the coordinator journal is inspected outside a run. + for name, id := range byService { + p.pinned[name] = id + } + sort.Slice(items, func(i, j int) bool { return items[i].ID < items[j].ID }) + return items, nil +} + +func (p *ProjectController) wait(ctx context.Context, id string, running bool) error { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + s, err := p.inspect(ctx, id) + if err != nil { + return err + } + if s.Running == running && (!running || s.Healthy) { + return nil + } + select { + case <-ctx.Done(): + return errors.New("project writer state did not settle before deadline") + case <-ticker.C: + } + } +} + +// Stop sends SIGTERM only; there is no delayed SIGKILL which could fire after +// recovery starts the container again. +func (p *ProjectController) Stop(ctx context.Context, id string) error { + bounded, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + s, err := p.inspect(bounded, id) + if err != nil { + return err + } + if !s.Running { + return nil + } + if err = p.request(bounded, "POST", "/containers/"+id+"/kill?signal=SIGTERM", nil); err != nil { + return err + } + return p.wait(bounded, id, false) +} + +func (p *ProjectController) Start(ctx context.Context, id string) error { + bounded, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + s, err := p.inspect(bounded, id) + if err != nil { + return err + } + if !s.Running { + if err = p.request(bounded, "POST", "/containers/"+id+"/start", nil); err != nil { + return err + } + } + return p.wait(bounded, id, true) +} diff --git a/internal/libraryfreeze/project_test.go b/internal/libraryfreeze/project_test.go new file mode 100644 index 0000000..2c87bfe --- /dev/null +++ b/internal/libraryfreeze/project_test.go @@ -0,0 +1,185 @@ +package libraryfreeze + +import ( + "context" + "encoding/json" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type projectFixture struct { + appID string + frontID string + appRunning bool + appHealthy bool + signals []string + services []string +} + +func newProjectFixture(t *testing.T) (*projectFixture, string) { + t.Helper() + f := &projectFixture{ + appID: strings.Repeat("a", 64), + frontID: strings.Repeat("b", 64), + appRunning: true, + appHealthy: true, + services: []string{"app", "frontend"}, + } + // A short path: unix socket addresses are length-limited and t.TempDir() + // embeds the (long) test name. + root, err := os.MkdirTemp(os.TempDir(), "kgp-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(root) }) + socket := filepath.Join(root, "engine.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + labels := func(service string) map[string]string { + return map[string]string{composeProjectLabel: "weknora", composeServiceLabel: service} + } + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/containers/json"): + if !strings.Contains(r.URL.Query().Get("filters"), "weknora") { + t.Error("missing project filter") + } + items := []map[string]any{} + for _, service := range f.services { + id := f.appID + if service == "frontend" { + id = f.frontID + } + items = append(items, map[string]any{"Id": id, "Labels": labels(service)}) + } + _ = json.NewEncoder(w).Encode(items) + case r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/json"): + id := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/json"), "/containers/") + service := "app" + if id == f.frontID { + service = "frontend" + } + status := "exited" + health := any(nil) + if id == f.appID && f.appRunning { + status = "running" + if f.appHealthy { + health = map[string]string{"Status": "healthy"} + } else { + health = map[string]string{"Status": "starting"} + } + } + _ = json.NewEncoder(w).Encode(map[string]any{"Id": id, "Config": map[string]any{"Labels": labels(service)}, + "State": map[string]any{"Running": id == f.appID && f.appRunning, "Status": status, "Health": health}}) + case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/kill"): + if r.URL.Query().Get("signal") != "SIGTERM" { + t.Error("unsafe stop signal") + } + if !strings.Contains(r.URL.Path, f.appID) { + t.Error("stopped an undeclared container") + } + f.signals = append(f.signals, "stop") + f.appRunning = false + w.WriteHeader(204) + case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/start"): + f.signals = append(f.signals, "start") + f.appRunning = true + w.WriteHeader(204) + default: + w.WriteHeader(404) + } + })} + go server.Serve(listener) + t.Cleanup(func() { server.Close() }) + return f, socket +} + +func TestProjectControllerStopsOnlyDeclaredWritersByPinnedIdentity(t *testing.T) { + fixture, socket := newProjectFixture(t) + control, err := NewProjectController(socket, "weknora", []string{"app"}, + map[string]string{"frontend": "serves the UI only; never writes knowledge"}, time.Second) + if err != nil { + t.Fatal(err) + } + defer control.Close() + items, err := control.Inventory(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 || items[0].ID != fixture.appID || !items[0].Running || !items[0].Healthy { + t.Fatalf("unexpected writer inventory %+v", items) + } + if err = control.Stop(context.Background(), fixture.appID); err != nil { + t.Fatal(err) + } + if fixture.appRunning { + t.Fatal("writer still running after stop") + } + if err = control.Start(context.Background(), fixture.appID); err != nil { + t.Fatal(err) + } + if len(fixture.signals) != 2 || fixture.signals[0] != "stop" || fixture.signals[1] != "start" { + t.Fatalf("unexpected signal order %v", fixture.signals) + } +} + +func TestProjectControllerRefusesUndeclaredSibling(t *testing.T) { + fixture, socket := newProjectFixture(t) + control, err := NewProjectController(socket, "weknora", []string{"app"}, map[string]string{}, time.Second) + if err != nil { + t.Fatal(err) + } + defer control.Close() + if _, err = control.Inventory(context.Background()); err == nil { + t.Fatal("undeclared sibling service was accepted") + } + if len(fixture.signals) != 0 { + t.Fatal("refused inventory still signalled a container") + } +} + +func TestProjectControllerRefusesReplacedContainerIdentity(t *testing.T) { + fixture, socket := newProjectFixture(t) + control, err := NewProjectController(socket, "weknora", []string{"app"}, + map[string]string{"frontend": "serves the UI only; never writes knowledge"}, time.Second) + if err != nil { + t.Fatal(err) + } + defer control.Close() + if _, err = control.Inventory(context.Background()); err != nil { + t.Fatal(err) + } + // A recreated container keeps the service name but is a different instance: + // stopping it would hit a container the maintenance window never declared. + fixture.appID = strings.Repeat("c", 64) + if _, err = control.Inventory(context.Background()); err == nil { + t.Fatal("recreated writer identity was accepted") + } +} + +func TestProjectControllerWaitsForHealthOnStart(t *testing.T) { + fixture, socket := newProjectFixture(t) + control, err := NewProjectController(socket, "weknora", []string{"app"}, + map[string]string{"frontend": "serves the UI only; never writes knowledge"}, time.Second) + if err != nil { + t.Fatal(err) + } + defer control.Close() + if _, err = control.Inventory(context.Background()); err != nil { + t.Fatal(err) + } + fixture.appRunning = false + fixture.appHealthy = false + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if err = control.Start(ctx, fixture.appID); err == nil { + t.Fatal("start succeeded without health") + } +} diff --git a/internal/releaseadapter/v071/library_preflight.go b/internal/releaseadapter/v071/library_preflight.go new file mode 100644 index 0000000..abf2f31 --- /dev/null +++ b/internal/releaseadapter/v071/library_preflight.go @@ -0,0 +1,46 @@ +package v071 + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/librarysnapshot" +) + +// Operator preflight surface. These wrappers expose the *same* checks used by +// the capture path so an operator command can inspect a real source without +// sealing an artifact, stopping a writer, or writing anything. They add no new +// acceptance rule: a preflight which passes here is still only a candidate. +// +// Nothing here authorizes publication: artifact-acceptance-v1 still requires +// snapshot-bound content, question/answer and performance acceptance, and +// production release remains a separate, double-gated decision. + +// LibrarySourceSettled reports whether the whole source database has finished +// recorded processing. It is the identical query CaptureLibrary enforces before +// and inside the maintenance window. +func LibrarySourceSettled(ctx context.Context, db *pgxpool.Pool) error { + return checkLibrarySourceSettlement(ctx, db) +} + +// ValidateLibraryCandidateForPreflight runs the candidate validator used by +// ArchiveLibraryCandidate (identity, bidirectional content coverage, generated +// question coverage, resources and digests) without archiving anything. +func ValidateLibraryCandidateForPreflight(identity PostgresIdentity, pg PostgresSnapshot, es SearchSnapshot) error { + return validateLibraryCandidate(identity, pg, es) +} + +// LibraryContentTables is the stable list of PostgreSQL tables a library +// snapshot must contain, so an operator view does not hard-code table names. +func LibraryContentTables() []string { + names := make([]string, 0, len(libraryContentQueries)+1) + names = append(names, "resources") + for _, entry := range libraryContentQueries { + names = append(names, entry.name) + } + return names +} + +// LibrarySnapshotCompatibility is the archive-level compatibility contract. +type LibrarySnapshotCompatibility = librarysnapshot.Compatibility diff --git a/scripts/library-capture.sh b/scripts/library-capture.sh new file mode 100755 index 0000000..81ac478 --- /dev/null +++ b/scripts/library-capture.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Whole-library capture entry point for the local WKNFull test stack. +# +# Every service of the Compose project is declared below: exactly one writer +# (app) and a reason for each sibling which must NOT be stopped. ProjectController +# refuses to run if the project gains a service that is not declared here, so a +# new container cannot be silently left outside the maintenance window. +# +# Credentials are read from the running containers at call time and are never +# printed. Nothing here writes to the source stack except SIGTERM/SIGSTART of the +# declared writer. +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +BIN="${LIBRARYCTL_BIN:-$ROOT/libraryctl}" +if [ ! -x "$BIN" ]; then + echo "LIBRARYCTL_MISSING: build it with 'go build -o libraryctl ./cmd/libraryctl'" >&2 + exit 1 +fi +# Reasons must not contain commas: the flag is comma-separated. +NON_WRITERS="frontend=UI only; no knowledge writes" +NON_WRITERS+=",mcp=read-only MCP server" +NON_WRITERS+=",dobotai-target=alpine placeholder; no knowledge writes" +NON_WRITERS+=",docreader=parse RPC service; app persists parse output and settlement requires zero pending ops" +NON_WRITERS+=",elasticsearch=search store; not an initiator" +NON_WRITERS+=",postgres=database store; not an initiator" +NON_WRITERS+=",redis=queue/cache store; not an initiator" +NON_WRITERS+=",neo4j=graph store; not an initiator" +NON_WRITERS+=",langfuse-web=LLM observability UI" +NON_WRITERS+=",langfuse-worker=LLM observability worker; own storage" +NON_WRITERS+=",langfuse-clickhouse=observability store" +NON_WRITERS+=",langfuse-minio=observability object store" +NON_WRITERS+=",langfuse-db-init=one-shot init; exited" +NON_WRITERS+=",searxng=read-only search proxy" +NON_WRITERS+=",searxng-init=one-shot init; exited" +exec "$BIN" "$@" \ + -source-app WKNFull-app \ + -source-pg WKNFull-postgres \ + -source-es WKNFull-elasticsearch \ + -governance-pg knowledge-governance-p1-postgres-1 \ + -project weknora071full \ + -writers app \ + -non-writers "$NON_WRITERS" \ No newline at end of file diff --git a/web/src/App.tsx b/web/src/App.tsx index 0d213e6..582db36 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -4,6 +4,7 @@ import { DocumentWorkbench } from './DocumentWorkbench' import { SourceCatalogBrowser } from './SourceCatalogBrowser' import { WeKnoraMonitor } from './WeKnoraMonitor' import { VersionHistory } from './VersionHistory' +import { LibraryVersions } from './LibraryVersions' import { QAOperations } from './QAOperations' import { SourceLinkManager } from './SourceLinkManager' import { SystemSettings } from './SystemSettings' @@ -62,6 +63,7 @@ const modules: ReadonlyArray<{ id: ModuleId; label: string; icon: string }> = [ { id: 'tasks', label: '解析监控', icon: '↻' }, { id: 'qaops', label: '问答运营', icon: '▤' }, { id: 'releases', label: '发布管理', icon: '⇧' }, + { id: 'library', label: '整库版本', icon: '▦' }, { id: 'settings', label: '系统设置', icon: '⚙' }, { id: 'versions', label: '版本记录', icon: '⌥' }, ] @@ -2970,6 +2972,8 @@ export default function App() { ) : workspace.activeTab === 'knowledge' ? ( + ) : workspace.activeTab === 'library' ? ( + ) : workspace.activeTab === 'versions' ? ( ) : workspace.activeTab === 'qaops' ? ( diff --git a/web/src/LibraryVersions.tsx b/web/src/LibraryVersions.tsx new file mode 100644 index 0000000..ca9b1d3 --- /dev/null +++ b/web/src/LibraryVersions.tsx @@ -0,0 +1,258 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { + fetchLibrarySnapshots, + formatBytes, + scopeLabel, + type LibrarySnapshot, + type LibrarySnapshotList, +} from './librarySnapshots' +import './libraryVersions.css' + +// 「整库版本」页:只读展示已封存的整库候选产物(对内/对外各一份最新)。 +// 页面不提供封存按钮:封存要停真实写入者并读来源凭据,只能在目标主机用 +// scripts/library-capture.sh 执行。候选 != 验收通过,也 != 可发布。 +const stamp = (value: string) => + value && Number.isFinite(new Date(value).getTime()) + ? new Date(value).toLocaleString('zh-CN', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hour12: false, + }) + : '—' + +const metric = (snapshot: LibrarySnapshot, key: string) => String(snapshot.totals[key] ?? 0) + +export function LibraryVersions() { + const [page, setPage] = useState(null) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [expanded, setExpanded] = useState(null) + const [scopeFilter, setScopeFilter] = useState<'all' | 'internal' | 'external'>('all') + + const load = useCallback(async (signal?: AbortSignal) => { + setLoading(true) + try { + const result = await fetchLibrarySnapshots(signal) + setPage(result) + setError(null) + } catch (loadError) { + if ((loadError as Error).name === 'AbortError') return + setError(loadError instanceof Error ? loadError.message : '加载整库版本失败') + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + const controller = new AbortController() + void load(controller.signal) + return () => controller.abort() + }, [load]) + + const snapshots = page?.snapshots ?? [] + const latestByScope = useMemo(() => { + const map: Record = {} + for (const snapshot of snapshots) { + if (!map[snapshot.scope]) map[snapshot.scope] = snapshot + } + return map + }, [snapshots]) + const visible = snapshots.filter((snapshot) => scopeFilter === 'all' || snapshot.scope === scopeFilter) + + return ( +
+
+
+

整库版本

+

+ 每个条目是一份完整封存的整库候选:数据库十表、原始索引正文与向量、对象资源与 Wiki 快照, + 带逐项摘要与兼容契约。封存期间真实写入者被停住,完成后已恢复。 +

+
+
+ + 共 {snapshots.length} 份 · 归档占用 {formatBytes(page?.stored_bytes ?? 0)} + + +
+
+ + {page && !page.available ? ( +
+ 未接线 +

{page.reason || '归档根未配置'}

+
+ ) : null} + {error ? ( +
+ 读取失败 +

{error}

+
+ ) : null} + + {page?.available ? ( + <> +
+ {(['internal', 'external'] as const).map((scope) => { + const latest = latestByScope[scope] + return ( +
+
+

{scopeLabel(scope)}

+ {latest ? '已封存' : '尚无封存'} +
+ {latest ? ( + <> +

封存于 {stamp(latest.created_at)}

+
+
+
文档
+
{metric(latest, 'documents')}
+
+
+
分块
+
{metric(latest, 'chunks')}
+
+
+
索引
+
{metric(latest, 'index_documents')}
+
+
+
资源
+
{metric(latest, 'resources')}
+
+
+
Wiki 页
+
{metric(latest, 'wiki_pages')}
+
+
+
体积
+
{formatBytes(latest.logical_bytes)}
+
+
+

+ 快照 {latest.snapshot_id.slice(0, 16)} +

+ + ) : ( +

+ 该库没有封存产物。封存在目标主机执行: + scripts/library-capture.sh capture -scope {scope} +

+ )} +
+ ) + })} +
+ +
+
+ {( + [ + { id: 'all', label: '全部' }, + { id: 'internal', label: '对内' }, + { id: 'external', label: '对外' }, + ] as const + ).map((item) => ( + + ))} +
+ {page.capture_note} +
+ + {visible.length === 0 ? ( +

还没有封存产物。

+ ) : ( + + + + + + + + + + + + + + + + + {visible.map((snapshot) => ( + + + + + + + + + + + + + ))} + +
封存时间范围文档分块索引资源Wiki体积模型 / 维度快照
{stamp(snapshot.created_at)}{scopeLabel(snapshot.scope)}{metric(snapshot, 'documents')}{metric(snapshot, 'chunks')}{metric(snapshot, 'index_documents')}{metric(snapshot, 'resources')}{metric(snapshot, 'wiki_pages')} + {formatBytes(snapshot.logical_bytes)} + + {snapshot.compatibility.embedding_model || '—'} / {snapshot.compatibility.embedding_dimension} + + +
+ )} + + {visible + .filter((snapshot) => snapshot.snapshot_id === expanded) + .map((snapshot) => ( +
+
+ {scopeLabel(snapshot.scope)} · {snapshot.snapshot_id} + + 策略 {snapshot.validation_policy || '—'} · WeKnora {snapshot.compatibility.weknora_version} · + schema {snapshot.compatibility.schema_version} · 映射{' '} + {snapshot.compatibility.index_mapping_sha256.slice(0, 12)} + +
+
    + {snapshot.parts.map((part) => ( +
  • + {part.kind} + {part.name} + {part.count} 项 + {formatBytes(part.bytes)} + {part.sha256.slice(0, 12)} +
  • + ))} +
+
+ ))} + + ) : null} +
+ ) +} \ No newline at end of file diff --git a/web/src/librarySnapshots.ts b/web/src/librarySnapshots.ts new file mode 100644 index 0000000..493b578 --- /dev/null +++ b/web/src/librarySnapshots.ts @@ -0,0 +1,137 @@ +// 「整库版本」页的只读客户端:列已经封存的整库候选产物。 +// +// 这里没有任何写操作:封存需要维护窗口(停真实写入者)与来源凭据,只能由运维 +// 命令在目标主机执行。页面只负责如实展示已封存的内容,不提供"开始同步"这类 +// 没接线的假按钮。 + +export interface LibrarySnapshotPart { + kind: string + name: string + sha256: string + count: number + bytes: number +} + +export interface LibraryCompatibility { + weknora_version: string + schema_version: number + retriever_engine: string + embedding_model: string + embedding_dimension: number + index_mapping_sha256: string +} + +export interface LibrarySnapshot { + snapshot_id: string + scope: string + knowledge_base_id: string + tenant_id: number + created_at: string + validation_policy: string + format: string + compatibility: LibraryCompatibility + parts: LibrarySnapshotPart[] + logical_bytes: number + unique_bytes: number + shared_bytes: number + totals: Record +} + +export interface LibrarySnapshotList { + available: boolean + reason: string + archive_root: string + snapshots: LibrarySnapshot[] + stored_bytes: number + capture_note: string +} + +const record = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) +const text = (value: unknown) => (typeof value === 'string' ? value : '') +const num = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) ? value : 0) +const bool = (value: unknown) => value === true + +function parsePart(raw: unknown): LibrarySnapshotPart | null { + if (!record(raw)) return null + const kind = text(raw.kind) + const name = text(raw.name) + if (!kind || !name) return null + return { kind, name, sha256: text(raw.sha256), count: num(raw.count), bytes: num(raw.bytes) } +} + +function parseSnapshot(raw: unknown): LibrarySnapshot | null { + if (!record(raw)) return null + const snapshotID = text(raw.snapshot_id) + if (!snapshotID) return null + const compatibility = record(raw.compatibility) ? raw.compatibility : {} + const totals: Record = {} + if (record(raw.totals)) { + for (const [key, value] of Object.entries(raw.totals)) totals[key] = num(value) + } + return { + snapshot_id: snapshotID, + scope: text(raw.scope), + knowledge_base_id: text(raw.knowledge_base_id), + tenant_id: num(raw.tenant_id), + created_at: text(raw.created_at), + validation_policy: text(raw.validation_policy), + format: text(raw.format), + compatibility: { + weknora_version: text(compatibility.weknora_version), + schema_version: num(compatibility.schema_version), + retriever_engine: text(compatibility.retriever_engine), + embedding_model: text(compatibility.embedding_model), + embedding_dimension: num(compatibility.embedding_dimension), + index_mapping_sha256: text(compatibility.index_mapping_sha256), + }, + parts: Array.isArray(raw.parts) + ? raw.parts.map(parsePart).filter((item): item is LibrarySnapshotPart => item !== null) + : [], + logical_bytes: num(raw.logical_bytes), + unique_bytes: num(raw.unique_bytes), + shared_bytes: num(raw.shared_bytes), + totals, + } +} + +export async function fetchLibrarySnapshots(signal?: AbortSignal): Promise { + const response = await fetch('/api/v1/library/snapshots', { + headers: { Accept: 'application/json' }, + signal, + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok && !record(value)) { + throw new Error(`加载整库版本失败(HTTP ${response.status})`) + } + if (!record(value)) throw new Error('整库版本响应无效') + const snapshots = Array.isArray(value.snapshots) + ? value.snapshots.map(parseSnapshot).filter((item): item is LibrarySnapshot => item !== null) + : [] + return { + available: bool(value.available), + reason: text(value.reason), + archive_root: text(value.archive_root), + snapshots, + stored_bytes: num(value.stored_bytes), + capture_note: text(value.capture_note), + } +} + +export function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes <= 0) return '0 B' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let index = 0 + let value = bytes + while (value >= 1024 && index < units.length - 1) { + value /= 1024 + index += 1 + } + return `${value >= 10 || index === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[index]}` +} + +export function scopeLabel(scope: string): string { + if (scope === 'internal') return '对内' + if (scope === 'external') return '对外' + return scope || '未知范围' +} \ No newline at end of file diff --git a/web/src/libraryVersions.css b/web/src/libraryVersions.css new file mode 100644 index 0000000..593a4ac --- /dev/null +++ b/web/src/libraryVersions.css @@ -0,0 +1,265 @@ +/* Whole-library versions: scope cards first, then a compact sealed-artifact table. */ +.library-versions { + display: flex; + flex-direction: column; + gap: 18px; + min-width: 0; + padding: 24px; + color: var(--ink-base); +} +.library-versions-header, +.library-versions-toolbar, +.library-snapshot-parts, +.library-scope-card { + background: var(--bg-panel, #fff); + border: 1px solid var(--border-base, #e2e8f0); + border-radius: 10px; + padding: 20px; + min-width: 0; +} +.library-versions-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; + flex-wrap: wrap; +} +.library-versions h2 { + margin: 0 0 6px; + font-size: 22px; +} +.library-versions-sub { + margin: 0; + max-width: 68ch; + color: var(--ink-mute, #64748b); + font-size: 13px; + line-height: 1.6; +} +.library-versions-actions { + display: flex; + align-items: center; + gap: 12px; + flex-shrink: 0; +} +.library-versions-usage { + font-size: 13px; + color: var(--ink-mute, #64748b); + font-variant-numeric: tabular-nums; +} +.library-versions-actions button, +.library-versions-filters button { + padding: 7px 14px; + border-radius: 8px; + border: 1px solid var(--border-base, #e2e8f0); + background: var(--bg-panel, #fff); + color: inherit; + cursor: pointer; +} +.library-versions-actions button:disabled { + cursor: progress; + opacity: 0.6; +} +.library-versions-blocked { + border: 1px solid #f3c2b8; + background: #fff6f3; + border-radius: 10px; + padding: 16px 20px; +} +.library-versions-blocked strong { + display: block; + margin-bottom: 4px; + color: #a03a1e; +} +.library-versions-blocked p { + margin: 0; + font-size: 13px; + color: #7c2d12; +} +.library-versions-scopes { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: 16px; +} +.library-scope-card header { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 12px; +} +.library-scope-card h3 { + margin: 0; + font-size: 18px; +} +.library-scope-state { + font-size: 12px; + padding: 2px 10px; + border-radius: 999px; + background: #ecfdf5; + color: #047857; + border: 1px solid #a7f3d0; +} +.library-scope-card.empty .library-scope-state { + background: #f8fafc; + color: #64748b; + border-color: #e2e8f0; +} +.library-scope-time { + margin: 8px 0 14px; + font-size: 12px; + color: var(--ink-mute, #64748b); +} +.library-scope-metrics { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; + margin: 0 0 12px; +} +.library-scope-metrics div { + border: 1px solid var(--border-base, #e2e8f0); + border-radius: 8px; + padding: 8px 10px; +} +.library-scope-metrics dt { + font-size: 12px; + color: var(--ink-mute, #64748b); +} +.library-scope-metrics dd { + margin: 2px 0 0; + font-size: 18px; + font-variant-numeric: tabular-nums; +} +.library-scope-id, +.library-scope-hint { + margin: 0; + font-size: 12px; + color: var(--ink-mute, #64748b); + overflow-wrap: anywhere; +} +.library-scope-hint code, +.library-scope-id code { + font-size: 12px; + background: #f1f5f9; + border-radius: 5px; + padding: 1px 6px; +} +.library-versions-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + padding: 12px 16px; + flex-wrap: wrap; +} +.library-versions-filters { + display: flex; + gap: 8px; +} +.library-versions-filters button.active { + border-color: #2563eb; + color: #1d4ed8; + background: #eff6ff; +} +.library-versions-note { + font-size: 12px; + color: var(--ink-mute, #64748b); + max-width: 70ch; + line-height: 1.5; +} +.library-versions-empty { + margin: 0; + padding: 24px; + border: 1px dashed var(--border-base, #e2e8f0); + border-radius: 10px; + color: var(--ink-mute, #64748b); + font-size: 13px; +} +.library-versions-table { + width: 100%; + border-collapse: collapse; + background: var(--bg-panel, #fff); + border: 1px solid var(--border-base, #e2e8f0); + border-radius: 10px; + overflow: hidden; + font-size: 13px; +} +.library-versions-table th, +.library-versions-table td { + padding: 10px 12px; + text-align: left; + border-bottom: 1px solid var(--border-base, #eef2f7); + white-space: nowrap; +} +.library-versions-table th { + font-weight: 600; + font-size: 12px; + color: var(--ink-mute, #64748b); + background: #f8fafc; +} +.library-versions-table td { + font-variant-numeric: tabular-nums; +} +.library-snapshot-toggle { + border: 1px solid var(--border-base, #e2e8f0); + border-radius: 7px; + background: #fff; + padding: 4px 8px; + cursor: pointer; + font-size: 12px; + color: inherit; +} +.library-snapshot-parts header { + display: flex; + flex-direction: column; + gap: 4px; + margin-bottom: 12px; + font-size: 12px; + color: var(--ink-mute, #64748b); + overflow-wrap: anywhere; +} +.library-snapshot-parts header strong { + font-size: 14px; + color: var(--ink-base); +} +.library-snapshot-parts ul { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 6px; +} +.library-snapshot-parts li { + display: grid; + grid-template-columns: 90px minmax(140px, 2fr) 80px 90px 110px; + gap: 10px; + align-items: center; + font-size: 12px; + padding: 6px 8px; + border-radius: 7px; + background: #f8fafc; + overflow-wrap: anywhere; +} +.library-part-kind { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--ink-mute, #64748b); +} +.library-part-name { + font-weight: 600; +} +@media (max-width: 760px) { + .library-versions { + padding: 12px; + } + .library-versions-table { + display: block; + overflow-x: auto; + } + .library-snapshot-parts li { + grid-template-columns: 1fr; + } + .library-scope-metrics { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} \ No newline at end of file diff --git a/web/src/workspace.ts b/web/src/workspace.ts index 66f4205..086d982 100644 --- a/web/src/workspace.ts +++ b/web/src/workspace.ts @@ -1,11 +1,20 @@ -export type ModuleId = 'dashboard' | 'sources' | 'knowledge' | 'tasks' | 'releases' | 'settings' | 'versions' | 'qaops' +export type ModuleId = + | 'dashboard' + | 'sources' + | 'knowledge' + | 'tasks' + | 'releases' + | 'library' + | 'settings' + | 'versions' + | 'qaops' export interface WorkspaceState { openTabs: ModuleId[] activeTab: ModuleId } -const moduleIds = new Set(['dashboard', 'tasks', 'releases', 'settings', 'qaops']) +const moduleIds = new Set(['dashboard', 'tasks', 'releases', 'library', 'settings', 'qaops']) export const initialWorkspace: WorkspaceState = { openTabs: ['dashboard'],