feat: 知识治理平台(含已验证发布/回滚契约)
全栈知识治理平台首次入库(P1-P10 + 发布链路模拟验证闭环): - 治理核心:源目录/变更集/人工审核/发布批次/发布执行(fail-closed 闸口) - 测试通道:WeKnora 测试 KB 上传/轮询/验收;poll 完成时写入 parser_generation/index_generation,修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071(本地 source=测试角色 / 云端 target=生产角色 模拟验证通过): * ES 导出/导入 mapping 自适应(.keyword 子字段、可选 tenant_id) * OSS locator 支持嵌套 scheme;chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换;verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id(strict 解码) - 三通道发布+回滚演练实测:publish 4/4 verified,检索命中, rollback 4/4 rolled_back,目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一,120 用例通过 - 门禁:go vet / go test 全包通过,gofmt 干净
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
type releaseRuntime struct {
|
||||
publisher *releaseworkflow.RoutingPublisher
|
||||
pools []*pgxpool.Pool
|
||||
}
|
||||
|
||||
func (runtime *releaseRuntime) Close() {
|
||||
for index := len(runtime.pools) - 1; index >= 0; index-- {
|
||||
runtime.pools[index].Close()
|
||||
}
|
||||
}
|
||||
|
||||
func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) {
|
||||
runtime := &releaseRuntime{}
|
||||
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
|
||||
for _, route := range config.Routes {
|
||||
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, err
|
||||
}
|
||||
runtime.pools = append(runtime.pools, pools...)
|
||||
publishers[route.KnowledgeBaseID] = publisher
|
||||
}
|
||||
router, err := releaseworkflow.NewRoutingPublisher(publishers)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, errors.New("create release publisher routes")
|
||||
}
|
||||
runtime.publisher = router
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read source database credential")
|
||||
}
|
||||
targetDatabaseURL, err := readRestrictedValue(route.Target.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read target database credential")
|
||||
}
|
||||
if err := validateDistinctDatabaseURLs(sourceDatabaseURL, targetDatabaseURL); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sourcePool, err := openReleasePool(ctx, sourceDatabaseURL)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("connect release source database")
|
||||
}
|
||||
pools := []*pgxpool.Pool{sourcePool}
|
||||
targetPool, err := openReleasePool(ctx, targetDatabaseURL)
|
||||
if err != nil {
|
||||
sourcePool.Close()
|
||||
return nil, nil, errors.New("connect release target database")
|
||||
}
|
||||
pools = append(pools, targetPool)
|
||||
cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
for index := len(pools) - 1; index >= 0; index-- {
|
||||
pools[index].Close()
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
sourceSearchToken, err := readOptionalRestrictedValue(route.Source.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read source Elasticsearch credential"))
|
||||
}
|
||||
targetSearchToken, err := readOptionalRestrictedValue(route.Target.Elasticsearch.TokenFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target Elasticsearch credential"))
|
||||
}
|
||||
sourceSearch, err := v071.NewSearchExporter(v071.SearchExporterConfig{BaseURL: route.Source.Elasticsearch.BaseURL, Index: route.Source.Elasticsearch.Index, AuthorizationToken: sourceSearchToken, PageSize: 500})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source Elasticsearch exporter"))
|
||||
}
|
||||
targetSearchConfig := v071.SearchExporterConfig{BaseURL: route.Target.Elasticsearch.BaseURL, Index: route.Target.Elasticsearch.Index, AuthorizationToken: targetSearchToken, PageSize: 500}
|
||||
targetSearchInspector, err := v071.NewSearchExporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch inspector"))
|
||||
}
|
||||
targetSearch, err := v071.NewSearchImporter(targetSearchConfig)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target Elasticsearch importer"))
|
||||
}
|
||||
|
||||
sourceObject, err := buildS3ObjectStore(route.Source.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object reader"))
|
||||
}
|
||||
targetObject, err := buildS3ObjectStore(route.Target.Object)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object store"))
|
||||
}
|
||||
objectExporter, err := v071.NewObjectExporter(sourceObject, v071.ObjectExporterLimits{})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source object exporter"))
|
||||
}
|
||||
objectImporter, err := v071.NewObjectImporter(targetObject, v071.TargetObjectConfig{StorageBackendID: route.Target.Object.StorageBackendID, Provider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target object importer"))
|
||||
}
|
||||
|
||||
sourceContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: sourcePool, Mapping: sourceSearch, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Source.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source release contract inspector"))
|
||||
}
|
||||
targetContract, err := v071.NewTargetContractInspector(v071.TargetContractInspectorConfig{DB: targetPool, Mapping: targetSearchInspector, KnowledgeBaseID: route.KnowledgeBaseID, TenantID: route.TenantID, IndexName: route.Target.Elasticsearch.Index})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target release contract inspector"))
|
||||
}
|
||||
postgresExporter, err := v071.NewPostgresExporter(sourcePool)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create source PostgreSQL exporter"))
|
||||
}
|
||||
postgresImporter, err := v071.NewPostgresImporter(targetPool, v071.PostgresImporterConfig{TargetStorageBackendID: route.Target.Object.StorageBackendID, TargetProvider: route.Target.Object.Provider})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target PostgreSQL importer"))
|
||||
}
|
||||
documentExporter, err := v071.NewDocumentExporter(v071.DocumentExporterConfig{Contract: sourceContract, Postgres: postgresExporter, Search: sourceSearch, Objects: objectExporter, TenantID: route.TenantID, EmbeddingModel: route.EmbeddingModel, Now: time.Now})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document exporter"))
|
||||
}
|
||||
|
||||
archive, err := v071.NewBundleArchive(filepath.Join(archiveRoot, route.KnowledgeBaseID), 1024*1024*1024)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("open release bundle archive"))
|
||||
}
|
||||
apiKey, err := readRestrictedValue(route.Target.WeKnora.APIKeyFile)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("read target WeKnora credential"))
|
||||
}
|
||||
weknoraClient, err := weknora.NewClient(weknora.ClientConfig{BaseURL: route.Target.WeKnora.BaseURL, APIKey: apiKey, Timeout: 60 * time.Second, AllowDockerHTTP: route.Target.WeKnora.AllowDockerHTTP})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verification client"))
|
||||
}
|
||||
verifier, err := v071.NewWeKnoraVerifier(weknoraClient)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create target WeKnora verifier"))
|
||||
}
|
||||
documentTarget, err := v071.NewDocumentTarget(v071.DocumentTargetConfig{Archive: archive, Contract: targetContract, Postgres: postgresImporter, Search: targetSearch, Objects: objectImporter, Verifier: verifier})
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document target"))
|
||||
}
|
||||
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 release publisher"))
|
||||
}
|
||||
return publisher, pools, nil
|
||||
}
|
||||
|
||||
func buildS3ObjectStore(config releaseObjectConfig) (*v071.S3ObjectStore, error) {
|
||||
accessKey, err := readRestrictedValue(config.AccessKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secretKey, err := readRestrictedValue(config.SecretKeyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsed, err := url.Parse(config.Endpoint)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid object endpoint")
|
||||
}
|
||||
return v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{Provider: config.Provider, Endpoint: config.Endpoint, Region: config.Region, AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: config.Bucket, Secure: parsed.Scheme == "https", ForcePathStyle: config.Provider == "minio"})
|
||||
}
|
||||
|
||||
func readRestrictedValue(path string) (string, error) {
|
||||
return weknora.ReadAPIKeyFile(path)
|
||||
}
|
||||
|
||||
func readOptionalRestrictedValue(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", nil
|
||||
}
|
||||
return readRestrictedValue(path)
|
||||
}
|
||||
|
||||
func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) {
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := pool.Ping(ctx); err != nil {
|
||||
pool.Close()
|
||||
return nil, err
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func validateDistinctDatabaseURLs(sourceURL, targetURL string) error {
|
||||
source, err := pgxpool.ParseConfig(sourceURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid source database credential")
|
||||
}
|
||||
target, err := pgxpool.ParseConfig(targetURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid target database credential")
|
||||
}
|
||||
sourceIdentity := strings.ToLower(source.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(source.ConnConfig.Port), 10) + "|" + source.ConnConfig.Database
|
||||
targetIdentity := strings.ToLower(target.ConnConfig.Host) + "|" + strconv.FormatUint(uint64(target.ConnConfig.Port), 10) + "|" + target.ConnConfig.Database
|
||||
if sourceIdentity == targetIdentity {
|
||||
return errors.New("release source and target database must be distinct")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user