From b3c93319665ef8d2d8b48e9a7c6e4e91778b5e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Tue, 15 Sep 2026 09:46:31 +0800 Subject: [PATCH] =?UTF-8?q?feat(ingest):=20=E6=BA=90=E4=B8=8E=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E6=96=87=E4=BB=B6=E5=90=8D=E8=BF=87=E6=BB=A4=E5=8F=8A?= =?UTF-8?q?=E9=92=89=E9=92=89=E6=96=87=E6=A1=A3=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - filename_filter_rules table (000035) with source-root and per-directory rows - additive inheritance: source default + every ancestor directory contributes - filename_is_filtered() SQL predicate applied to catalog browsing, subtree counts, change-set diffing, routing fan-out and artifact fetch manifests - fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch jobs, retires planned publications and blocks items when rules change - GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized extensions) + Settings and Catalog filter editors - failJobSQL no longer swallows the claimed row when the failure projection upserts (fixes spurious 'lease lost' retries) - DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403 from file/download fixed), header-signature downloads supported --- api/openapi.yaml | 23 +++ internal/governance/filename_filter.go | 86 +++++++++ internal/governance/filename_filter_test.go | 49 ++++++ internal/httpapi/filename_filters.go | 81 +++++++++ internal/httpapi/server.go | 11 ++ internal/httpapi/server_test.go | 37 ++++ .../platform/postgres/artifact_fetch_store.go | 15 +- internal/platform/postgres/catalog_routing.go | 1 + .../platform/postgres/filename_filters.go | 163 ++++++++++++++++++ .../filename_filters_integration_test.go | 128 ++++++++++++++ .../000035_filename_filters.down.sql | 13 ++ .../migrations/000035_filename_filters.up.sql | 69 ++++++++ internal/platform/postgres/migrations_test.go | 8 +- internal/platform/postgres/queue_sql.go | 5 +- internal/platform/postgres/source_catalog.go | 37 +++- internal/workflows/artifactfetch/handler.go | 4 + .../workflows/artifactfetch/handler_test.go | 42 +++++ scripts/artifact_fetch.py | 112 ++++++++++-- scripts/artifact_fetch_test.py | 47 +++++ web/src/FilenameFilterEditor.test.tsx | 26 +++ web/src/FilenameFilterEditor.tsx | 90 ++++++++++ web/src/SourceCatalogBrowser.tsx | 8 +- web/src/SystemSettings.tsx | 2 + web/src/filenameFilters.ts | 48 ++++++ web/src/systemSettings.css | 5 + 25 files changed, 1080 insertions(+), 30 deletions(-) create mode 100644 internal/governance/filename_filter.go create mode 100644 internal/governance/filename_filter_test.go create mode 100644 internal/httpapi/filename_filters.go create mode 100644 internal/platform/postgres/filename_filters.go create mode 100644 internal/platform/postgres/filename_filters_integration_test.go create mode 100644 internal/platform/postgres/migrations/000035_filename_filters.down.sql create mode 100644 internal/platform/postgres/migrations/000035_filename_filters.up.sql create mode 100644 internal/workflows/artifactfetch/handler_test.go create mode 100644 scripts/artifact_fetch_test.py create mode 100644 web/src/FilenameFilterEditor.test.tsx create mode 100644 web/src/FilenameFilterEditor.tsx create mode 100644 web/src/filenameFilters.ts diff --git a/api/openapi.yaml b/api/openapi.yaml index b508a65..6920fa1 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -425,6 +425,29 @@ } } }, + "/api/v1/filename-filters": { + "get": { + "tags": ["Source connections"], + "operationId": "listFilenameFilters", + "summary": "列出来源和目录的文件扩展名筛选规则", + "parameters": [{"name":"connection_id","in":"query","required":true,"schema":{"type":"string"}}], + "responses": {"200":{"description":"筛选规则"},"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"}} + }, + "put": { + "tags": ["Source connections"], + "operationId": "setFilenameFilter", + "summary": "设置来源或目录文件扩展名筛选", + "parameters": [{"$ref":"#/components/parameters/IdempotencyKey"}], + "responses": {"200":{"description":"筛选规则已生效"},"400":{"$ref":"#/components/responses/BadRequest"},"404":{"$ref":"#/components/responses/NotFound"},"409":{"$ref":"#/components/responses/Conflict"},"500":{"$ref":"#/components/responses/InternalError"}} + }, + "delete": { + "tags": ["Source connections"], + "operationId": "clearFilenameFilter", + "summary": "移除本级筛选并恢复继承", + "parameters": [{"$ref":"#/components/parameters/IdempotencyKey"}], + "responses": {"200":{"description":"本级筛选已移除"},"400":{"$ref":"#/components/responses/BadRequest"},"404":{"$ref":"#/components/responses/NotFound"},"409":{"$ref":"#/components/responses/Conflict"},"500":{"$ref":"#/components/responses/InternalError"}} + } + }, "/api/v1/scope-rules": { "get": { "tags": [ diff --git a/internal/governance/filename_filter.go b/internal/governance/filename_filter.go new file mode 100644 index 0000000..0e55975 --- /dev/null +++ b/internal/governance/filename_filter.go @@ -0,0 +1,86 @@ +package governance + +import ( + "errors" + "path/filepath" + "sort" + "strings" + "time" +) + +var ( + ErrInvalidFilenameFilter = errors.New("invalid filename filter") + ErrFilenameFilterTargetNotFound = errors.New("filename filter target not found") +) + +// FilenameFilterRule is additive: the source rule uses an empty directory path; +// a directory inherits that rule plus every matching ancestor rule. +type FilenameFilterRule struct { + ConnectionID string `json:"connection_id"` + DirectoryPath string `json:"directory_path"` + ExcludedExtensions []string `json:"excluded_extensions"` + UpdatedBy string `json:"updated_by"` + UpdatedAt time.Time `json:"updated_at"` +} + +type FilenameFilterMutation struct { + ConnectionID string + DirectoryPath string + ExcludedExtensions []string + OperatorRef string + Clear bool + Now time.Time +} + +func NormalizeExcludedExtensions(values []string) ([]string, error) { + if len(values) == 0 || len(values) > 100 { + return nil, errors.New("at least one and no more than 100 extensions are required") + } + seen := make(map[string]bool, len(values)) + result := make([]string, 0, len(values)) + for _, raw := range values { + value := strings.ToLower(strings.TrimSpace(raw)) + value = strings.TrimPrefix(value, ".") + if value == "" || len(value) > 16 { + return nil, errors.New("file extension is invalid") + } + for _, character := range value { + if character >= 'a' && character <= 'z' || character >= '0' && character <= '9' || character == '_' || character == '-' { + continue + } + return nil, errors.New("file extension is invalid") + } + if !seen[value] { + seen[value] = true + result = append(result, value) + } + } + sort.Strings(result) + return result, nil +} + +func (request FilenameFilterMutation) Validate() error { + if strings.TrimSpace(request.ConnectionID) == "" || len(request.ConnectionID) > 36 || !validScopePath(request.DirectoryPath) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 || request.Now.IsZero() { + return errors.New("filename filter target is invalid") + } + if request.Clear { + if len(request.ExcludedExtensions) != 0 { + return errors.New("clear filename filter must not include extensions") + } + return nil + } + normalized, err := NormalizeExcludedExtensions(request.ExcludedExtensions) + if err != nil || len(normalized) != len(request.ExcludedExtensions) { + return errors.New("excluded extensions must be normalized and unique") + } + for index := range normalized { + if normalized[index] != request.ExcludedExtensions[index] { + return errors.New("excluded extensions must be normalized and sorted") + } + } + return nil +} + +func FilenameExtension(name string) string { + return strings.TrimPrefix(strings.ToLower(filepath.Ext(name)), ".") +} diff --git a/internal/governance/filename_filter_test.go b/internal/governance/filename_filter_test.go new file mode 100644 index 0000000..ccf1b14 --- /dev/null +++ b/internal/governance/filename_filter_test.go @@ -0,0 +1,49 @@ +package governance + +import ( + "reflect" + "testing" + "time" +) + +func TestNormalizeExcludedExtensionsCanonicalizesAndDeduplicates(t *testing.T) { + got, err := NormalizeExcludedExtensions([]string{".ZIP", " rar ", "Mp4", "zip"}) + if err != nil { + t.Fatal(err) + } + want := []string{"mp4", "rar", "zip"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("NormalizeExcludedExtensions() = %#v, want %#v", got, want) + } +} + +func TestNormalizeExcludedExtensionsRejectsFilenamePatternsAndUnsafeValues(t *testing.T) { + for _, input := range [][]string{{"*.zip"}, {"a/b"}, {"zip;rar"}, {""}, {"this-extension-is-far-too-long"}} { + if _, err := NormalizeExcludedExtensions(input); err == nil { + t.Fatalf("NormalizeExcludedExtensions(%#v) succeeded", input) + } + } +} + +func TestFilenameFilterMutationAcceptsSourceAndNestedDirectory(t *testing.T) { + now := time.Now() + for _, path := range []string{"", "02 产品资料/固件"} { + request := FilenameFilterMutation{ConnectionID: "conn-1", DirectoryPath: path, ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin-1", Now: now} + if err := request.Validate(); err != nil { + t.Errorf("Validate(%q) error = %v", path, err) + } + } +} + +func TestFilenameFilterMutationRejectsUnsafePathAndUnnormalizedExtension(t *testing.T) { + now := time.Now() + for _, request := range []FilenameFilterMutation{ + {ConnectionID: "conn-1", DirectoryPath: "../private", ExcludedExtensions: []string{"zip"}, OperatorRef: "admin", Now: now}, + {ConnectionID: "conn-1", ExcludedExtensions: []string{".zip"}, OperatorRef: "admin", Now: now}, + {ConnectionID: "", ExcludedExtensions: []string{"zip"}, OperatorRef: "admin", Now: now}, + } { + if err := request.Validate(); err == nil { + t.Fatalf("Validate(%#v) succeeded", request) + } + } +} diff --git a/internal/httpapi/filename_filters.go b/internal/httpapi/filename_filters.go new file mode 100644 index 0000000..c21f147 --- /dev/null +++ b/internal/httpapi/filename_filters.go @@ -0,0 +1,81 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +type filenameFilterInput struct { + ConnectionID string `json:"connection_id"` + DirectoryPath string `json:"directory_path"` + ExcludedExtensions []string `json:"excluded_extensions"` +} + +func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) { + connectionID := strings.TrimSpace(r.URL.Query().Get("connection_id")) + if connectionID == "" || len(connectionID) > 36 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "connection_id is required"}) + return + } + rules, err := store.ListFilenameFilterRules(r.Context(), connectionID) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list filename filters"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"data": rules, "summary": map[string]int{"total": len(rules)}}) +} + +func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config, clear bool) { + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input filenameFilterInput + decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"}) + return + } + extensions := []string(nil) + if !clear { + var err error + extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"}) + return + } + } + request := governance.FilenameFilterMutation{ + ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath), + ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(), + } + if err := request.Validate(); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"}) + return + } + rule, replayed, err := config.Store.MutateFilenameFilterRuleWithOperation(r.Context(), request, operationID) + if errors.Is(err, postgresstore.ErrOperationRequestConflict) { + writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"}) + return + } + if errors.Is(err, governance.ErrFilenameFilterTargetNotFound) { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "filename filter target not found"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not update filename filter"}) + return + } + operation := "created" + if replayed { + operation = "replayed" + } + writeJSON(w, http.StatusOK, map[string]any{"data": rule, "operation": operation}) +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 87f416d..fde9d9c 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -83,6 +83,8 @@ type Store interface { ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error) MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error) MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error) + ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error) + MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error) StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error) StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error) @@ -252,6 +254,15 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("DELETE /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) { mutateScopeRule(w, r, config, true) }) + mux.HandleFunc("GET /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) { + listFilenameFilters(w, r, config.Store) + }) + mux.HandleFunc("PUT /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) { + mutateFilenameFilter(w, r, config, false) + }) + mux.HandleFunc("DELETE /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) { + mutateFilenameFilter(w, r, config, true) + }) mux.HandleFunc("GET /api/v1/source-connections", func(w http.ResponseWriter, r *http.Request) { listSourceConnections(w, r, config.Store) }) diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 678bfe7..a79b66b 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" "net/http/httptest" + "slices" "strings" "testing" "time" @@ -256,6 +257,26 @@ func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) { } } +func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) { + now := time.Date(2026, 9, 15, 2, 0, 0, 0, time.UTC) + store := &fakeJobStore{filenameFilterResult: governance.FilenameFilterRule{ + ConnectionID: "conn-safe", DirectoryPath: "固件", ExcludedExtensions: []string{"mp4", "rar", "zip"}, UpdatedBy: "engineer-1", UpdatedAt: now, + }} + handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"固件","excluded_extensions":[".ZIP","rar","Mp4","zip"]}`)) + request.Header.Set("Idempotency-Key", "op-filter-set") + handler.ServeHTTP(response, request) + if response.Code != http.StatusOK || store.filenameFilterOperationID != "op-filter-set" || + !slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"mp4", "rar", "zip"}) || + !strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) { + t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String()) + } +} + func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) { now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC) hash := strings.Repeat("c", 64) @@ -1515,6 +1536,12 @@ type fakeJobStore struct { scopeRuleOperationID string scopeRuleReplayed bool scopeRuleErr error + filenameFilters []governance.FilenameFilterRule + filenameFilterResult governance.FilenameFilterRule + filenameFilterRequest governance.FilenameFilterMutation + filenameFilterOperationID string + filenameFilterReplayed bool + filenameFilterErr error catalogSources []governance.CatalogSource catalogPage governance.CatalogNodePage catalogFilter governance.CatalogNodeFilter @@ -1818,6 +1845,16 @@ func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request g return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr } +func (s *fakeJobStore) ListFilenameFilterRules(_ context.Context, _ string) ([]governance.FilenameFilterRule, error) { + return s.filenameFilters, s.filenameFilterErr +} + +func (s *fakeJobStore) MutateFilenameFilterRuleWithOperation(_ context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) { + s.filenameFilterRequest = request + s.filenameFilterOperationID = operationID + return s.filenameFilterResult, s.filenameFilterReplayed, s.filenameFilterErr +} + func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) { return s.catalogSources, s.catalogErr } diff --git a/internal/platform/postgres/artifact_fetch_store.go b/internal/platform/postgres/artifact_fetch_store.go index 66e43b7..6eaea42 100644 --- a/internal/platform/postgres/artifact_fetch_store.go +++ b/internal/platform/postgres/artifact_fetch_store.go @@ -20,19 +20,30 @@ func loadArtifactFetchManifest(ctx context.Context, db interface { QueryRow(context.Context, string, ...any) pgx.Row }, connectionID, nodeID string) (artifactfetch.Manifest, error) { var manifest artifactfetch.Manifest + var fetchState string err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name, - intent.size_bytes,intent.modified_at,intent.locator_hash + intent.size_bytes,intent.modified_at,intent.locator_hash,intent.fetch_state FROM catalog_import_intents intent JOIN source_connections connection ON connection.id=intent.source_connection_id WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID). Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name, - &manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash) + &manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash, &fetchState) if errors.Is(err, pgx.ErrNoRows) { return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone } if err != nil { return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err) } + if fetchState == "filtered" { + return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered + } + var filtered bool + if err := db.QueryRow(ctx, `SELECT filename_is_filtered($1,$2,$3)`, connectionID, manifest.RelativePath, manifest.Name).Scan(&filtered); err != nil { + return artifactfetch.Manifest{}, fmt.Errorf("evaluate artifact filename filter: %w", err) + } + if filtered { + return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered + } manifest.ConnectionID = connectionID manifest.NodeID = nodeID return manifest, nil diff --git a/internal/platform/postgres/catalog_routing.go b/internal/platform/postgres/catalog_routing.go index 91ab2ab..8232b2a 100644 --- a/internal/platform/postgres/catalog_routing.go +++ b/internal/platform/postgres/catalog_routing.go @@ -117,6 +117,7 @@ SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.m FROM descendants JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id WHERE node.node_type='file' + AND NOT filename_is_filtered($1,node.relative_path,node.name) ORDER BY node.node_id`, request.ConnectionID, nodeIDs) if err != nil { return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err) diff --git a/internal/platform/postgres/filename_filters.go b/internal/platform/postgres/filename_filters.go new file mode 100644 index 0000000..84eb903 --- /dev/null +++ b/internal/platform/postgres/filename_filters.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error) { + if connectionID == "" || len(connectionID) > 36 { + return nil, governance.ErrInvalidFilenameFilter + } + rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at +FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID) + if err != nil { + return nil, fmt.Errorf("list filename filters: %w", err) + } + defer rows.Close() + result := make([]governance.FilenameFilterRule, 0) + for rows.Next() { + var rule governance.FilenameFilterRule + if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil { + return nil, fmt.Errorf("scan filename filter: %w", err) + } + result = append(result, rule) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate filename filters: %w", err) + } + return result, nil +} + +func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) { + if err := request.Validate(); err != nil { + return governance.FilenameFilterRule{}, false, err + } + fingerprint, err := operationFingerprint(struct { + ConnectionID string `json:"connection_id"` + DirectoryPath string `json:"directory_path"` + ExcludedExtensions []string `json:"excluded_extensions"` + OperatorRef string `json:"operator_ref"` + Clear bool `json:"clear"` + }{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear}) + if err != nil { + return governance.FilenameFilterRule{}, false, err + } + payload, replayed, err := s.executeOperation(ctx, "filename.filter.mutate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) { + rule, err := mutateFilenameFilterRuleTx(ctx, tx, request) + if err != nil { + return nil, err + } + return json.Marshal(rule) + }) + if err != nil { + return governance.FilenameFilterRule{}, false, err + } + var rule governance.FilenameFilterRule + if err := json.Unmarshal(payload, &rule); err != nil { + return governance.FilenameFilterRule{}, false, fmt.Errorf("decode filename filter result: %w", err) + } + return rule, replayed, nil +} + +func mutateFilenameFilterRuleTx(ctx context.Context, tx pgx.Tx, request governance.FilenameFilterMutation) (governance.FilenameFilterRule, error) { + var exists bool + if request.DirectoryPath == "" { + if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, request.ConnectionID).Scan(&exists); err != nil { + return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter source: %w", err) + } + } else { + if err := tx.QueryRow(ctx, `SELECT EXISTS( +SELECT 1 FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id +WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative_path=$2)`, request.ConnectionID, request.DirectoryPath).Scan(&exists); err != nil { + return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter directory: %w", err) + } + } + if !exists { + return governance.FilenameFilterRule{}, governance.ErrFilenameFilterTargetNotFound + } + if request.Clear { + if _, err := tx.Exec(ctx, `DELETE FROM filename_filter_rules WHERE source_connection_id=$1 AND directory_path=$2`, request.ConnectionID, request.DirectoryPath); err != nil { + return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err) + } + } else { + if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at) +VALUES($1,$2,$3,$4,$5,$5) +ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`, + request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil { + return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err) + } + } + if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil { + return governance.FilenameFilterRule{}, err + } + return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil +} + +func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error { + // Restore entries first, while fetch_state still identifies a previous filter. + if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state=intent.visibility,updated_at=$2 +FROM catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' AND intent.source_item_id=item.id + AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil { + return fmt.Errorf("restore filename-filtered items: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=NULL,updated_at=$2 +FROM source_item_versions version, catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' + AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id + AND publication.processing_state='planned'`, connectionID, now); err != nil { + return fmt.Errorf("restore filename-filtered publications: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='pending',attempt=0,available_at=$2,lease_owner=NULL,lease_until=NULL,last_error='',last_error_code='',updated_at=$2 +FROM catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' + AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id + AND job.payload->>'node_id'=intent.node_id AND job.state IN ('cancelled','failed','retry')`, connectionID, now); err != nil { + return fmt.Errorf("restore filename-filtered fetch jobs: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET +fetch_state=CASE WHEN EXISTS( + SELECT 1 FROM source_items item JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision + WHERE item.id=intent.source_item_id AND version.content_state='verified' +) THEN 'ready' ELSE 'pending' END,last_error='',updated_at=$2 +WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' + AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil { + return fmt.Errorf("restore filename-filtered intents: %w", err) + } + + if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=COALESCE(superseded_at,$2),updated_at=$2 +FROM source_item_versions version, catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id + AND publication.processing_state='planned' AND publication.superseded_at IS NULL`, connectionID, now); err != nil { + return fmt.Errorf("retire filename-filtered publications: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state='blocked',updated_at=$2 +FROM catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND intent.source_item_id=item.id`, connectionID, now); err != nil { + return fmt.Errorf("block filename-filtered items: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='cancelled',lease_owner=NULL,lease_until=NULL,last_error='filtered by filename rule',last_error_code='',updated_at=$2 +FROM catalog_import_intents intent +WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id + AND job.payload->>'node_id'=intent.node_id AND job.state IN ('pending','retry','failed')`, connectionID, now); err != nil { + return fmt.Errorf("cancel filename-filtered fetch jobs: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET fetch_state='filtered',last_error='',updated_at=$2 +WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name) + AND intent.fetch_state<>'ready'`, connectionID, now); err != nil { + return fmt.Errorf("mark filename-filtered intents: %w", err) + } + return nil +} diff --git a/internal/platform/postgres/filename_filters_integration_test.go b/internal/platform/postgres/filename_filters_integration_test.go new file mode 100644 index 0000000..11afbca --- /dev/null +++ b/internal/platform/postgres/filename_filters_integration_test.go @@ -0,0 +1,128 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +func TestFilenameFilterRuleHidesAndStopsMatchingArtifactsThenRestoresThem(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-filter','dingtalk','过滤测试',true); +INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin,policy_state) +VALUES('item-zip','conn-filter','zip',repeat('1',64),'固件.ZIP','file','r1','internal','configured_scope','internal'), + ('item-pdf','conn-filter','pdf',repeat('2',64),'手册.pdf','file','r1','internal','configured_scope','internal'); +INSERT INTO source_item_versions(id,source_item_id,revision,content_state) +VALUES('version-zip','item-zip','r1','unknown'),('version-pdf','item-pdf','r1','unknown'); +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-filter','conn-filter',repeat('a',64),'过滤测试',4,2,2,NOW()); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes) +VALUES('snapshot-filter','root-filter',NULL,'folder','过滤测试','',repeat('a',64),NULL,NULL,-1,1,2,30), + ('snapshot-filter','folder-filter','root-filter','folder','固件','固件',repeat('b',64),NULL,NULL,0,2,2,30), + ('snapshot-filter','node-zip','folder-filter','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),'item-zip',10,0,0,1,10), + ('snapshot-filter','node-pdf','folder-filter','file','手册.pdf','固件/手册.pdf',repeat('d',64),'item-pdf',20,1,0,1,20); +INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-filter','snapshot-filter',NOW()); +INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,source_item_id,decided_by,relative_path,name,locator_hash,fetch_state) +VALUES('conn-filter','node-zip','internal','item-zip','admin','固件/固件.ZIP','固件.ZIP',repeat('c',64),'pending'), + ('conn-filter','node-pdf','internal','item-pdf','admin','固件/手册.pdf','手册.pdf',repeat('d',64),'pending'); +INSERT INTO jobs(id,type,payload,state,idempotency_key) +VALUES('job-filter-zip','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-zip","version_id":"version-zip"}','retry','fetch-zip'), + ('job-filter-pdf','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-pdf","version_id":"version-pdf"}','pending','fetch-pdf'); +INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope) +VALUES('pub-filter-zip','version-zip','kb-test','planned','internal');`) + if err != nil { + t.Fatal(err) + } + + store := NewStore(pool) + now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC) + rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{ + ConnectionID: "conn-filter", DirectoryPath: "", ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin", Now: now, + }, "op-filter-set") + if err != nil || replayed || len(rule.ExcludedExtensions) != 3 { + t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err) + } + + var zipState, pdfState, zipJob, pdfJob, zipPolicy string + var superseded *time.Time + if err := pool.QueryRow(ctx, `SELECT + (SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'), + (SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-pdf'), + (SELECT state FROM jobs WHERE id='job-filter-zip'), + (SELECT state FROM jobs WHERE id='job-filter-pdf'), + (SELECT policy_state FROM source_items WHERE id='item-zip'), + (SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &pdfState, &zipJob, &pdfJob, &zipPolicy, &superseded); err != nil { + t.Fatal(err) + } + if zipState != "filtered" || pdfState != "pending" || zipJob != "cancelled" || pdfJob != "pending" || zipPolicy != "blocked" || superseded == nil { + t.Fatalf("filtered states zip=%s pdf=%s jobs=%s/%s policy=%s superseded=%v", zipState, pdfState, zipJob, pdfJob, zipPolicy, superseded) + } + page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-filter", ParentNodeID: "folder-filter", Limit: 50}) + if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].Name != "手册.pdf" || page.Current.SubtreeFileCount != 1 || page.Current.SubtreeBytes != 20 { + t.Fatalf("filtered page=%#v err=%v", page, err) + } + routed, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{ + ConnectionID: "conn-filter", NodeIDs: []string{"folder-filter"}, Visibility: governance.VisibilityInternal, + TargetKnowledgeBaseID: "kb-route", OperatorRef: "admin", Reason: "过滤后路由", Now: now, + }) + if err != nil || routed.SelectedFiles != 1 { + t.Fatalf("filtered route=%#v err=%v", routed, err) + } + _, err = pool.Exec(ctx, ` +INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at) +VALUES('snapshot-filter-old','conn-filter',repeat('e',64),'过滤测试',2,1,1,NOW()-INTERVAL '1 hour'); +INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,sort_order,direct_child_count,subtree_file_count,subtree_bytes) +VALUES('snapshot-filter-old','old-root',NULL,'folder','过滤测试','',repeat('e',64),-1,1,1,5), + ('snapshot-filter-old','old-zip','old-root','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),0,0,1,5); +INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at) +VALUES('change-filter','conn-filter','snapshot-filter-old','snapshot-filter',NOW()); +SELECT compare_catalog_change_set('change-filter');`) + if err != nil { + t.Fatal(err) + } + var changes, adds, updates int + var onlyTitle string + if err := pool.QueryRow(ctx, `SELECT change_set.add_count,change_set.update_count,COUNT(item.*),COALESCE(MAX(item.title),'') +FROM source_catalog_change_sets change_set LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id +WHERE change_set.id='change-filter' GROUP BY change_set.add_count,change_set.update_count`).Scan(&adds, &updates, &changes, &onlyTitle); err != nil { + t.Fatal(err) + } + if changes != 1 || adds != 1 || updates != 0 || onlyTitle != "手册.pdf" { + t.Fatalf("filtered diff changes=%d adds=%d updates=%d title=%q", changes, adds, updates, onlyTitle) + } + + _, _, err = store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{ + ConnectionID: "conn-filter", DirectoryPath: "", OperatorRef: "admin", Clear: true, Now: now.Add(time.Minute), + }, "op-filter-clear") + if err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT + (SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'), + (SELECT state FROM jobs WHERE id='job-filter-zip'), + (SELECT policy_state FROM source_items WHERE id='item-zip'), + (SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &zipJob, &zipPolicy, &superseded); err != nil { + t.Fatal(err) + } + if zipState != "pending" || zipJob != "pending" || zipPolicy != "internal" || superseded != nil { + t.Fatalf("restored states fetch=%s job=%s policy=%s superseded=%v", zipState, zipJob, zipPolicy, superseded) + } +} diff --git a/internal/platform/postgres/migrations/000035_filename_filters.down.sql b/internal/platform/postgres/migrations/000035_filename_filters.down.sql new file mode 100644 index 0000000..12f82aa --- /dev/null +++ b/internal/platform/postgres/migrations/000035_filename_filters.down.sql @@ -0,0 +1,13 @@ +DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR); +ALTER FUNCTION compare_catalog_change_set_unfiltered(VARCHAR) RENAME TO compare_catalog_change_set; + +UPDATE catalog_import_intents SET fetch_state='missing_local' WHERE fetch_state='filtered'; +ALTER TABLE catalog_import_intents + DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check; +ALTER TABLE catalog_import_intents + ADD CONSTRAINT catalog_import_intents_fetch_state_check + CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed')); + +DROP FUNCTION IF EXISTS filename_is_filtered(VARCHAR, TEXT, TEXT); +DROP FUNCTION IF EXISTS filename_extension(TEXT); +DROP TABLE IF EXISTS filename_filter_rules; diff --git a/internal/platform/postgres/migrations/000035_filename_filters.up.sql b/internal/platform/postgres/migrations/000035_filename_filters.up.sql new file mode 100644 index 0000000..6aa5a3d --- /dev/null +++ b/internal/platform/postgres/migrations/000035_filename_filters.up.sql @@ -0,0 +1,69 @@ +-- Source/directory filename filters. Rules are additive: the root path is the +-- source default and every matching ancestor directory contributes exclusions. +CREATE TABLE IF NOT EXISTS filename_filter_rules ( + source_connection_id VARCHAR(36) NOT NULL + REFERENCES source_connections(id) ON DELETE CASCADE, + directory_path TEXT NOT NULL DEFAULT '', + excluded_extensions TEXT[] NOT NULL, + updated_by VARCHAR(128) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source_connection_id, directory_path), + CHECK (cardinality(excluded_extensions) BETWEEN 1 AND 100) +); + +ALTER TABLE catalog_import_intents + DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check; +ALTER TABLE catalog_import_intents + ADD CONSTRAINT catalog_import_intents_fetch_state_check + CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed', 'filtered')); + +CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$ + SELECT CASE + WHEN COALESCE(p_name, '') ~ '\.[^.]+$' + THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$')) + ELSE '' + END +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION filename_is_filtered( + p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT +) RETURNS BOOLEAN AS $$ + SELECT EXISTS ( + SELECT 1 + FROM filename_filter_rules rule + CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension + WHERE rule.source_connection_id = p_connection_id + AND (rule.directory_path = '' + OR p_relative_path LIKE rule.directory_path || '/%') + AND extension = filename_extension(p_name) + ) +$$ LANGUAGE SQL STABLE PARALLEL SAFE; + +-- Keep the existing complete classifier as the implementation, then remove +-- filtered names before recounting. This preserves all move/scope semantics. +ALTER FUNCTION compare_catalog_change_set(VARCHAR) RENAME TO compare_catalog_change_set_unfiltered; +CREATE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$ +BEGIN + PERFORM compare_catalog_change_set_unfiltered(p_change_set_id); + + DELETE FROM source_catalog_change_items item + USING source_catalog_change_sets change_set + WHERE item.change_set_id = p_change_set_id + AND change_set.id = item.change_set_id + AND filename_is_filtered( + change_set.source_connection_id, + COALESCE(NULLIF(item.new_path, ''), item.old_path), + item.title + ); + + UPDATE source_catalog_change_sets change_set SET + add_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='add'), + update_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='update'), + move_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move'), + move_scope_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move_scope'), + delete_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='delete'), + no_change_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='no_change') + WHERE change_set.id = p_change_set_id; +END +$$ LANGUAGE plpgsql; diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 159e315..95dd0f6 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 34 { - t.Fatalf("migration count = %d, want 34", len(migrations)) + if len(migrations) != 35 { + t.Fatalf("migration count = %d, want 35", len(migrations)) } if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") { t.Fatalf("migration 1 = %#v", migrations[0]) @@ -101,6 +101,9 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if migrations[31].Version != 32 || !strings.Contains(migrations[31].UpSQL, "CREATE TABLE IF NOT EXISTS users") || !strings.Contains(migrations[31].UpSQL, "user_sessions") { t.Fatalf("migration 32 = %#v", migrations[31]) } + if migrations[34].Version != 35 || !strings.Contains(migrations[34].UpSQL, "CREATE TABLE IF NOT EXISTS filename_filter_rules") || !strings.Contains(migrations[34].UpSQL, "'filtered'") { + t.Fatalf("migration 35 = %#v", migrations[34]) + } } func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) { @@ -620,6 +623,7 @@ func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) { "case when attempt >= max_attempts then 'failed'", "else 'retry'", "lease_owner = null", + "select id from updated", }, }, } diff --git a/internal/platform/postgres/queue_sql.go b/internal/platform/postgres/queue_sql.go index 91f0f55..27a46bd 100644 --- a/internal/platform/postgres/queue_sql.go +++ b/internal/platform/postgres/queue_sql.go @@ -92,7 +92,7 @@ WITH updated AS ( COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv, COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt, state -) +), projected_failure AS ( INSERT INTO job_failures ( job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure, business_change_set_id, business_version_id, business_action, business_target_kb_id, @@ -117,6 +117,9 @@ SET publication_id = EXCLUDED.publication_id, resolution_actor_ref = '', resolution_reason = '', created_at = EXCLUDED.created_at +RETURNING job_id +) +SELECT id FROM updated ` const listJobsSQL = ` diff --git a/internal/platform/postgres/source_catalog.go b/internal/platform/postgres/source_catalog.go index ad252d2..049bd63 100644 --- a/internal/platform/postgres/source_catalog.go +++ b/internal/platform/postgres/source_catalog.go @@ -15,7 +15,11 @@ var ErrCatalogNodeNotFound = errors.New("catalog node not found") func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) { rows, err := s.db.Query(ctx, ` SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id, - snapshot.folder_count - 1, snapshot.file_count, head.activated_at, + snapshot.folder_count - 1, + (SELECT COUNT(*)::integer FROM source_catalog_nodes file + WHERE file.snapshot_id=snapshot.id AND file.node_type='file' + AND NOT filename_is_filtered(connection.id,file.relative_path,file.name)), + head.activated_at, 'ready', COALESCE(refresh.duration_ms,0) FROM source_catalog_heads head JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id @@ -62,8 +66,19 @@ func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogN page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset} var parentID *string err := s.db.QueryRow(ctx, ` -SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count, - node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at +SELECT node.node_id,node.parent_node_id,node.node_type,node.name, + (SELECT COUNT(*)::integer FROM source_catalog_nodes child + WHERE child.snapshot_id=node.snapshot_id AND child.parent_node_id=node.node_id + AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))), + (SELECT COUNT(*)::integer FROM source_catalog_nodes file + WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file' + AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%') + AND NOT filename_is_filtered($1,file.relative_path,file.name)), + (SELECT COALESCE(SUM(COALESCE(file.size_bytes,0)),0)::bigint FROM source_catalog_nodes file + WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file' + AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%') + AND NOT filename_is_filtered($1,file.relative_path,file.name)), + node.size_bytes,node.modified_at FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan( @@ -112,7 +127,8 @@ SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter if err := s.db.QueryRow(ctx, ` SELECT COUNT(*)::integer FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id -WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil { +WHERE head.source_connection_id=$1 AND node.parent_node_id=$2 + AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil { return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err) } @@ -122,6 +138,7 @@ WITH RECURSIVE current_head AS ( ), selected AS MATERIALIZED ( SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id WHERE node.parent_node_id=$2 + AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name)) ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id LIMIT $3 OFFSET $4 ), descendants(ancestor_id,snapshot_id,node_id) AS ( @@ -135,9 +152,10 @@ WITH RECURSIVE current_head AS ( COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility, publication.id AS publication_id,publication.ingest_state,publication.processing_state, current_version.content_state AS version_content_state,content.content_state AS object_content_state, - content.local_spool_ref + content.local_spool_ref,leaf.size_bytes FROM descendants JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file' + AND NOT filename_is_filtered($1,leaf.relative_path,leaf.name) LEFT JOIN source_items item ON item.id=leaf.source_item_id LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision LEFT JOIN content_objects content ON content.id=current_version.content_object_id @@ -158,6 +176,8 @@ WITH RECURSIVE current_head AS ( ) publication ON TRUE ), stats AS ( SELECT ancestor_id, + COUNT(*)::integer AS eligible_file_count, + COALESCE(SUM(COALESCE(size_bytes,0)),0)::bigint AS eligible_bytes, COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count, COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count, COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count, @@ -169,7 +189,12 @@ WITH RECURSIVE current_head AS ( FROM file_state GROUP BY ancestor_id ) SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name, - selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes, + CASE WHEN selected.node_type='file' THEN 0 ELSE ( + SELECT COUNT(*)::integer FROM source_catalog_nodes child + WHERE child.snapshot_id=selected.snapshot_id AND child.parent_node_id=selected.node_id + AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name)) + ) END, + COALESCE(stats.eligible_file_count,0),COALESCE(stats.eligible_bytes,0), selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'), COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0), COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0) diff --git a/internal/workflows/artifactfetch/handler.go b/internal/workflows/artifactfetch/handler.go index 93b10fc..db9bd1d 100644 --- a/internal/workflows/artifactfetch/handler.go +++ b/internal/workflows/artifactfetch/handler.go @@ -95,6 +95,9 @@ func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error { } manifest, err := handler.store.LoadArtifactFetchManifest(ctx, request.ConnectionID, request.NodeID) if err != nil { + if errors.Is(err, ErrManifestFiltered) { + return nil + } if errors.Is(err, ErrManifestGone) { return jobs.Permanent(jobs.ErrCodeStateConflict, err) } @@ -226,6 +229,7 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result } var ErrManifestGone = errors.New("artifact fetch manifest is gone") +var ErrManifestFiltered = errors.New("artifact fetch excluded by filename filter") // limitedWriter captures at most limit bytes of script stderr; full output is // never retained (it could echo source details we do not want in failures). diff --git a/internal/workflows/artifactfetch/handler_test.go b/internal/workflows/artifactfetch/handler_test.go new file mode 100644 index 0000000..0de30ec --- /dev/null +++ b/internal/workflows/artifactfetch/handler_test.go @@ -0,0 +1,42 @@ +package artifactfetch + +import ( + "context" + "encoding/json" + "testing" + "time" + + "knowledge-governance/internal/jobs" +) + +type filteredStore struct{} + +func (filteredStore) LoadArtifactFetchManifest(context.Context, string, string) (Manifest, error) { + return Manifest{}, ErrManifestFiltered +} +func (filteredStore) MarkArtifactFetched(context.Context, string, string, int64, string, string, bool, time.Time) error { + return nil +} +func (filteredStore) MarkArtifactFetchMissing(context.Context, string, string, time.Time) error { + return nil +} +func (filteredStore) MarkArtifactFetchFailed(context.Context, string, string, string, time.Time) error { + return nil +} + +type panicRunner struct{} + +func (panicRunner) Run(context.Context, Manifest) (Result, error) { + panic("filtered artifacts must not start a source fetch") +} + +func TestHandlerCompletesFilteredArtifactWithoutRunningFetcher(t *testing.T) { + handler, err := NewHandler(Config{Store: filteredStore{}, Runner: panicRunner{}}) + if err != nil { + t.Fatal(err) + } + payload, _ := json.Marshal(Request{ConnectionID: "conn-1", NodeID: "node-1", VersionID: "version-1"}) + if err := handler.Handle(context.Background(), jobs.Job{Payload: payload}); err != nil { + t.Fatalf("Handle() error = %v", err) + } +} diff --git a/scripts/artifact_fetch.py b/scripts/artifact_fetch.py index cb48b48..4c0c179 100644 --- a/scripts/artifact_fetch.py +++ b/scripts/artifact_fetch.py @@ -392,32 +392,112 @@ def dingtalk_download(manifest, max_bytes): return live_path +def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None): + request_headers = { + "a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""), + "cookie": headers.get("cookie", ""), "content-type": "application/json", + "accept": "application/json", "accept-language": "zh-CN", + "origin": "https://alidocs.dingtalk.com", "referer": headers["referer"], + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + } + request_headers.update(extra_headers or {}) + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None + request = urllib.request.Request(url, headers=request_headers, data=body, method=method) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + raise FetchError("DingTalk web API returned HTTP %d" % error.code, + retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code) + except (urllib.error.URLError, TimeoutError) as error: + raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable") + + +def dingtalk_put_bytes(url, payload): + request = urllib.request.Request(url, data=payload, method="PUT", headers={ + "content-length": str(len(payload)), "content-type": "", + "referer": "https://alidocs.dingtalk.com/", "user-agent": "Mozilla/5.0", + }) + try: + with urllib.request.urlopen(request, timeout=90) as response: + if response.status not in (200, 201, 204): + raise FetchError("DingTalk export upload returned HTTP %d" % response.status, + retryable=response.status >= 500, code="export_upload") + except urllib.error.HTTPError as error: + raise FetchError("DingTalk export upload returned HTTP %d" % error.code, + retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code) + + +def dingtalk_export_adoc(info, headers, manifest, max_bytes, spreadsheet=False): + doc_key = str(info.get("docKey") or "") + dentry_key = str(info.get("dentryKey") or "") + if not doc_key: + raise MissingSource("DingTalk online document has no docKey") + source_app = "spreadsheet" if spreadsheet else "doc" + doc_type = "sheet" if spreadsheet else "doc" + referer = ("https://alidocs.dingtalk.com/note/edit?dt_editor_toolbar=true&biz_ver=10" + "&docType=%s&docId=%s&dontjump=true" % (doc_type, urllib.parse.quote(doc_key))) + extra = {"referer": referer, "source_doc_app": source_app, + "a-doc-key": doc_key, "a-dentry-key": dentry_key} + document = dingtalk_json_api("https://alidocs.dingtalk.com/api/document/data", headers, + {"docKey": doc_key}, extra_headers=extra, method="POST") + checkpoint = (((document.get("data") or {}).get("documentContent") or {}).get("checkpoint") or {}) + content = checkpoint.get("content") or "" + if not isinstance(content, str) or not content: + raise FetchError("DingTalk online document content is empty", retryable=False, code="empty_document") + content_bytes = content.encode("utf-8") + if len(content_bytes) > max_bytes: + raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit") + upload = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/resources/9/upload_info", headers, + {"size": len(content_bytes), "resourceName": manifest["name"], "contentType": ""}, + extra_headers={**extra, "a-host-doc-key": ""}, method="POST") + upload_info = upload.get("data") or {} + storage_path, upload_url = upload_info.get("storagePath"), upload_info.get("uploadUrl") + if not storage_path or not upload_url: + raise FetchError("DingTalk export upload slot is missing", retryable=True, code="export_slot") + dingtalk_put_bytes(upload_url, content_bytes) + submit_body = {"storagePath": storage_path, + "exportType": "dingTalksheetToxlsx" if spreadsheet else "dingTalkdocTodocx"} + if spreadsheet: + submit_body["extra"] = {"imgCnt": 0} + submitted = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/submitExportJob", headers, + submit_body, extra_headers=extra, method="POST") + job_id = str((submitted.get("data") or {}).get("jobId") or "") + if not job_id: + raise FetchError("DingTalk export job was not created", retryable=True, code="export_submit") + deadline = time.time() + 120 + while time.time() < deadline: + status = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/queryExportJobInfo?jobId=" + urllib.parse.quote(job_id), + headers, extra_headers=extra) + detail = status.get("data") or {} + if detail.get("status") == "success" and detail.get("ossUrl"): + return http_get(detail["ossUrl"], {}, 180, max_bytes) + (False,) + if detail.get("status") in ("fail", "failed", "error"): + raise FetchError("DingTalk export job failed", retryable=False, code="export_failed") + time.sleep(3) + raise FetchError("DingTalk export job timed out", retryable=True, code="export_timeout") + + def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes): - opener = urllib.request.build_opener() - - def api(url, payload=None): - request = urllib.request.Request(url, headers={ - "a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""), - "cookie": headers.get("cookie", ""), "content-type": "application/json", - "referer": headers["referer"], "user-agent": "Mozilla/5.0", - }, data=json.dumps(payload).encode() if payload is not None else None) - try: - with opener.open(request, timeout=60) as response: - return json.loads(response.read().decode("utf-8")) - except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error: - raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable") - - info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key)) + info = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key), headers) data = info.get("data") or {} + extension = str(data.get("extension") or pathlib.PurePosixPath(manifest["name"]).suffix.lstrip(".")).lower() + if extension in ("adoc", "axls"): + return dingtalk_export_adoc(data, headers, manifest, max_bytes, spreadsheet=extension == "axls") space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "") if not space_id or not dentry_id: raise MissingSource("DingTalk dentry not resolvable") - download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id)) + download = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id), headers) payload = download.get("data") or {} if payload.get("downloadType") == "URL_PRE_SIGNATURE": urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or [] if urls: return http_get(urls[0], {}, 180, max_bytes) + (False,) + if payload.get("downloadType") == "HEADER_SIGNATURE": + signature = payload.get("headerSignatureInfo") or {} + urls = signature.get("resourceUrl") or [] + if urls: + return http_get(urls[0], signature.get("headers") or {}, 180, max_bytes) + (False,) return None diff --git a/scripts/artifact_fetch_test.py b/scripts/artifact_fetch_test.py new file mode 100644 index 0000000..a4022a7 --- /dev/null +++ b/scripts/artifact_fetch_test.py @@ -0,0 +1,47 @@ +import importlib.util +import pathlib +import unittest +from unittest import mock + +MODULE_PATH = pathlib.Path(__file__).with_name("artifact_fetch.py") +spec = importlib.util.spec_from_file_location("artifact_fetch", MODULE_PATH) +assert spec is not None and spec.loader is not None +artifact_fetch = importlib.util.module_from_spec(spec) +spec.loader.exec_module(artifact_fetch) + + +class DingTalkOnlineDocumentExportTest(unittest.TestCase): + def test_adoc_uses_export_job_instead_of_ordinary_file_download(self): + calls = [] + + def api(url, headers, payload=None, extra_headers=None, method=None): + calls.append((url, payload)) + if "/dentry/info" in url: + return {"data": {"docKey": "doc-key", "dentryKey": "dentry-key", "extension": "adoc"}} + if "/api/document/data" in url: + return {"data": {"documentContent": {"checkpoint": {"content": "document-json"}}}} + if "/upload_info" in url: + return {"data": {"storagePath": "tmp/path", "uploadUrl": "https://oss.example/upload"}} + if "/submitExportJob" in url: + return {"data": {"jobId": "job-1"}} + if "/queryExportJobInfo" in url: + return {"data": {"status": "success", "ossUrl": "https://oss.example/result.docx"}} + self.fail("unexpected API " + url) + + with mock.patch.object(artifact_fetch, "dingtalk_json_api", side_effect=api), \ + mock.patch.object(artifact_fetch, "dingtalk_put_bytes") as put, \ + mock.patch.object(artifact_fetch, "http_get", return_value=("/tmp/result", 123, "a" * 64)) as download: + result = artifact_fetch.live_export_dingtalk_document( + "raw-node", {"a-token": "x", "x-xsrf-token": "y", "cookie": "z", "referer": "https://alidocs.dingtalk.com/"}, + {"name": "说明.adoc", "relative_path": "说明.adoc"}, 1_000_000, + ) + + self.assertEqual(result, ("/tmp/result", 123, "a" * 64, False)) + self.assertTrue(any("submitExportJob" in url for url, _ in calls)) + self.assertFalse(any("/file/download" in url for url, _ in calls)) + put.assert_called_once() + download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/src/FilenameFilterEditor.test.tsx b/web/src/FilenameFilterEditor.test.tsx new file mode 100644 index 0000000..6be4d58 --- /dev/null +++ b/web/src/FilenameFilterEditor.test.tsx @@ -0,0 +1,26 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it } from 'vitest' + +import { FilenameFilterEditorView } from './FilenameFilterEditor' + +describe('FilenameFilterEditorView', () => { + it('shows source exclusions and inherited directory exclusions', () => { + const html = renderToStaticMarkup( undefined} + onSave={() => undefined} + onClear={() => undefined} + />) + expect(html).toContain('文件名筛选') + expect(html).toContain('.zip') + expect(html).toContain('.rar') + expect(html).toContain('.mp4') + expect(html).toContain('继承排除') + }) +}) diff --git a/web/src/FilenameFilterEditor.tsx b/web/src/FilenameFilterEditor.tsx new file mode 100644 index 0000000..d5a979f --- /dev/null +++ b/web/src/FilenameFilterEditor.tsx @@ -0,0 +1,90 @@ +import { useEffect, useMemo, useState } from 'react' + +import { clearFilenameFilter, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters' +import type { FilenameFilterRule } from './filenameFilters' +import './systemSettings.css' + +interface ViewProps { + label: string + directExtensions: string[] + inheritedExtensions: string[] + draft: string + busy: boolean + message: string | null + error: string | null + onDraft: (value: string) => void + onSave: () => void + onClear: () => void +} + +const tags = (values: string[]) => values.map((value) => .{value}) + +export function FilenameFilterEditorView({ label, directExtensions, inheritedExtensions, draft, busy, message, error, onDraft, onSave, onClear }: ViewProps) { + return
+
文件名筛选{label} · 按扩展名排除,大小写不敏感
+ {inheritedExtensions.length ?
继承排除{tags(inheritedExtensions)}
: null} + {directExtensions.length ?
本级排除{tags(directExtensions)}
: 本级尚未设置;只继承上级规则。} +
+ onDraft(event.target.value)} aria-label={`${label}排除扩展名`} /> + + {directExtensions.length ? : null} +
+ 保存后立即停止匹配文件的新取件与入库;原始来源文件和快照审计记录保留。 + {message ?

{message}

: null} + {error ?

{error}

: null} +
+} + +export function FilenameFilterEditor({ connectionID, directoryPath = '', label, onChanged }: { connectionID: string; directoryPath?: string; label: string; onChanged?: () => void }) { + const [rules, setRules] = useState([]) + const [draft, setDraft] = useState('zip, rar, mp4') + const [busy, setBusy] = useState(false) + const [message, setMessage] = useState(null) + const [error, setError] = useState(null) + const direct = rules.find((rule) => rule.directory_path === directoryPath) + const inherited = useMemo(() => { + const values = new Set() + for (const rule of rules) { + if (rule.directory_path === directoryPath) continue + if (rule.directory_path === '' || directoryPath.startsWith(`${rule.directory_path}/`)) { + for (const extension of rule.excluded_extensions) values.add(extension) + } + } + return [...values].sort() + }, [directoryPath, rules]) + + const load = async (signal?: AbortSignal) => { + const next = await fetchFilenameFilters(connectionID, signal) + setRules(next) + const current = next.find((rule) => rule.directory_path === directoryPath) + setDraft(current?.excluded_extensions.join(', ') || 'zip, rar, mp4') + } + useEffect(() => { + const controller = new AbortController() + void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件名筛选失败') }) + return () => controller.abort() + }, [connectionID, directoryPath]) + + const save = async () => { + const extensions = parseExtensionDraft(draft) + setBusy(true); setError(null); setMessage(null) + try { + await saveFilenameFilter(connectionID, directoryPath, extensions) + await load() + setMessage(`已排除 ${extensions.map((value) => `.${value}`).join('、')};目录、差异、取件和入库立即生效。`) + onChanged?.() + } catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件名筛选失败') } + finally { setBusy(false) } + } + const clear = async () => { + setBusy(true); setError(null); setMessage(null) + try { + await clearFilenameFilter(connectionID, directoryPath) + await load() + setMessage('已移除本级筛选,恢复继承上级规则。') + onChanged?.() + } catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件名筛选失败') } + finally { setBusy(false) } + } + return void save()} onClear={() => void clear()} /> +} diff --git a/web/src/SourceCatalogBrowser.tsx b/web/src/SourceCatalogBrowser.tsx index 9a1179f..4125411 100644 --- a/web/src/SourceCatalogBrowser.tsx +++ b/web/src/SourceCatalogBrowser.tsx @@ -6,6 +6,7 @@ import { triggerCatalogRefresh } from './sourceCatalog' import { fetchJobs } from './jobs' import type { Job } from './jobs' import { WorkspaceContextTabsSetter } from './workspaceContext' +import { FilenameFilterEditor } from './FilenameFilterEditor' import './sourceCatalogBrowser.css' const sourceLabels:Record={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'} @@ -20,9 +21,9 @@ interface ViewProps{ refreshJob?:Job|null onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void} - refreshHidden?:boolean;onToggleRefreshHidden?:()=>void + refreshHidden?:boolean;onToggleRefreshHidden?:()=>void;onFiltersChanged?:()=>void } -export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden}:ViewProps){ +export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden,onFiltersChanged}:ViewProps){ const [selectedIDs,setSelectedIDs]=useState([]) const [appliedPolicies,setAppliedPolicies]=useState>({}) const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1) @@ -65,6 +66,7 @@ export function SourceCatalogBrowserView({sources,selectedSource,page,loading,er
{page.current.name}{selectedSource.snapshot_only?'本地只读快照,不代表实时连接':'实时目录'}
{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}含所有子目录文件 {page.current.subtree_file_count}总大小 {bytes(page.current.subtree_bytes)}
+ item.name).join('/')} label={atSourceRoot?'整个来源':'当前目录'} onChanged={onFiltersChanged} /> {visibleItems.length?
@@ -107,5 +109,5 @@ export function SourceCatalogBrowser(){ const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)} const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,reason:`目录页策略:${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份;已记入意图 ${result.intents_recorded} 份;排队从源头取件 ${result.fetch_queued} 份;列入待入库 ${result.planned_publications} 份;标记禁止 ${result.blocked_items} 份;目录外无文件可标注 ${result.unbound_files} 份。到“解析监控 → 待入库”点开始处理即可。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}} const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}} - return {setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} /> + return {setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} onFiltersChanged={()=>setRevision(value=>value+1)} /> } diff --git a/web/src/SystemSettings.tsx b/web/src/SystemSettings.tsx index dc96540..a73c9f3 100644 --- a/web/src/SystemSettings.tsx +++ b/web/src/SystemSettings.tsx @@ -3,6 +3,7 @@ import { useCallback, useEffect, useState } from 'react' import { MembersSection } from './MembersSection' import { DingtalkSessionControl } from './DingtalkSessionControl' import { DingtalkSpaceRegistrar } from './DingtalkSpaceRegistrar' +import { FilenameFilterEditor } from './FilenameFilterEditor' import { RoutingBindingsSection } from './RoutingBindingsSection' import { useOptionalAuth } from './auth' import { fetchMonitorOverview } from './monitor' @@ -122,6 +123,7 @@ function SourceSettingsCard({ connection, onSourcesChanged }: { connection: Sour
:
} {connection.source_type === "dingtalk" && isAdmin ? : null} + {message ?

{message}

: null} diff --git a/web/src/filenameFilters.ts b/web/src/filenameFilters.ts new file mode 100644 index 0000000..9f45ae1 --- /dev/null +++ b/web/src/filenameFilters.ts @@ -0,0 +1,48 @@ +import { newOperationID } from './operationKey' + +export interface FilenameFilterRule { + connection_id: string + directory_path: string + excluded_extensions: string[] + updated_by: string + updated_at: string +} + +const record = (value: unknown): value is Record => typeof value === 'object' && value !== null + +export async function fetchFilenameFilters(connectionID: string, signal?: AbortSignal): Promise { + const query = new URLSearchParams({ connection_id: connectionID }) + const response = await fetch(`/api/v1/filename-filters?${query}`, { headers: { Accept: 'application/json' }, signal }) + if (!response.ok) throw new Error(`加载文件名筛选失败(HTTP ${response.status})`) + const envelope = await response.json() as unknown + if (!record(envelope) || !Array.isArray(envelope.data)) throw new Error('文件名筛选响应无效') + return envelope.data.map((value) => { + if (!record(value) || typeof value.connection_id !== 'string' || typeof value.directory_path !== 'string' || + !Array.isArray(value.excluded_extensions) || !value.excluded_extensions.every((item) => typeof item === 'string') || + typeof value.updated_by !== 'string' || typeof value.updated_at !== 'string') throw new Error('文件名筛选条目无效') + return value as unknown as FilenameFilterRule + }) +} + +export async function saveFilenameFilter(connectionID: string, directoryPath: string, extensions: string[]): Promise { + const response = await fetch('/api/v1/filename-filters', { + method: 'PUT', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() }, + body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, excluded_extensions: extensions }), + }) + if (!response.ok) throw new Error(`保存文件名筛选失败(HTTP ${response.status})`) + const envelope = await response.json() as { data?: FilenameFilterRule } + if (!envelope.data) throw new Error('文件名筛选响应无效') + return envelope.data +} + +export async function clearFilenameFilter(connectionID: string, directoryPath: string): Promise { + const response = await fetch('/api/v1/filename-filters', { + method: 'DELETE', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() }, + body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath }), + }) + if (!response.ok) throw new Error(`清除文件名筛选失败(HTTP ${response.status})`) +} + +export function parseExtensionDraft(value: string): string[] { + return [...new Set(value.split(/[,,\s]+/).map((item) => item.trim().toLowerCase().replace(/^\./, '')).filter(Boolean))].sort() +} diff --git a/web/src/systemSettings.css b/web/src/systemSettings.css index 84f271e..5e97afa 100644 --- a/web/src/systemSettings.css +++ b/web/src/systemSettings.css @@ -35,3 +35,8 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d .dingtalk-qr-loading span{width:42px;height:42px;border:4px solid #dbe4ee;border-top-color:var(--brand);border-radius:50%;animation:dingtalk-qr-spin .8s linear infinite} @keyframes dingtalk-qr-spin{to{transform:rotate(360deg)}} .dingtalk-qr-card .modal-actions{justify-content:center} +.filename-filter-editor{flex-basis:100%;margin-top:10px;padding:12px;border:1px solid var(--border-base);border-radius:10px;background:var(--bg-soft)} +.filename-filter-heading{display:flex;justify-content:space-between;gap:10px;margin-bottom:7px}.filename-filter-heading div{display:grid;gap:3px}.filename-filter-heading small{color:var(--ink-mute)} +.filename-filter-inherited,.filename-filter-direct{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin:6px 0}.filename-filter-inherited span,.filename-filter-direct span{font-size:10px;color:var(--ink-mute)} +.filename-filter-editor code{padding:2px 7px;border-radius:999px;background:#eaf2ff;color:#2456a6;font-family:inherit;font-size:11px} +.filename-filter-form{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:9px 0}.filename-filter-form input{min-width:220px;flex:1;padding:7px 9px;border:1px solid var(--border-strong);border-radius:7px;background:var(--bg-panel)}