feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
+23
View File
@@ -425,6 +425,29 @@
}
}
},
"/api/v1/filename-filters": {
"get": {
"tags": ["Source connections"],
"operationId": "listFilenameFilters",
"summary": "列出来源和目录的文件扩展名筛选规则",
"parameters": [{"name":"connection_id","in":"query","required":true,"schema":{"type":"string"}}],
"responses": {"200":{"description":"筛选规则"},"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"}}
},
"put": {
"tags": ["Source connections"],
"operationId": "setFilenameFilter",
"summary": "设置来源或目录文件扩展名筛选",
"parameters": [{"$ref":"#/components/parameters/IdempotencyKey"}],
"responses": {"200":{"description":"筛选规则已生效"},"400":{"$ref":"#/components/responses/BadRequest"},"404":{"$ref":"#/components/responses/NotFound"},"409":{"$ref":"#/components/responses/Conflict"},"500":{"$ref":"#/components/responses/InternalError"}}
},
"delete": {
"tags": ["Source connections"],
"operationId": "clearFilenameFilter",
"summary": "移除本级筛选并恢复继承",
"parameters": [{"$ref":"#/components/parameters/IdempotencyKey"}],
"responses": {"200":{"description":"本级筛选已移除"},"400":{"$ref":"#/components/responses/BadRequest"},"404":{"$ref":"#/components/responses/NotFound"},"409":{"$ref":"#/components/responses/Conflict"},"500":{"$ref":"#/components/responses/InternalError"}}
}
},
"/api/v1/scope-rules": {
"get": {
"tags": [
+86
View File
@@ -0,0 +1,86 @@
package governance
import (
"errors"
"path/filepath"
"sort"
"strings"
"time"
)
var (
ErrInvalidFilenameFilter = errors.New("invalid filename filter")
ErrFilenameFilterTargetNotFound = errors.New("filename filter target not found")
)
// FilenameFilterRule is additive: the source rule uses an empty directory path;
// a directory inherits that rule plus every matching ancestor rule.
type FilenameFilterRule struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
ExcludedExtensions []string `json:"excluded_extensions"`
UpdatedBy string `json:"updated_by"`
UpdatedAt time.Time `json:"updated_at"`
}
type FilenameFilterMutation struct {
ConnectionID string
DirectoryPath string
ExcludedExtensions []string
OperatorRef string
Clear bool
Now time.Time
}
func NormalizeExcludedExtensions(values []string) ([]string, error) {
if len(values) == 0 || len(values) > 100 {
return nil, errors.New("at least one and no more than 100 extensions are required")
}
seen := make(map[string]bool, len(values))
result := make([]string, 0, len(values))
for _, raw := range values {
value := strings.ToLower(strings.TrimSpace(raw))
value = strings.TrimPrefix(value, ".")
if value == "" || len(value) > 16 {
return nil, errors.New("file extension is invalid")
}
for _, character := range value {
if character >= 'a' && character <= 'z' || character >= '0' && character <= '9' || character == '_' || character == '-' {
continue
}
return nil, errors.New("file extension is invalid")
}
if !seen[value] {
seen[value] = true
result = append(result, value)
}
}
sort.Strings(result)
return result, nil
}
func (request FilenameFilterMutation) Validate() error {
if strings.TrimSpace(request.ConnectionID) == "" || len(request.ConnectionID) > 36 || !validScopePath(request.DirectoryPath) || strings.TrimSpace(request.OperatorRef) == "" || len(request.OperatorRef) > 128 || request.Now.IsZero() {
return errors.New("filename filter target is invalid")
}
if request.Clear {
if len(request.ExcludedExtensions) != 0 {
return errors.New("clear filename filter must not include extensions")
}
return nil
}
normalized, err := NormalizeExcludedExtensions(request.ExcludedExtensions)
if err != nil || len(normalized) != len(request.ExcludedExtensions) {
return errors.New("excluded extensions must be normalized and unique")
}
for index := range normalized {
if normalized[index] != request.ExcludedExtensions[index] {
return errors.New("excluded extensions must be normalized and sorted")
}
}
return nil
}
func FilenameExtension(name string) string {
return strings.TrimPrefix(strings.ToLower(filepath.Ext(name)), ".")
}
@@ -0,0 +1,49 @@
package governance
import (
"reflect"
"testing"
"time"
)
func TestNormalizeExcludedExtensionsCanonicalizesAndDeduplicates(t *testing.T) {
got, err := NormalizeExcludedExtensions([]string{".ZIP", " rar ", "Mp4", "zip"})
if err != nil {
t.Fatal(err)
}
want := []string{"mp4", "rar", "zip"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("NormalizeExcludedExtensions() = %#v, want %#v", got, want)
}
}
func TestNormalizeExcludedExtensionsRejectsFilenamePatternsAndUnsafeValues(t *testing.T) {
for _, input := range [][]string{{"*.zip"}, {"a/b"}, {"zip;rar"}, {""}, {"this-extension-is-far-too-long"}} {
if _, err := NormalizeExcludedExtensions(input); err == nil {
t.Fatalf("NormalizeExcludedExtensions(%#v) succeeded", input)
}
}
}
func TestFilenameFilterMutationAcceptsSourceAndNestedDirectory(t *testing.T) {
now := time.Now()
for _, path := range []string{"", "02 产品资料/固件"} {
request := FilenameFilterMutation{ConnectionID: "conn-1", DirectoryPath: path, ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin-1", Now: now}
if err := request.Validate(); err != nil {
t.Errorf("Validate(%q) error = %v", path, err)
}
}
}
func TestFilenameFilterMutationRejectsUnsafePathAndUnnormalizedExtension(t *testing.T) {
now := time.Now()
for _, request := range []FilenameFilterMutation{
{ConnectionID: "conn-1", DirectoryPath: "../private", ExcludedExtensions: []string{"zip"}, OperatorRef: "admin", Now: now},
{ConnectionID: "conn-1", ExcludedExtensions: []string{".zip"}, OperatorRef: "admin", Now: now},
{ConnectionID: "", ExcludedExtensions: []string{"zip"}, OperatorRef: "admin", Now: now},
} {
if err := request.Validate(); err == nil {
t.Fatalf("Validate(%#v) succeeded", request)
}
}
}
+81
View File
@@ -0,0 +1,81 @@
package httpapi
import (
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
type filenameFilterInput struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
ExcludedExtensions []string `json:"excluded_extensions"`
}
func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) {
connectionID := strings.TrimSpace(r.URL.Query().Get("connection_id"))
if connectionID == "" || len(connectionID) > 36 {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "connection_id is required"})
return
}
rules, err := store.ListFilenameFilterRules(r.Context(), connectionID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list filename filters"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"data": rules, "summary": map[string]int{"total": len(rules)}})
}
func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config, clear bool) {
operationID, ok := requireOperationID(w, r)
if !ok {
return
}
var input filenameFilterInput
decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
return
}
extensions := []string(nil)
if !clear {
var err error
extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"})
return
}
}
request := governance.FilenameFilterMutation{
ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath),
ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(),
}
if err := request.Validate(); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
return
}
rule, replayed, err := config.Store.MutateFilenameFilterRuleWithOperation(r.Context(), request, operationID)
if errors.Is(err, postgresstore.ErrOperationRequestConflict) {
writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"})
return
}
if errors.Is(err, governance.ErrFilenameFilterTargetNotFound) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "filename filter target not found"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not update filename filter"})
return
}
operation := "created"
if replayed {
operation = "replayed"
}
writeJSON(w, http.StatusOK, map[string]any{"data": rule, "operation": operation})
}
+11
View File
@@ -83,6 +83,8 @@ type Store interface {
ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error)
MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error)
MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error)
ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error)
MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error)
ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error)
StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error)
StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error)
@@ -252,6 +254,15 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("DELETE /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) {
mutateScopeRule(w, r, config, true)
})
mux.HandleFunc("GET /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
listFilenameFilters(w, r, config.Store)
})
mux.HandleFunc("PUT /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
mutateFilenameFilter(w, r, config, false)
})
mux.HandleFunc("DELETE /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
mutateFilenameFilter(w, r, config, true)
})
mux.HandleFunc("GET /api/v1/source-connections", func(w http.ResponseWriter, r *http.Request) {
listSourceConnections(w, r, config.Store)
})
+37
View File
@@ -7,6 +7,7 @@ import (
"errors"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
@@ -256,6 +257,26 @@ func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) {
}
}
func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) {
now := time.Date(2026, 9, 15, 2, 0, 0, 0, time.UTC)
store := &fakeJobStore{filenameFilterResult: governance.FilenameFilterRule{
ConnectionID: "conn-safe", DirectoryPath: "固件", ExcludedExtensions: []string{"mp4", "rar", "zip"}, UpdatedBy: "engineer-1", UpdatedAt: now,
}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }})
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"固件","excluded_extensions":[".ZIP","rar","Mp4","zip"]}`))
request.Header.Set("Idempotency-Key", "op-filter-set")
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK || store.filenameFilterOperationID != "op-filter-set" ||
!slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"mp4", "rar", "zip"}) ||
!strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String())
}
}
func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) {
now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC)
hash := strings.Repeat("c", 64)
@@ -1515,6 +1536,12 @@ type fakeJobStore struct {
scopeRuleOperationID string
scopeRuleReplayed bool
scopeRuleErr error
filenameFilters []governance.FilenameFilterRule
filenameFilterResult governance.FilenameFilterRule
filenameFilterRequest governance.FilenameFilterMutation
filenameFilterOperationID string
filenameFilterReplayed bool
filenameFilterErr error
catalogSources []governance.CatalogSource
catalogPage governance.CatalogNodePage
catalogFilter governance.CatalogNodeFilter
@@ -1818,6 +1845,16 @@ func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request g
return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr
}
func (s *fakeJobStore) ListFilenameFilterRules(_ context.Context, _ string) ([]governance.FilenameFilterRule, error) {
return s.filenameFilters, s.filenameFilterErr
}
func (s *fakeJobStore) MutateFilenameFilterRuleWithOperation(_ context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
s.filenameFilterRequest = request
s.filenameFilterOperationID = operationID
return s.filenameFilterResult, s.filenameFilterReplayed, s.filenameFilterErr
}
func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) {
return s.catalogSources, s.catalogErr
}
@@ -20,19 +20,30 @@ func loadArtifactFetchManifest(ctx context.Context, db interface {
QueryRow(context.Context, string, ...any) pgx.Row
}, connectionID, nodeID string) (artifactfetch.Manifest, error) {
var manifest artifactfetch.Manifest
var fetchState string
err := db.QueryRow(ctx, `SELECT connection.source_type,connection.name,intent.relative_path,intent.name,
intent.size_bytes,intent.modified_at,intent.locator_hash
intent.size_bytes,intent.modified_at,intent.locator_hash,intent.fetch_state
FROM catalog_import_intents intent
JOIN source_connections connection ON connection.id=intent.source_connection_id
WHERE intent.source_connection_id=$1 AND intent.node_id=$2`, connectionID, nodeID).
Scan(&manifest.SourceType, &manifest.SourceName, &manifest.RelativePath, &manifest.Name,
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash)
&manifest.SizeBytes, &manifest.ModifiedAt, &manifest.LocatorHash, &fetchState)
if errors.Is(err, pgx.ErrNoRows) {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestGone
}
if err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("load artifact fetch manifest: %w", err)
}
if fetchState == "filtered" {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
}
var filtered bool
if err := db.QueryRow(ctx, `SELECT filename_is_filtered($1,$2,$3)`, connectionID, manifest.RelativePath, manifest.Name).Scan(&filtered); err != nil {
return artifactfetch.Manifest{}, fmt.Errorf("evaluate artifact filename filter: %w", err)
}
if filtered {
return artifactfetch.Manifest{}, artifactfetch.ErrManifestFiltered
}
manifest.ConnectionID = connectionID
manifest.NodeID = nodeID
return manifest, nil
@@ -117,6 +117,7 @@ SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.m
FROM descendants
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
WHERE node.node_type='file'
AND NOT filename_is_filtered($1,node.relative_path,node.name)
ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
if err != nil {
return governance.CatalogRouteResult{}, fmt.Errorf("select catalog route files: %w", err)
@@ -0,0 +1,163 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"knowledge-governance/internal/governance"
)
func (s *Store) ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error) {
if connectionID == "" || len(connectionID) > 36 {
return nil, governance.ErrInvalidFilenameFilter
}
rows, err := s.db.Query(ctx, `SELECT source_connection_id,directory_path,excluded_extensions,updated_by,updated_at
FROM filename_filter_rules WHERE source_connection_id=$1 ORDER BY LENGTH(directory_path),directory_path`, connectionID)
if err != nil {
return nil, fmt.Errorf("list filename filters: %w", err)
}
defer rows.Close()
result := make([]governance.FilenameFilterRule, 0)
for rows.Next() {
var rule governance.FilenameFilterRule
if err := rows.Scan(&rule.ConnectionID, &rule.DirectoryPath, &rule.ExcludedExtensions, &rule.UpdatedBy, &rule.UpdatedAt); err != nil {
return nil, fmt.Errorf("scan filename filter: %w", err)
}
result = append(result, rule)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate filename filters: %w", err)
}
return result, nil
}
func (s *Store) MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
if err := request.Validate(); err != nil {
return governance.FilenameFilterRule{}, false, err
}
fingerprint, err := operationFingerprint(struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
ExcludedExtensions []string `json:"excluded_extensions"`
OperatorRef string `json:"operator_ref"`
Clear bool `json:"clear"`
}{request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Clear})
if err != nil {
return governance.FilenameFilterRule{}, false, err
}
payload, replayed, err := s.executeOperation(ctx, "filename.filter.mutate", operationID, fingerprint, request.Now, func(tx pgx.Tx) (json.RawMessage, error) {
rule, err := mutateFilenameFilterRuleTx(ctx, tx, request)
if err != nil {
return nil, err
}
return json.Marshal(rule)
})
if err != nil {
return governance.FilenameFilterRule{}, false, err
}
var rule governance.FilenameFilterRule
if err := json.Unmarshal(payload, &rule); err != nil {
return governance.FilenameFilterRule{}, false, fmt.Errorf("decode filename filter result: %w", err)
}
return rule, replayed, nil
}
func mutateFilenameFilterRuleTx(ctx context.Context, tx pgx.Tx, request governance.FilenameFilterMutation) (governance.FilenameFilterRule, error) {
var exists bool
if request.DirectoryPath == "" {
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM source_connections WHERE id=$1)`, request.ConnectionID).Scan(&exists); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter source: %w", err)
}
} else {
if err := tx.QueryRow(ctx, `SELECT EXISTS(
SELECT 1 FROM source_catalog_heads head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_type='folder' AND node.relative_path=$2)`, request.ConnectionID, request.DirectoryPath).Scan(&exists); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("inspect filename filter directory: %w", err)
}
}
if !exists {
return governance.FilenameFilterRule{}, governance.ErrFilenameFilterTargetNotFound
}
if request.Clear {
if _, err := tx.Exec(ctx, `DELETE FROM filename_filter_rules WHERE source_connection_id=$1 AND directory_path=$2`, request.ConnectionID, request.DirectoryPath); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("clear filename filter: %w", err)
}
} else {
if _, err := tx.Exec(ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,created_at,updated_at)
VALUES($1,$2,$3,$4,$5,$5)
ON CONFLICT(source_connection_id,directory_path) DO UPDATE SET excluded_extensions=EXCLUDED.excluded_extensions,updated_by=EXCLUDED.updated_by,updated_at=EXCLUDED.updated_at`,
request.ConnectionID, request.DirectoryPath, request.ExcludedExtensions, request.OperatorRef, request.Now); err != nil {
return governance.FilenameFilterRule{}, fmt.Errorf("write filename filter: %w", err)
}
}
if err := reconcileFilenameFilters(ctx, tx, request.ConnectionID, request.Now); err != nil {
return governance.FilenameFilterRule{}, err
}
return governance.FilenameFilterRule{ConnectionID: request.ConnectionID, DirectoryPath: request.DirectoryPath, ExcludedExtensions: request.ExcludedExtensions, UpdatedBy: request.OperatorRef, UpdatedAt: request.Now}, nil
}
func reconcileFilenameFilters(ctx context.Context, tx pgx.Tx, connectionID string, now time.Time) error {
// Restore entries first, while fetch_state still identifies a previous filter.
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state=intent.visibility,updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered' AND intent.source_item_id=item.id
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered items: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=NULL,updated_at=$2
FROM source_item_versions version, catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
AND publication.processing_state='planned'`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered publications: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='pending',attempt=0,available_at=$2,lease_owner=NULL,lease_until=NULL,last_error='',last_error_code='',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('cancelled','failed','retry')`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered fetch jobs: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET
fetch_state=CASE WHEN EXISTS(
SELECT 1 FROM source_items item JOIN source_item_versions version ON version.source_item_id=item.id AND version.revision=item.revision
WHERE item.id=intent.source_item_id AND version.content_state='verified'
) THEN 'ready' ELSE 'pending' END,last_error='',updated_at=$2
WHERE intent.source_connection_id=$1 AND intent.fetch_state='filtered'
AND NOT filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)`, connectionID, now); err != nil {
return fmt.Errorf("restore filename-filtered intents: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE test_publications publication SET superseded_at=COALESCE(superseded_at,$2),updated_at=$2
FROM source_item_versions version, catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND version.source_item_id=intent.source_item_id AND publication.source_item_version_id=version.id
AND publication.processing_state='planned' AND publication.superseded_at IS NULL`, connectionID, now); err != nil {
return fmt.Errorf("retire filename-filtered publications: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE source_items item SET policy_state='blocked',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND intent.source_item_id=item.id`, connectionID, now); err != nil {
return fmt.Errorf("block filename-filtered items: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE jobs job SET state='cancelled',lease_owner=NULL,lease_until=NULL,last_error='filtered by filename rule',last_error_code='',updated_at=$2
FROM catalog_import_intents intent
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND job.type='artifact.fetch' AND job.payload->>'source_connection_id'=intent.source_connection_id
AND job.payload->>'node_id'=intent.node_id AND job.state IN ('pending','retry','failed')`, connectionID, now); err != nil {
return fmt.Errorf("cancel filename-filtered fetch jobs: %w", err)
}
if _, err := tx.Exec(ctx, `UPDATE catalog_import_intents intent SET fetch_state='filtered',last_error='',updated_at=$2
WHERE intent.source_connection_id=$1 AND filename_is_filtered(intent.source_connection_id,intent.relative_path,intent.name)
AND intent.fetch_state<>'ready'`, connectionID, now); err != nil {
return fmt.Errorf("mark filename-filtered intents: %w", err)
}
return nil
}
@@ -0,0 +1,128 @@
package postgres
import (
"context"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/governance"
)
func TestFilenameFilterRuleHidesAndStopsMatchingArtifactsThenRestoresThem(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-filter','dingtalk','过滤测试',true);
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin,policy_state)
VALUES('item-zip','conn-filter','zip',repeat('1',64),'固件.ZIP','file','r1','internal','configured_scope','internal'),
('item-pdf','conn-filter','pdf',repeat('2',64),'手册.pdf','file','r1','internal','configured_scope','internal');
INSERT INTO source_item_versions(id,source_item_id,revision,content_state)
VALUES('version-zip','item-zip','r1','unknown'),('version-pdf','item-pdf','r1','unknown');
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-filter','conn-filter',repeat('a',64),'过滤测试',4,2,2,NOW());
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,source_item_id,size_bytes,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-filter','root-filter',NULL,'folder','过滤测试','',repeat('a',64),NULL,NULL,-1,1,2,30),
('snapshot-filter','folder-filter','root-filter','folder','固件','固件',repeat('b',64),NULL,NULL,0,2,2,30),
('snapshot-filter','node-zip','folder-filter','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),'item-zip',10,0,0,1,10),
('snapshot-filter','node-pdf','folder-filter','file','手册.pdf','固件/手册.pdf',repeat('d',64),'item-pdf',20,1,0,1,20);
INSERT INTO source_catalog_heads(source_connection_id,snapshot_id,activated_at) VALUES('conn-filter','snapshot-filter',NOW());
INSERT INTO catalog_import_intents(source_connection_id,node_id,visibility,source_item_id,decided_by,relative_path,name,locator_hash,fetch_state)
VALUES('conn-filter','node-zip','internal','item-zip','admin','固件/固件.ZIP','固件.ZIP',repeat('c',64),'pending'),
('conn-filter','node-pdf','internal','item-pdf','admin','固件/手册.pdf','手册.pdf',repeat('d',64),'pending');
INSERT INTO jobs(id,type,payload,state,idempotency_key)
VALUES('job-filter-zip','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-zip","version_id":"version-zip"}','retry','fetch-zip'),
('job-filter-pdf','artifact.fetch','{"source_connection_id":"conn-filter","node_id":"node-pdf","version_id":"version-pdf"}','pending','fetch-pdf');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,processing_state,target_scope)
VALUES('pub-filter-zip','version-zip','kb-test','planned','internal');`)
if err != nil {
t.Fatal(err)
}
store := NewStore(pool)
now := time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC)
rule, replayed, err := store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-filter", DirectoryPath: "", ExcludedExtensions: []string{"mp4", "rar", "zip"}, OperatorRef: "admin", Now: now,
}, "op-filter-set")
if err != nil || replayed || len(rule.ExcludedExtensions) != 3 {
t.Fatalf("set rule=%#v replayed=%v err=%v", rule, replayed, err)
}
var zipState, pdfState, zipJob, pdfJob, zipPolicy string
var superseded *time.Time
if err := pool.QueryRow(ctx, `SELECT
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-pdf'),
(SELECT state FROM jobs WHERE id='job-filter-zip'),
(SELECT state FROM jobs WHERE id='job-filter-pdf'),
(SELECT policy_state FROM source_items WHERE id='item-zip'),
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &pdfState, &zipJob, &pdfJob, &zipPolicy, &superseded); err != nil {
t.Fatal(err)
}
if zipState != "filtered" || pdfState != "pending" || zipJob != "cancelled" || pdfJob != "pending" || zipPolicy != "blocked" || superseded == nil {
t.Fatalf("filtered states zip=%s pdf=%s jobs=%s/%s policy=%s superseded=%v", zipState, pdfState, zipJob, pdfJob, zipPolicy, superseded)
}
page, err := store.ListCatalogNodes(ctx, governance.CatalogNodeFilter{ConnectionID: "conn-filter", ParentNodeID: "folder-filter", Limit: 50})
if err != nil || page.Total != 1 || len(page.Items) != 1 || page.Items[0].Name != "手册.pdf" || page.Current.SubtreeFileCount != 1 || page.Current.SubtreeBytes != 20 {
t.Fatalf("filtered page=%#v err=%v", page, err)
}
routed, err := store.RouteCatalogFiles(ctx, governance.CatalogRouteRequest{
ConnectionID: "conn-filter", NodeIDs: []string{"folder-filter"}, Visibility: governance.VisibilityInternal,
TargetKnowledgeBaseID: "kb-route", OperatorRef: "admin", Reason: "过滤后路由", Now: now,
})
if err != nil || routed.SelectedFiles != 1 {
t.Fatalf("filtered route=%#v err=%v", routed, err)
}
_, err = pool.Exec(ctx, `
INSERT INTO source_catalog_snapshots(id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
VALUES('snapshot-filter-old','conn-filter',repeat('e',64),'过滤测试',2,1,1,NOW()-INTERVAL '1 hour');
INSERT INTO source_catalog_nodes(snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash,sort_order,direct_child_count,subtree_file_count,subtree_bytes)
VALUES('snapshot-filter-old','old-root',NULL,'folder','过滤测试','',repeat('e',64),-1,1,1,5),
('snapshot-filter-old','old-zip','old-root','file','固件.ZIP','固件/固件.ZIP',repeat('c',64),0,0,1,5);
INSERT INTO source_catalog_change_sets(id,source_connection_id,from_snapshot_id,to_snapshot_id,created_at)
VALUES('change-filter','conn-filter','snapshot-filter-old','snapshot-filter',NOW());
SELECT compare_catalog_change_set('change-filter');`)
if err != nil {
t.Fatal(err)
}
var changes, adds, updates int
var onlyTitle string
if err := pool.QueryRow(ctx, `SELECT change_set.add_count,change_set.update_count,COUNT(item.*),COALESCE(MAX(item.title),'')
FROM source_catalog_change_sets change_set LEFT JOIN source_catalog_change_items item ON item.change_set_id=change_set.id
WHERE change_set.id='change-filter' GROUP BY change_set.add_count,change_set.update_count`).Scan(&adds, &updates, &changes, &onlyTitle); err != nil {
t.Fatal(err)
}
if changes != 1 || adds != 1 || updates != 0 || onlyTitle != "手册.pdf" {
t.Fatalf("filtered diff changes=%d adds=%d updates=%d title=%q", changes, adds, updates, onlyTitle)
}
_, _, err = store.MutateFilenameFilterRuleWithOperation(ctx, governance.FilenameFilterMutation{
ConnectionID: "conn-filter", DirectoryPath: "", OperatorRef: "admin", Clear: true, Now: now.Add(time.Minute),
}, "op-filter-clear")
if err != nil {
t.Fatal(err)
}
if err := pool.QueryRow(ctx, `SELECT
(SELECT fetch_state FROM catalog_import_intents WHERE node_id='node-zip'),
(SELECT state FROM jobs WHERE id='job-filter-zip'),
(SELECT policy_state FROM source_items WHERE id='item-zip'),
(SELECT superseded_at FROM test_publications WHERE id='pub-filter-zip')`).Scan(&zipState, &zipJob, &zipPolicy, &superseded); err != nil {
t.Fatal(err)
}
if zipState != "pending" || zipJob != "pending" || zipPolicy != "internal" || superseded != nil {
t.Fatalf("restored states fetch=%s job=%s policy=%s superseded=%v", zipState, zipJob, zipPolicy, superseded)
}
}
@@ -0,0 +1,13 @@
DROP FUNCTION IF EXISTS compare_catalog_change_set(VARCHAR);
ALTER FUNCTION compare_catalog_change_set_unfiltered(VARCHAR) RENAME TO compare_catalog_change_set;
UPDATE catalog_import_intents SET fetch_state='missing_local' WHERE fetch_state='filtered';
ALTER TABLE catalog_import_intents
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
ALTER TABLE catalog_import_intents
ADD CONSTRAINT catalog_import_intents_fetch_state_check
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed'));
DROP FUNCTION IF EXISTS filename_is_filtered(VARCHAR, TEXT, TEXT);
DROP FUNCTION IF EXISTS filename_extension(TEXT);
DROP TABLE IF EXISTS filename_filter_rules;
@@ -0,0 +1,69 @@
-- Source/directory filename filters. Rules are additive: the root path is the
-- source default and every matching ancestor directory contributes exclusions.
CREATE TABLE IF NOT EXISTS filename_filter_rules (
source_connection_id VARCHAR(36) NOT NULL
REFERENCES source_connections(id) ON DELETE CASCADE,
directory_path TEXT NOT NULL DEFAULT '',
excluded_extensions TEXT[] NOT NULL,
updated_by VARCHAR(128) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (source_connection_id, directory_path),
CHECK (cardinality(excluded_extensions) BETWEEN 1 AND 100)
);
ALTER TABLE catalog_import_intents
DROP CONSTRAINT IF EXISTS catalog_import_intents_fetch_state_check;
ALTER TABLE catalog_import_intents
ADD CONSTRAINT catalog_import_intents_fetch_state_check
CHECK (fetch_state IN ('pending', 'fetching', 'ready', 'missing_local', 'failed', 'filtered'));
CREATE OR REPLACE FUNCTION filename_extension(p_name TEXT) RETURNS TEXT AS $$
SELECT CASE
WHEN COALESCE(p_name, '') ~ '\.[^.]+$'
THEN LOWER(SUBSTRING(p_name FROM '\.([^.]+)$'))
ELSE ''
END
$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
CREATE OR REPLACE FUNCTION filename_is_filtered(
p_connection_id VARCHAR(36), p_relative_path TEXT, p_name TEXT
) RETURNS BOOLEAN AS $$
SELECT EXISTS (
SELECT 1
FROM filename_filter_rules rule
CROSS JOIN LATERAL UNNEST(rule.excluded_extensions) extension
WHERE rule.source_connection_id = p_connection_id
AND (rule.directory_path = ''
OR p_relative_path LIKE rule.directory_path || '/%')
AND extension = filename_extension(p_name)
)
$$ LANGUAGE SQL STABLE PARALLEL SAFE;
-- Keep the existing complete classifier as the implementation, then remove
-- filtered names before recounting. This preserves all move/scope semantics.
ALTER FUNCTION compare_catalog_change_set(VARCHAR) RENAME TO compare_catalog_change_set_unfiltered;
CREATE FUNCTION compare_catalog_change_set(p_change_set_id VARCHAR(36)) RETURNS VOID AS $$
BEGIN
PERFORM compare_catalog_change_set_unfiltered(p_change_set_id);
DELETE FROM source_catalog_change_items item
USING source_catalog_change_sets change_set
WHERE item.change_set_id = p_change_set_id
AND change_set.id = item.change_set_id
AND filename_is_filtered(
change_set.source_connection_id,
COALESCE(NULLIF(item.new_path, ''), item.old_path),
item.title
);
UPDATE source_catalog_change_sets change_set SET
add_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='add'),
update_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='update'),
move_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move'),
move_scope_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='move_scope'),
delete_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='delete'),
no_change_count = (SELECT COUNT(*) FROM source_catalog_change_items item WHERE item.change_set_id=p_change_set_id AND item.action='no_change')
WHERE change_set.id = p_change_set_id;
END
$$ LANGUAGE plpgsql;
@@ -11,8 +11,8 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 34 {
t.Fatalf("migration count = %d, want 34", len(migrations))
if len(migrations) != 35 {
t.Fatalf("migration count = %d, want 35", len(migrations))
}
if migrations[0].Version != 1 || !strings.Contains(migrations[0].UpSQL, "CREATE TABLE IF NOT EXISTS jobs") {
t.Fatalf("migration 1 = %#v", migrations[0])
@@ -101,6 +101,9 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if migrations[31].Version != 32 || !strings.Contains(migrations[31].UpSQL, "CREATE TABLE IF NOT EXISTS users") || !strings.Contains(migrations[31].UpSQL, "user_sessions") {
t.Fatalf("migration 32 = %#v", migrations[31])
}
if migrations[34].Version != 35 || !strings.Contains(migrations[34].UpSQL, "CREATE TABLE IF NOT EXISTS filename_filter_rules") || !strings.Contains(migrations[34].UpSQL, "'filtered'") {
t.Fatalf("migration 35 = %#v", migrations[34])
}
}
func TestProcessingConfirmationMigrationSeparatesPlanningFromUpload(t *testing.T) {
@@ -620,6 +623,7 @@ func TestMutationSQLPreservesIdempotencyAndLeaseOwnership(t *testing.T) {
"case when attempt >= max_attempts then 'failed'",
"else 'retry'",
"lease_owner = null",
"select id from updated",
},
},
}
+4 -1
View File
@@ -92,7 +92,7 @@ WITH updated AS (
COALESCE(business_change_set_id,'') AS bc, COALESCE(business_version_id,'') AS bv,
COALESCE(business_action,'') AS ba, COALESCE(business_target_kb_id,'') AS bt,
state
)
), projected_failure AS (
INSERT INTO job_failures (
job_id, job_type, publication_id, error_code, last_error, attempt, max_attempts_at_failure,
business_change_set_id, business_version_id, business_action, business_target_kb_id,
@@ -117,6 +117,9 @@ SET publication_id = EXCLUDED.publication_id,
resolution_actor_ref = '',
resolution_reason = '',
created_at = EXCLUDED.created_at
RETURNING job_id
)
SELECT id FROM updated
`
const listJobsSQL = `
+31 -6
View File
@@ -15,7 +15,11 @@ var ErrCatalogNodeNotFound = errors.New("catalog node not found")
func (s *Store) ListCatalogSources(ctx context.Context) ([]governance.CatalogSource, error) {
rows, err := s.db.Query(ctx, `
SELECT connection.id, connection.source_type, snapshot.source_label, root.node_id,
snapshot.folder_count - 1, snapshot.file_count, head.activated_at,
snapshot.folder_count - 1,
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
WHERE file.snapshot_id=snapshot.id AND file.node_type='file'
AND NOT filename_is_filtered(connection.id,file.relative_path,file.name)),
head.activated_at,
'ready', COALESCE(refresh.duration_ms,0)
FROM source_catalog_heads head
JOIN source_catalog_snapshots snapshot ON snapshot.id=head.snapshot_id AND snapshot.source_connection_id=head.source_connection_id
@@ -62,8 +66,19 @@ func (s *Store) ListCatalogNodes(ctx context.Context, filter governance.CatalogN
page := governance.CatalogNodePage{Items: []governance.CatalogNode{}, Limit: filter.Limit, Offset: filter.Offset}
var parentID *string
err := s.db.QueryRow(ctx, `
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,node.direct_child_count,
node.subtree_file_count,node.subtree_bytes,node.size_bytes,node.modified_at
SELECT node.node_id,node.parent_node_id,node.node_type,node.name,
(SELECT COUNT(*)::integer FROM source_catalog_nodes child
WHERE child.snapshot_id=node.snapshot_id AND child.parent_node_id=node.node_id
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))),
(SELECT COUNT(*)::integer FROM source_catalog_nodes file
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
(SELECT COALESCE(SUM(COALESCE(file.size_bytes,0)),0)::bigint FROM source_catalog_nodes file
WHERE file.snapshot_id=node.snapshot_id AND file.node_type='file'
AND (node.relative_path='' OR file.relative_path LIKE node.relative_path||'/%')
AND NOT filename_is_filtered($1,file.relative_path,file.name)),
node.size_bytes,node.modified_at
FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.node_id=$2 AND node.node_type='folder'`, filter.ConnectionID, filter.ParentNodeID).Scan(
@@ -112,7 +127,8 @@ SELECT node_id,name FROM trail ORDER BY depth DESC`, filter.ConnectionID, filter
if err := s.db.QueryRow(ctx, `
SELECT COUNT(*)::integer FROM source_catalog_heads head
JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
WHERE head.source_connection_id=$1 AND node.parent_node_id=$2
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))`, filter.ConnectionID, filter.ParentNodeID).Scan(&page.Total); err != nil {
return governance.CatalogNodePage{}, fmt.Errorf("count catalog nodes: %w", err)
}
@@ -122,6 +138,7 @@ WITH RECURSIVE current_head AS (
), selected AS MATERIALIZED (
SELECT node.* FROM current_head head JOIN source_catalog_nodes node ON node.snapshot_id=head.snapshot_id
WHERE node.parent_node_id=$2
AND (node.node_type='folder' OR NOT filename_is_filtered($1,node.relative_path,node.name))
ORDER BY CASE node.node_type WHEN 'folder' THEN 0 ELSE 1 END,node.sort_order,lower(node.name),node.node_id
LIMIT $3 OFFSET $4
), descendants(ancestor_id,snapshot_id,node_id) AS (
@@ -135,9 +152,10 @@ WITH RECURSIVE current_head AS (
COALESCE(item_scope.visibility,directory_scope.visibility,item.visibility) AS effective_visibility,
publication.id AS publication_id,publication.ingest_state,publication.processing_state,
current_version.content_state AS version_content_state,content.content_state AS object_content_state,
content.local_spool_ref
content.local_spool_ref,leaf.size_bytes
FROM descendants
JOIN source_catalog_nodes leaf ON leaf.snapshot_id=descendants.snapshot_id AND leaf.node_id=descendants.node_id AND leaf.node_type='file'
AND NOT filename_is_filtered($1,leaf.relative_path,leaf.name)
LEFT JOIN source_items item ON item.id=leaf.source_item_id
LEFT JOIN source_item_versions current_version ON current_version.source_item_id=item.id AND current_version.revision=item.revision
LEFT JOIN content_objects content ON content.id=current_version.content_object_id
@@ -158,6 +176,8 @@ WITH RECURSIVE current_head AS (
) publication ON TRUE
), stats AS (
SELECT ancestor_id,
COUNT(*)::integer AS eligible_file_count,
COALESCE(SUM(COALESCE(size_bytes,0)),0)::bigint AS eligible_bytes,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL)::integer AS imported_count,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND processing_state='completed')::integer AS parsed_count,
COUNT(DISTINCT source_item_id) FILTER (WHERE publication_id IS NOT NULL AND (ingest_state='uploading' OR processing_state IN ('processing','finalizing')))::integer AS active_count,
@@ -169,7 +189,12 @@ WITH RECURSIVE current_head AS (
FROM file_state GROUP BY ancestor_id
)
SELECT selected.node_id,selected.parent_node_id,selected.node_type,selected.name,
selected.direct_child_count,selected.subtree_file_count,selected.subtree_bytes,
CASE WHEN selected.node_type='file' THEN 0 ELSE (
SELECT COUNT(*)::integer FROM source_catalog_nodes child
WHERE child.snapshot_id=selected.snapshot_id AND child.parent_node_id=selected.node_id
AND (child.node_type='folder' OR NOT filename_is_filtered($1,child.relative_path,child.name))
) END,
COALESCE(stats.eligible_file_count,0),COALESCE(stats.eligible_bytes,0),
selected.size_bytes,selected.modified_at,selected.content_sha256,COALESCE(stats.visibility,'unknown'),
COALESCE(stats.imported_count,0),COALESCE(stats.parsed_count,0),
COALESCE(stats.active_count,0),COALESCE(stats.failed_count,0),COALESCE(stats.routable_count,0)
@@ -95,6 +95,9 @@ func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
}
manifest, err := handler.store.LoadArtifactFetchManifest(ctx, request.ConnectionID, request.NodeID)
if err != nil {
if errors.Is(err, ErrManifestFiltered) {
return nil
}
if errors.Is(err, ErrManifestGone) {
return jobs.Permanent(jobs.ErrCodeStateConflict, err)
}
@@ -226,6 +229,7 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result
}
var ErrManifestGone = errors.New("artifact fetch manifest is gone")
var ErrManifestFiltered = errors.New("artifact fetch excluded by filename filter")
// limitedWriter captures at most limit bytes of script stderr; full output is
// never retained (it could echo source details we do not want in failures).
@@ -0,0 +1,42 @@
package artifactfetch
import (
"context"
"encoding/json"
"testing"
"time"
"knowledge-governance/internal/jobs"
)
type filteredStore struct{}
func (filteredStore) LoadArtifactFetchManifest(context.Context, string, string) (Manifest, error) {
return Manifest{}, ErrManifestFiltered
}
func (filteredStore) MarkArtifactFetched(context.Context, string, string, int64, string, string, bool, time.Time) error {
return nil
}
func (filteredStore) MarkArtifactFetchMissing(context.Context, string, string, time.Time) error {
return nil
}
func (filteredStore) MarkArtifactFetchFailed(context.Context, string, string, string, time.Time) error {
return nil
}
type panicRunner struct{}
func (panicRunner) Run(context.Context, Manifest) (Result, error) {
panic("filtered artifacts must not start a source fetch")
}
func TestHandlerCompletesFilteredArtifactWithoutRunningFetcher(t *testing.T) {
handler, err := NewHandler(Config{Store: filteredStore{}, Runner: panicRunner{}})
if err != nil {
t.Fatal(err)
}
payload, _ := json.Marshal(Request{ConnectionID: "conn-1", NodeID: "node-1", VersionID: "version-1"})
if err := handler.Handle(context.Background(), jobs.Job{Payload: payload}); err != nil {
t.Fatalf("Handle() error = %v", err)
}
}
+96 -16
View File
@@ -392,32 +392,112 @@ def dingtalk_download(manifest, max_bytes):
return live_path
def dingtalk_json_api(url, headers, payload=None, extra_headers=None, method=None):
request_headers = {
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
"cookie": headers.get("cookie", ""), "content-type": "application/json",
"accept": "application/json", "accept-language": "zh-CN",
"origin": "https://alidocs.dingtalk.com", "referer": headers["referer"],
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}
request_headers.update(extra_headers or {})
body = json.dumps(payload, ensure_ascii=False).encode("utf-8") if payload is not None else None
request = urllib.request.Request(url, headers=request_headers, data=body, method=method)
try:
with urllib.request.urlopen(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as error:
raise FetchError("DingTalk web API returned HTTP %d" % error.code,
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
except (urllib.error.URLError, TimeoutError) as error:
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
def dingtalk_put_bytes(url, payload):
request = urllib.request.Request(url, data=payload, method="PUT", headers={
"content-length": str(len(payload)), "content-type": "",
"referer": "https://alidocs.dingtalk.com/", "user-agent": "Mozilla/5.0",
})
try:
with urllib.request.urlopen(request, timeout=90) as response:
if response.status not in (200, 201, 204):
raise FetchError("DingTalk export upload returned HTTP %d" % response.status,
retryable=response.status >= 500, code="export_upload")
except urllib.error.HTTPError as error:
raise FetchError("DingTalk export upload returned HTTP %d" % error.code,
retryable=error.code == 429 or error.code >= 500, code="http_%d" % error.code)
def dingtalk_export_adoc(info, headers, manifest, max_bytes, spreadsheet=False):
doc_key = str(info.get("docKey") or "")
dentry_key = str(info.get("dentryKey") or "")
if not doc_key:
raise MissingSource("DingTalk online document has no docKey")
source_app = "spreadsheet" if spreadsheet else "doc"
doc_type = "sheet" if spreadsheet else "doc"
referer = ("https://alidocs.dingtalk.com/note/edit?dt_editor_toolbar=true&biz_ver=10"
"&docType=%s&docId=%s&dontjump=true" % (doc_type, urllib.parse.quote(doc_key)))
extra = {"referer": referer, "source_doc_app": source_app,
"a-doc-key": doc_key, "a-dentry-key": dentry_key}
document = dingtalk_json_api("https://alidocs.dingtalk.com/api/document/data", headers,
{"docKey": doc_key}, extra_headers=extra, method="POST")
checkpoint = (((document.get("data") or {}).get("documentContent") or {}).get("checkpoint") or {})
content = checkpoint.get("content") or ""
if not isinstance(content, str) or not content:
raise FetchError("DingTalk online document content is empty", retryable=False, code="empty_document")
content_bytes = content.encode("utf-8")
if len(content_bytes) > max_bytes:
raise FetchError("source file exceeds ingest size limit", retryable=False, code="over_limit")
upload = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/resources/9/upload_info", headers,
{"size": len(content_bytes), "resourceName": manifest["name"], "contentType": ""},
extra_headers={**extra, "a-host-doc-key": ""}, method="POST")
upload_info = upload.get("data") or {}
storage_path, upload_url = upload_info.get("storagePath"), upload_info.get("uploadUrl")
if not storage_path or not upload_url:
raise FetchError("DingTalk export upload slot is missing", retryable=True, code="export_slot")
dingtalk_put_bytes(upload_url, content_bytes)
submit_body = {"storagePath": storage_path,
"exportType": "dingTalksheetToxlsx" if spreadsheet else "dingTalkdocTodocx"}
if spreadsheet:
submit_body["extra"] = {"imgCnt": 0}
submitted = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/submitExportJob", headers,
submit_body, extra_headers=extra, method="POST")
job_id = str((submitted.get("data") or {}).get("jobId") or "")
if not job_id:
raise FetchError("DingTalk export job was not created", retryable=True, code="export_submit")
deadline = time.time() + 120
while time.time() < deadline:
status = dingtalk_json_api("https://alidocs.dingtalk.com/core/api/document/queryExportJobInfo?jobId=" + urllib.parse.quote(job_id),
headers, extra_headers=extra)
detail = status.get("data") or {}
if detail.get("status") == "success" and detail.get("ossUrl"):
return http_get(detail["ossUrl"], {}, 180, max_bytes) + (False,)
if detail.get("status") in ("fail", "failed", "error"):
raise FetchError("DingTalk export job failed", retryable=False, code="export_failed")
time.sleep(3)
raise FetchError("DingTalk export job timed out", retryable=True, code="export_timeout")
def live_export_dingtalk_document(raw_key, headers, manifest, max_bytes):
opener = urllib.request.build_opener()
def api(url, payload=None):
request = urllib.request.Request(url, headers={
"a-token": headers["a-token"], "x-xsrf-token": headers.get("x-xsrf-token", ""),
"cookie": headers.get("cookie", ""), "content-type": "application/json",
"referer": headers["referer"], "user-agent": "Mozilla/5.0",
}, data=json.dumps(payload).encode() if payload is not None else None)
try:
with opener.open(request, timeout=60) as response:
return json.loads(response.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
raise FetchError("DingTalk web API unavailable (%s)" % type(error).__name__, code="source_unavailable")
info = api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key))
info = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v2/dentry/info?dentryUuid=" + urllib.parse.quote(raw_key), headers)
data = info.get("data") or {}
extension = str(data.get("extension") or pathlib.PurePosixPath(manifest["name"]).suffix.lstrip(".")).lower()
if extension in ("adoc", "axls"):
return dingtalk_export_adoc(data, headers, manifest, max_bytes, spreadsheet=extension == "axls")
space_id, dentry_id = str(data.get("spaceId") or ""), str(data.get("dentryId") or "")
if not space_id or not dentry_id:
raise MissingSource("DingTalk dentry not resolvable")
download = api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id))
download = dingtalk_json_api("https://alidocs.dingtalk.com/box/api/v1/file/download?spaceId=%s&dentryId=%s" % (space_id, dentry_id), headers)
payload = download.get("data") or {}
if payload.get("downloadType") == "URL_PRE_SIGNATURE":
urls = (payload.get("ossUrlPreSignatureInfo") or {}).get("preSignUrls") or []
if urls:
return http_get(urls[0], {}, 180, max_bytes) + (False,)
if payload.get("downloadType") == "HEADER_SIGNATURE":
signature = payload.get("headerSignatureInfo") or {}
urls = signature.get("resourceUrl") or []
if urls:
return http_get(urls[0], signature.get("headers") or {}, 180, max_bytes) + (False,)
return None
+47
View File
@@ -0,0 +1,47 @@
import importlib.util
import pathlib
import unittest
from unittest import mock
MODULE_PATH = pathlib.Path(__file__).with_name("artifact_fetch.py")
spec = importlib.util.spec_from_file_location("artifact_fetch", MODULE_PATH)
assert spec is not None and spec.loader is not None
artifact_fetch = importlib.util.module_from_spec(spec)
spec.loader.exec_module(artifact_fetch)
class DingTalkOnlineDocumentExportTest(unittest.TestCase):
def test_adoc_uses_export_job_instead_of_ordinary_file_download(self):
calls = []
def api(url, headers, payload=None, extra_headers=None, method=None):
calls.append((url, payload))
if "/dentry/info" in url:
return {"data": {"docKey": "doc-key", "dentryKey": "dentry-key", "extension": "adoc"}}
if "/api/document/data" in url:
return {"data": {"documentContent": {"checkpoint": {"content": "document-json"}}}}
if "/upload_info" in url:
return {"data": {"storagePath": "tmp/path", "uploadUrl": "https://oss.example/upload"}}
if "/submitExportJob" in url:
return {"data": {"jobId": "job-1"}}
if "/queryExportJobInfo" in url:
return {"data": {"status": "success", "ossUrl": "https://oss.example/result.docx"}}
self.fail("unexpected API " + url)
with mock.patch.object(artifact_fetch, "dingtalk_json_api", side_effect=api), \
mock.patch.object(artifact_fetch, "dingtalk_put_bytes") as put, \
mock.patch.object(artifact_fetch, "http_get", return_value=("/tmp/result", 123, "a" * 64)) as download:
result = artifact_fetch.live_export_dingtalk_document(
"raw-node", {"a-token": "x", "x-xsrf-token": "y", "cookie": "z", "referer": "https://alidocs.dingtalk.com/"},
{"name": "说明.adoc", "relative_path": "说明.adoc"}, 1_000_000,
)
self.assertEqual(result, ("/tmp/result", 123, "a" * 64, False))
self.assertTrue(any("submitExportJob" in url for url, _ in calls))
self.assertFalse(any("/file/download" in url for url, _ in calls))
put.assert_called_once()
download.assert_called_once_with("https://oss.example/result.docx", {}, 180, 1_000_000)
if __name__ == "__main__":
unittest.main()
+26
View File
@@ -0,0 +1,26 @@
import { renderToStaticMarkup } from 'react-dom/server'
import { describe, expect, it } from 'vitest'
import { FilenameFilterEditorView } from './FilenameFilterEditor'
describe('FilenameFilterEditorView', () => {
it('shows source exclusions and inherited directory exclusions', () => {
const html = renderToStaticMarkup(<FilenameFilterEditorView
label="当前目录"
directExtensions={['mp4']}
inheritedExtensions={['rar', 'zip']}
draft="mp4"
busy={false}
message={null}
error={null}
onDraft={() => undefined}
onSave={() => undefined}
onClear={() => undefined}
/>)
expect(html).toContain('文件名筛选')
expect(html).toContain('.zip')
expect(html).toContain('.rar')
expect(html).toContain('.mp4')
expect(html).toContain('继承排除')
})
})
+90
View File
@@ -0,0 +1,90 @@
import { useEffect, useMemo, useState } from 'react'
import { clearFilenameFilter, fetchFilenameFilters, parseExtensionDraft, saveFilenameFilter } from './filenameFilters'
import type { FilenameFilterRule } from './filenameFilters'
import './systemSettings.css'
interface ViewProps {
label: string
directExtensions: string[]
inheritedExtensions: string[]
draft: string
busy: boolean
message: string | null
error: string | null
onDraft: (value: string) => void
onSave: () => void
onClear: () => void
}
const tags = (values: string[]) => values.map((value) => <code key={value}>.{value}</code>)
export function FilenameFilterEditorView({ label, directExtensions, inheritedExtensions, draft, busy, message, error, onDraft, onSave, onClear }: ViewProps) {
return <section className="filename-filter-editor" aria-label={`${label}文件名筛选`}>
<div className="filename-filter-heading"><div><strong>文件名筛选</strong><small>{label} · 按扩展名排除,大小写不敏感</small></div></div>
{inheritedExtensions.length ? <div className="filename-filter-inherited"><span>继承排除</span>{tags(inheritedExtensions)}</div> : null}
{directExtensions.length ? <div className="filename-filter-direct"><span>本级排除</span>{tags(directExtensions)}</div> : <small>本级尚未设置;只继承上级规则。</small>}
<div className="filename-filter-form">
<input value={draft} placeholder="zip, rar, mp4" onChange={(event) => onDraft(event.target.value)} aria-label={`${label}排除扩展名`} />
<button className="secondary" disabled={busy || parseExtensionDraft(draft).length === 0} onClick={onSave}>{busy ? '保存中…' : '保存筛选'}</button>
{directExtensions.length ? <button className="link" disabled={busy} onClick={onClear}>移除本级筛选</button> : null}
</div>
<small>保存后立即停止匹配文件的新取件与入库;原始来源文件和快照审计记录保留。</small>
{message ? <p className="success-message" role="status">{message}</p> : null}
{error ? <p className="error-message" role="alert">{error}</p> : null}
</section>
}
export function FilenameFilterEditor({ connectionID, directoryPath = '', label, onChanged }: { connectionID: string; directoryPath?: string; label: string; onChanged?: () => void }) {
const [rules, setRules] = useState<FilenameFilterRule[]>([])
const [draft, setDraft] = useState('zip, rar, mp4')
const [busy, setBusy] = useState(false)
const [message, setMessage] = useState<string | null>(null)
const [error, setError] = useState<string | null>(null)
const direct = rules.find((rule) => rule.directory_path === directoryPath)
const inherited = useMemo(() => {
const values = new Set<string>()
for (const rule of rules) {
if (rule.directory_path === directoryPath) continue
if (rule.directory_path === '' || directoryPath.startsWith(`${rule.directory_path}/`)) {
for (const extension of rule.excluded_extensions) values.add(extension)
}
}
return [...values].sort()
}, [directoryPath, rules])
const load = async (signal?: AbortSignal) => {
const next = await fetchFilenameFilters(connectionID, signal)
setRules(next)
const current = next.find((rule) => rule.directory_path === directoryPath)
setDraft(current?.excluded_extensions.join(', ') || 'zip, rar, mp4')
}
useEffect(() => {
const controller = new AbortController()
void load(controller.signal).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载文件名筛选失败') })
return () => controller.abort()
}, [connectionID, directoryPath])
const save = async () => {
const extensions = parseExtensionDraft(draft)
setBusy(true); setError(null); setMessage(null)
try {
await saveFilenameFilter(connectionID, directoryPath, extensions)
await load()
setMessage(`已排除 ${extensions.map((value) => `.${value}`).join('、')};目录、差异、取件和入库立即生效。`)
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '保存文件名筛选失败') }
finally { setBusy(false) }
}
const clear = async () => {
setBusy(true); setError(null); setMessage(null)
try {
await clearFilenameFilter(connectionID, directoryPath)
await load()
setMessage('已移除本级筛选,恢复继承上级规则。')
onChanged?.()
} catch (cause) { setError(cause instanceof Error ? cause.message : '清除文件名筛选失败') }
finally { setBusy(false) }
}
return <FilenameFilterEditorView label={label} directExtensions={direct?.excluded_extensions || []} inheritedExtensions={inherited} draft={draft} busy={busy} message={message} error={error} onDraft={setDraft} onSave={() => void save()} onClear={() => void clear()} />
}
+5 -3
View File
@@ -6,6 +6,7 @@ import { triggerCatalogRefresh } from './sourceCatalog'
import { fetchJobs } from './jobs'
import type { Job } from './jobs'
import { WorkspaceContextTabsSetter } from './workspaceContext'
import { FilenameFilterEditor } from './FilenameFilterEditor'
import './sourceCatalogBrowser.css'
const sourceLabels:Record<string,string>={dingtalk:'钉钉知识库',pp:'PP资料下载中心',alist:'AList文件源'}
@@ -20,9 +21,9 @@ interface ViewProps{
refreshJob?:Job|null
onSelectSource:(source:CatalogSource)=>void;onOpenFolder:(node:CatalogNode)=>void;onNavigateTo:(nodeID:string)=>void;onBack:()=>void;onPageChange:(offset:number)=>void;onRefresh:()=>void;onRoute?:(nodeIDs:string[],policy:CatalogPolicy)=>Promise<boolean>
history?:{open:boolean;snapshots:CatalogSnapshot[];active:CatalogSnapshot|null;page:CatalogNodePage|null;loading:boolean;error:string|null;onOpen:()=>void;onClose:()=>void;onOpenSnapshot:(snapshot:CatalogSnapshot)=>void;onOpenFolder:(snapshot:CatalogSnapshot,nodeID:string)=>void}
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void
refreshHidden?:boolean;onToggleRefreshHidden?:()=>void;onFiltersChanged?:()=>void
}
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden}:ViewProps){
export function SourceCatalogBrowserView({sources,selectedSource,page,loading,error,routingCapabilities={internal_enabled:false,external_enabled:false},routingMessage='',routingLoading=false,refreshJob=null,onSelectSource,onOpenFolder,onNavigateTo,onBack,onPageChange,onRefresh,onRoute=async()=>false,history,refreshHidden=false,onToggleRefreshHidden,onFiltersChanged}:ViewProps){
const [selectedIDs,setSelectedIDs]=useState<string[]>([])
const [appliedPolicies,setAppliedPolicies]=useState<Record<string,CatalogPolicy>>({})
const atSourceRoot=Boolean(page&&page.breadcrumbs.length===1)
@@ -65,6 +66,7 @@ export function SourceCatalogBrowserView({sources,selectedSource,page,loading,er
<nav aria-label="目录面包屑">{page.breadcrumbs.map((crumb,index)=><span key={crumb.node_id}>{index>0?<i aria-hidden="true">/</i>:null}<button type="button" className={index===page.breadcrumbs.length-1?'current':''} disabled={index===page.breadcrumbs.length-1} onClick={()=>onNavigateTo(crumb.node_id)}>{crumb.name}</button></span>)}</nav>
</div>
<div className="catalog-summary"><div><strong>{page.current.name}</strong><small>{selectedSource.snapshot_only?'本地只读快照,不代表实时连接':'实时目录'}</small></div><span>{atSourceRoot?`第一层目录 ${page.items.filter(node=>node.node_type==='folder').length}`:`当前子项 ${page.total}`}</span><span>含所有子目录文件 {page.current.subtree_file_count}</span><span>总大小 {bytes(page.current.subtree_bytes)}</span></div>
<FilenameFilterEditor connectionID={selectedSource.connection_id} directoryPath={page.breadcrumbs.slice(1).map((item)=>item.name).join('/')} label={atSourceRoot?'整个来源':'当前目录'} onChanged={onFiltersChanged} />
{visibleItems.length?<section className="catalog-routing">
<div className="routing-actions">
@@ -107,5 +109,5 @@ export function SourceCatalogBrowser(){
const back=()=>{if(!page||page.breadcrumbs.length<2)return;setParentID(page.breadcrumbs[page.breadcrumbs.length-2].node_id);setOffset(0)}
const route=async(nodeIDs:string[],policy:CatalogPolicy):Promise<boolean>=>{if(!selected||routingLoading)return false;setRoutingLoading(true);setRoutingMessage('');try{const result=await routeCatalogFiles({connection_id:selected.connection_id,node_ids:nodeIDs,policy,reason:`目录页策略:${scope(policy)}`});setRoutingMessage(`策略已保存 ${result.selected_files} 份;已记入意图 ${result.intents_recorded} 份;排队从源头取件 ${result.fetch_queued} 份;列入待入库 ${result.planned_publications} 份;标记禁止 ${result.blocked_items} 份;目录外无文件可标注 ${result.unbound_files} 份。到“解析监控 → 待入库”点开始处理即可。`);setRevision(value=>value+1);return true}catch(cause){setRoutingMessage(cause instanceof Error?cause.message:'文件处理失败');return false}finally{setRoutingLoading(false)}}
const refresh=async()=>{if(!selected||refreshJob&&['pending','running','retry'].includes(refreshJob.state))return;setError(null);try{setRefreshJob(await triggerCatalogRefresh(selected.connection_id))}catch(cause){setError(cause instanceof Error?cause.message:'启动实时刷新失败')}}
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} />
return <SourceCatalogBrowserView sources={sources} selectedSource={selected} page={page} loading={loading} error={error} routingCapabilities={routingCapabilities} routingMessage={routingMessage} routingLoading={routingLoading} refreshJob={refreshJob} onSelectSource={source=>{setRoutingMessage('');selectSource(source)}} onOpenFolder={node=>{setRoutingMessage('');setParentID(node.node_id);setOffset(0)}} onNavigateTo={nodeID=>{setRoutingMessage('');setParentID(nodeID);setOffset(0)}} onBack={back} onPageChange={setOffset} onRefresh={()=>void refresh()} onRoute={(nodeIDs,visibility)=>route(nodeIDs,visibility)} history={{open:historyOpen,snapshots:historySnapshots,active:historyActive,page:historyPage,loading:historyLoading,error:historyError,onOpen:()=>void openHistory(),onClose:closeHistory,onOpenSnapshot:snapshot=>void openHistorySnapshot(snapshot),onOpenFolder:(snapshot,nodeID)=>void openHistoryFolder(snapshot,nodeID)}} refreshHidden={refreshHidden} onToggleRefreshHidden={()=>setRefreshHidden(value=>!value)} onFiltersChanged={()=>setRevision(value=>value+1)} />
}
+2
View File
@@ -3,6 +3,7 @@ import { useCallback, useEffect, useState } from 'react'
import { MembersSection } from './MembersSection'
import { DingtalkSessionControl } from './DingtalkSessionControl'
import { DingtalkSpaceRegistrar } from './DingtalkSpaceRegistrar'
import { FilenameFilterEditor } from './FilenameFilterEditor'
import { RoutingBindingsSection } from './RoutingBindingsSection'
import { useOptionalAuth } from './auth'
import { fetchMonitorOverview } from './monitor'
@@ -122,6 +123,7 @@ function SourceSettingsCard({ connection, onSourcesChanged }: { connection: Sour
<button className="danger" disabled={!canApply} onClick={() => setPendingDisable(true)}>停用来源</button></div>
: <div className="source-schedule-actions"><button className="secondary" disabled={!canApply} onClick={() => void submit(true)}>保存并启用</button></div>}
{connection.source_type === "dingtalk" && isAdmin ? <DingtalkSessionControl /> : null}
<FilenameFilterEditor connectionID={connection.id} label="整个来源" onChanged={onSourcesChanged} />
<button className="link" onClick={() => void loadHistory()}>{showHistory ? '收起修订历史' : '查看修订历史'}</button>
</div>
{message ? <p className="success-message" role="status">{message}</p> : null}
+48
View File
@@ -0,0 +1,48 @@
import { newOperationID } from './operationKey'
export interface FilenameFilterRule {
connection_id: string
directory_path: string
excluded_extensions: string[]
updated_by: string
updated_at: string
}
const record = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
export async function fetchFilenameFilters(connectionID: string, signal?: AbortSignal): Promise<FilenameFilterRule[]> {
const query = new URLSearchParams({ connection_id: connectionID })
const response = await fetch(`/api/v1/filename-filters?${query}`, { headers: { Accept: 'application/json' }, signal })
if (!response.ok) throw new Error(`加载文件名筛选失败(HTTP ${response.status})`)
const envelope = await response.json() as unknown
if (!record(envelope) || !Array.isArray(envelope.data)) throw new Error('文件名筛选响应无效')
return envelope.data.map((value) => {
if (!record(value) || typeof value.connection_id !== 'string' || typeof value.directory_path !== 'string' ||
!Array.isArray(value.excluded_extensions) || !value.excluded_extensions.every((item) => typeof item === 'string') ||
typeof value.updated_by !== 'string' || typeof value.updated_at !== 'string') throw new Error('文件名筛选条目无效')
return value as unknown as FilenameFilterRule
})
}
export async function saveFilenameFilter(connectionID: string, directoryPath: string, extensions: string[]): Promise<FilenameFilterRule> {
const response = await fetch('/api/v1/filename-filters', {
method: 'PUT', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath, excluded_extensions: extensions }),
})
if (!response.ok) throw new Error(`保存文件名筛选失败(HTTP ${response.status})`)
const envelope = await response.json() as { data?: FilenameFilterRule }
if (!envelope.data) throw new Error('文件名筛选响应无效')
return envelope.data
}
export async function clearFilenameFilter(connectionID: string, directoryPath: string): Promise<void> {
const response = await fetch('/api/v1/filename-filters', {
method: 'DELETE', headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
body: JSON.stringify({ connection_id: connectionID, directory_path: directoryPath }),
})
if (!response.ok) throw new Error(`清除文件名筛选失败(HTTP ${response.status})`)
}
export function parseExtensionDraft(value: string): string[] {
return [...new Set(value.split(/[,,\s]+/).map((item) => item.trim().toLowerCase().replace(/^\./, '')).filter(Boolean))].sort()
}
+5
View File
@@ -35,3 +35,8 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d
.dingtalk-qr-loading span{width:42px;height:42px;border:4px solid #dbe4ee;border-top-color:var(--brand);border-radius:50%;animation:dingtalk-qr-spin .8s linear infinite}
@keyframes dingtalk-qr-spin{to{transform:rotate(360deg)}}
.dingtalk-qr-card .modal-actions{justify-content:center}
.filename-filter-editor{flex-basis:100%;margin-top:10px;padding:12px;border:1px solid var(--border-base);border-radius:10px;background:var(--bg-soft)}
.filename-filter-heading{display:flex;justify-content:space-between;gap:10px;margin-bottom:7px}.filename-filter-heading div{display:grid;gap:3px}.filename-filter-heading small{color:var(--ink-mute)}
.filename-filter-inherited,.filename-filter-direct{display:flex;align-items:center;gap:6px;flex-wrap:wrap;margin:6px 0}.filename-filter-inherited span,.filename-filter-direct span{font-size:10px;color:var(--ink-mute)}
.filename-filter-editor code{padding:2px 7px;border-radius:999px;background:#eaf2ff;color:#2456a6;font-family:inherit;font-size:11px}
.filename-filter-form{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:9px 0}.filename-filter-form input{min-width:220px;flex:1;padding:7px 9px;border:1px solid var(--border-strong);border-radius:7px;background:var(--bg-panel)}