feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
+81
View File
@@ -0,0 +1,81 @@
package httpapi
import (
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"knowledge-governance/internal/governance"
postgresstore "knowledge-governance/internal/platform/postgres"
)
type filenameFilterInput struct {
ConnectionID string `json:"connection_id"`
DirectoryPath string `json:"directory_path"`
ExcludedExtensions []string `json:"excluded_extensions"`
}
func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) {
connectionID := strings.TrimSpace(r.URL.Query().Get("connection_id"))
if connectionID == "" || len(connectionID) > 36 {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "connection_id is required"})
return
}
rules, err := store.ListFilenameFilterRules(r.Context(), connectionID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list filename filters"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"data": rules, "summary": map[string]int{"total": len(rules)}})
}
func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config, clear bool) {
operationID, ok := requireOperationID(w, r)
if !ok {
return
}
var input filenameFilterInput
decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&input); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
return
}
extensions := []string(nil)
if !clear {
var err error
extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"})
return
}
}
request := governance.FilenameFilterMutation{
ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath),
ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(),
}
if err := request.Validate(); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
return
}
rule, replayed, err := config.Store.MutateFilenameFilterRuleWithOperation(r.Context(), request, operationID)
if errors.Is(err, postgresstore.ErrOperationRequestConflict) {
writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"})
return
}
if errors.Is(err, governance.ErrFilenameFilterTargetNotFound) {
writeJSON(w, http.StatusNotFound, map[string]string{"error": "filename filter target not found"})
return
}
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not update filename filter"})
return
}
operation := "created"
if replayed {
operation = "replayed"
}
writeJSON(w, http.StatusOK, map[string]any{"data": rule, "operation": operation})
}
+11
View File
@@ -83,6 +83,8 @@ type Store interface {
ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error)
MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error)
MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error)
ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error)
MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error)
ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error)
StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error)
StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error)
@@ -252,6 +254,15 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("DELETE /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) {
mutateScopeRule(w, r, config, true)
})
mux.HandleFunc("GET /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
listFilenameFilters(w, r, config.Store)
})
mux.HandleFunc("PUT /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
mutateFilenameFilter(w, r, config, false)
})
mux.HandleFunc("DELETE /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
mutateFilenameFilter(w, r, config, true)
})
mux.HandleFunc("GET /api/v1/source-connections", func(w http.ResponseWriter, r *http.Request) {
listSourceConnections(w, r, config.Store)
})
+37
View File
@@ -7,6 +7,7 @@ import (
"errors"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
@@ -256,6 +257,26 @@ func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) {
}
}
func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) {
now := time.Date(2026, 9, 15, 2, 0, 0, 0, time.UTC)
store := &fakeJobStore{filenameFilterResult: governance.FilenameFilterRule{
ConnectionID: "conn-safe", DirectoryPath: "固件", ExcludedExtensions: []string{"mp4", "rar", "zip"}, UpdatedBy: "engineer-1", UpdatedAt: now,
}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }})
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"固件","excluded_extensions":[".ZIP","rar","Mp4","zip"]}`))
request.Header.Set("Idempotency-Key", "op-filter-set")
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK || store.filenameFilterOperationID != "op-filter-set" ||
!slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"mp4", "rar", "zip"}) ||
!strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String())
}
}
func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) {
now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC)
hash := strings.Repeat("c", 64)
@@ -1515,6 +1536,12 @@ type fakeJobStore struct {
scopeRuleOperationID string
scopeRuleReplayed bool
scopeRuleErr error
filenameFilters []governance.FilenameFilterRule
filenameFilterResult governance.FilenameFilterRule
filenameFilterRequest governance.FilenameFilterMutation
filenameFilterOperationID string
filenameFilterReplayed bool
filenameFilterErr error
catalogSources []governance.CatalogSource
catalogPage governance.CatalogNodePage
catalogFilter governance.CatalogNodeFilter
@@ -1818,6 +1845,16 @@ func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request g
return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr
}
func (s *fakeJobStore) ListFilenameFilterRules(_ context.Context, _ string) ([]governance.FilenameFilterRule, error) {
return s.filenameFilters, s.filenameFilterErr
}
func (s *fakeJobStore) MutateFilenameFilterRuleWithOperation(_ context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
s.filenameFilterRequest = request
s.filenameFilterOperationID = operationID
return s.filenameFilterResult, s.filenameFilterReplayed, s.filenameFilterErr
}
func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) {
return s.catalogSources, s.catalogErr
}