feat(ingest): 源与目录文件名过滤及钉钉文档导出
- filename_filter_rules table (000035) with source-root and per-directory rows - additive inheritance: source default + every ancestor directory contributes - filename_is_filtered() SQL predicate applied to catalog browsing, subtree counts, change-set diffing, routing fan-out and artifact fetch manifests - fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch jobs, retires planned publications and blocks items when rules change - GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized extensions) + Settings and Catalog filter editors - failJobSQL no longer swallows the claimed row when the failure projection upserts (fixes spurious 'lease lost' retries) - DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403 from file/download fixed), header-signature downloads supported
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
type filenameFilterInput struct {
|
||||
ConnectionID string `json:"connection_id"`
|
||||
DirectoryPath string `json:"directory_path"`
|
||||
ExcludedExtensions []string `json:"excluded_extensions"`
|
||||
}
|
||||
|
||||
func listFilenameFilters(w http.ResponseWriter, r *http.Request, store Store) {
|
||||
connectionID := strings.TrimSpace(r.URL.Query().Get("connection_id"))
|
||||
if connectionID == "" || len(connectionID) > 36 {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "connection_id is required"})
|
||||
return
|
||||
}
|
||||
rules, err := store.ListFilenameFilterRules(r.Context(), connectionID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list filename filters"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": rules, "summary": map[string]int{"total": len(rules)}})
|
||||
}
|
||||
|
||||
func mutateFilenameFilter(w http.ResponseWriter, r *http.Request, config Config, clear bool) {
|
||||
operationID, ok := requireOperationID(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var input filenameFilterInput
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<16))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&input); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
|
||||
return
|
||||
}
|
||||
extensions := []string(nil)
|
||||
if !clear {
|
||||
var err error
|
||||
extensions, err = governance.NormalizeExcludedExtensions(input.ExcludedExtensions)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid excluded extensions"})
|
||||
return
|
||||
}
|
||||
}
|
||||
request := governance.FilenameFilterMutation{
|
||||
ConnectionID: strings.TrimSpace(input.ConnectionID), DirectoryPath: strings.TrimSpace(input.DirectoryPath),
|
||||
ExcludedExtensions: extensions, OperatorRef: actorFor(r), Clear: clear, Now: config.Now().UTC(),
|
||||
}
|
||||
if err := request.Validate(); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid filename filter request"})
|
||||
return
|
||||
}
|
||||
rule, replayed, err := config.Store.MutateFilenameFilterRuleWithOperation(r.Context(), request, operationID)
|
||||
if errors.Is(err, postgresstore.ErrOperationRequestConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "Idempotency-Key conflicts with a different request"})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, governance.ErrFilenameFilterTargetNotFound) {
|
||||
writeJSON(w, http.StatusNotFound, map[string]string{"error": "filename filter target not found"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not update filename filter"})
|
||||
return
|
||||
}
|
||||
operation := "created"
|
||||
if replayed {
|
||||
operation = "replayed"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": rule, "operation": operation})
|
||||
}
|
||||
@@ -83,6 +83,8 @@ type Store interface {
|
||||
ListScopeRules(ctx context.Context, connectionID string) ([]governance.ScopeRule, error)
|
||||
MutateScopeRule(ctx context.Context, request governance.ScopeRuleMutation) (governance.ScopeRule, error)
|
||||
MutateScopeRuleWithOperation(ctx context.Context, request governance.ScopeRuleMutation, operationID string) (governance.ScopeRule, bool, error)
|
||||
ListFilenameFilterRules(ctx context.Context, connectionID string) ([]governance.FilenameFilterRule, error)
|
||||
MutateFilenameFilterRuleWithOperation(ctx context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error)
|
||||
ListTestPublications(ctx context.Context, filter governance.TestPublicationFilter) (governance.TestPublicationPage, error)
|
||||
StartTestPublications(ctx context.Context, request governance.StartTestPublicationsRequest) (governance.StartTestPublicationsResult, error)
|
||||
StartTestPublicationsWithOperation(ctx context.Context, request governance.StartTestPublicationsRequest, operationID string) (governance.StartTestPublicationsResult, bool, error)
|
||||
@@ -252,6 +254,15 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("DELETE /api/v1/scope-rules", func(w http.ResponseWriter, r *http.Request) {
|
||||
mutateScopeRule(w, r, config, true)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
|
||||
listFilenameFilters(w, r, config.Store)
|
||||
})
|
||||
mux.HandleFunc("PUT /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
|
||||
mutateFilenameFilter(w, r, config, false)
|
||||
})
|
||||
mux.HandleFunc("DELETE /api/v1/filename-filters", func(w http.ResponseWriter, r *http.Request) {
|
||||
mutateFilenameFilter(w, r, config, true)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/source-connections", func(w http.ResponseWriter, r *http.Request) {
|
||||
listSourceConnections(w, r, config.Store)
|
||||
})
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -256,6 +257,26 @@ func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) {
|
||||
now := time.Date(2026, 9, 15, 2, 0, 0, 0, time.UTC)
|
||||
store := &fakeJobStore{filenameFilterResult: governance.FilenameFilterRule{
|
||||
ConnectionID: "conn-safe", DirectoryPath: "固件", ExcludedExtensions: []string{"mp4", "rar", "zip"}, UpdatedBy: "engineer-1", UpdatedAt: now,
|
||||
}}
|
||||
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"固件","excluded_extensions":[".ZIP","rar","Mp4","zip"]}`))
|
||||
request.Header.Set("Idempotency-Key", "op-filter-set")
|
||||
handler.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusOK || store.filenameFilterOperationID != "op-filter-set" ||
|
||||
!slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"mp4", "rar", "zip"}) ||
|
||||
!strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) {
|
||||
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) {
|
||||
now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC)
|
||||
hash := strings.Repeat("c", 64)
|
||||
@@ -1515,6 +1536,12 @@ type fakeJobStore struct {
|
||||
scopeRuleOperationID string
|
||||
scopeRuleReplayed bool
|
||||
scopeRuleErr error
|
||||
filenameFilters []governance.FilenameFilterRule
|
||||
filenameFilterResult governance.FilenameFilterRule
|
||||
filenameFilterRequest governance.FilenameFilterMutation
|
||||
filenameFilterOperationID string
|
||||
filenameFilterReplayed bool
|
||||
filenameFilterErr error
|
||||
catalogSources []governance.CatalogSource
|
||||
catalogPage governance.CatalogNodePage
|
||||
catalogFilter governance.CatalogNodeFilter
|
||||
@@ -1818,6 +1845,16 @@ func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request g
|
||||
return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr
|
||||
}
|
||||
|
||||
func (s *fakeJobStore) ListFilenameFilterRules(_ context.Context, _ string) ([]governance.FilenameFilterRule, error) {
|
||||
return s.filenameFilters, s.filenameFilterErr
|
||||
}
|
||||
|
||||
func (s *fakeJobStore) MutateFilenameFilterRuleWithOperation(_ context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
|
||||
s.filenameFilterRequest = request
|
||||
s.filenameFilterOperationID = operationID
|
||||
return s.filenameFilterResult, s.filenameFilterReplayed, s.filenameFilterErr
|
||||
}
|
||||
|
||||
func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) {
|
||||
return s.catalogSources, s.catalogErr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user