feat(ingest): 源与目录文件名过滤及钉钉文档导出

- filename_filter_rules table (000035) with source-root and per-directory rows
- additive inheritance: source default + every ancestor directory contributes
- filename_is_filtered() SQL predicate applied to catalog browsing, subtree
  counts, change-set diffing, routing fan-out and artifact fetch manifests
- fetch_state 'filtered' (CHECK extended); reconcile cancels/restores fetch
  jobs, retires planned publications and blocks items when rules change
- GET/PUT/DELETE /api/v1/filename-filters (Idempotency-Key, normalized
  extensions) + Settings and Catalog filter editors
- failJobSQL no longer swallows the claimed row when the failure projection
  upserts (fixes spurious 'lease lost' retries)
- DingTalk live fetch: adoc/axls now use the export-job pipeline (HTTP 403
  from file/download fixed), header-signature downloads supported
This commit is contained in:
姚章浩
2026-09-15 09:46:31 +08:00
parent 2005e4e7da
commit b3c9331966
25 changed files with 1080 additions and 30 deletions
+37
View File
@@ -7,6 +7,7 @@ import (
"errors"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"time"
@@ -256,6 +257,26 @@ func TestHandlerRejectsUnsafeScopeRuleBeforeStore(t *testing.T) {
}
}
func TestHandlerNormalizesAndPersistsFilenameFilters(t *testing.T) {
now := time.Date(2026, 9, 15, 2, 0, 0, 0, time.UTC)
store := &fakeJobStore{filenameFilterResult: governance.FilenameFilterRule{
ConnectionID: "conn-safe", DirectoryPath: "固件", ExcludedExtensions: []string{"mp4", "rar", "zip"}, UpdatedBy: "engineer-1", UpdatedAt: now,
}}
handler, err := NewHandler(Config{Store: store, Now: func() time.Time { return now }})
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPut, "/api/v1/filename-filters", bytes.NewBufferString(`{"connection_id":"conn-safe","directory_path":"固件","excluded_extensions":[".ZIP","rar","Mp4","zip"]}`))
request.Header.Set("Idempotency-Key", "op-filter-set")
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK || store.filenameFilterOperationID != "op-filter-set" ||
!slices.Equal(store.filenameFilterRequest.ExcludedExtensions, []string{"mp4", "rar", "zip"}) ||
!strings.Contains(response.Body.String(), `"excluded_extensions":["mp4","rar","zip"]`) {
t.Fatalf("status=%d request=%#v body=%s", response.Code, store.filenameFilterRequest, response.Body.String())
}
}
func TestHandlerListsSnapshotHistoryAndReadsHistoricalNodes(t *testing.T) {
now := time.Date(2026, 9, 11, 14, 0, 0, 0, time.UTC)
hash := strings.Repeat("c", 64)
@@ -1515,6 +1536,12 @@ type fakeJobStore struct {
scopeRuleOperationID string
scopeRuleReplayed bool
scopeRuleErr error
filenameFilters []governance.FilenameFilterRule
filenameFilterResult governance.FilenameFilterRule
filenameFilterRequest governance.FilenameFilterMutation
filenameFilterOperationID string
filenameFilterReplayed bool
filenameFilterErr error
catalogSources []governance.CatalogSource
catalogPage governance.CatalogNodePage
catalogFilter governance.CatalogNodeFilter
@@ -1818,6 +1845,16 @@ func (s *fakeJobStore) MutateScopeRuleWithOperation(_ context.Context, request g
return s.scopeRuleResult, s.scopeRuleReplayed, s.scopeRuleErr
}
func (s *fakeJobStore) ListFilenameFilterRules(_ context.Context, _ string) ([]governance.FilenameFilterRule, error) {
return s.filenameFilters, s.filenameFilterErr
}
func (s *fakeJobStore) MutateFilenameFilterRuleWithOperation(_ context.Context, request governance.FilenameFilterMutation, operationID string) (governance.FilenameFilterRule, bool, error) {
s.filenameFilterRequest = request
s.filenameFilterOperationID = operationID
return s.filenameFilterResult, s.filenameFilterReplayed, s.filenameFilterErr
}
func (s *fakeJobStore) ListCatalogSources(context.Context) ([]governance.CatalogSource, error) {
return s.catalogSources, s.catalogErr
}