diff --git a/api/openapi.yaml b/api/openapi.yaml index 7207f50..ff53039 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1126,6 +1126,41 @@ } } }, + "/api/v1/source-catalog/available": { + "get": { + "tags": ["Source catalog"], + "operationId": "listAvailableSources", + "summary": "列出可加入的数据源(wiki 知识库 / drive 钉盘 / doc 在线文档表格 / pp 资料中心);四类同一形状,类型差异只体现在 content_type", + "parameters": [ + {"name": "type", "in": "query", "required": true, "schema": {"type": "string", "enum": ["wiki", "drive", "doc", "pp"]}}, + {"name": "q", "in": "query", "schema": {"type": "string", "maxLength": 120}}, + {"name": "limit", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": 200, "default": 50}} + ], + "responses": { + "200": {"$ref": "#/components/responses/Envelope"}, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"description": "需要登录"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"description": "dws 未登录或缺少读权限"} + } + } + }, + "/api/v1/source-catalog/validate": { + "post": { + "tags": ["Source catalog"], + "operationId": "validateSourceSelection", + "summary": "添加前真读一次:文档/表格校验可读、知识库与钉盘校验账号仍可见;读不到返回 503/409,不让进入下一步", + "requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object", "additionalProperties": false, "required": ["type", "refs"], "properties": {"type": {"type": "string", "enum": ["wiki", "drive", "doc", "pp"]}, "refs": {"type": "array", "minItems": 1, "maxItems": 50, "items": {"type": "string", "maxLength": 512}}}}}}}, + "responses": { + "200": {"$ref": "#/components/responses/Envelope"}, + "400": {"$ref": "#/components/responses/BadRequest"}, + "401": {"description": "需要登录"}, + "409": {"description": "账号已看不到所选来源"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"description": "dws 不可用或文档不可读"} + } + } + }, "/api/v1/source-catalog/sources": { "get": { "tags": [ diff --git a/cmd/api/main.go b/cmd/api/main.go index 10ce50c..6c05805 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -37,6 +37,15 @@ func (dingtalkSourceLister) ListAvailableDingtalkSources(ctx context.Context) (d return dingtalkdiscovery.Discover(ctx, dingtalkdiscovery.ExecRunner{}) } +// 同一份 dws 凭据也用来列在线文档/表格并校验可读。 +func (dingtalkSourceLister) ListDingtalkDocuments(ctx context.Context, query string, limit int) (dingtalkdiscovery.DocResult, error) { + return dingtalkdiscovery.DiscoverDocs(ctx, dingtalkdiscovery.ExecRunner{}, query, limit) +} + +func (dingtalkSourceLister) ValidateDingtalkDocument(ctx context.Context, node string) (dingtalkdiscovery.DocumentNode, error) { + return dingtalkdiscovery.ValidateDocument(ctx, dingtalkdiscovery.ExecRunner{}, node) +} + func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -184,6 +193,7 @@ func run(ctx context.Context, getenv func(string) string) error { ProductionReleaseAuthorizationToken: releaseAuthorizationToken, // 数据源发现(系统设置 → 数据源配置):用与 worker 同一份 dws 状态做只读列举。 DingtalkSources: dingtalkSourceLister{}, + DingtalkDocuments: dingtalkSourceLister{}, BackupSchedule: postgresstore.NewStore(pool), VersionHistory: versionHistoryStore, VersionHistoryRepo: versionHistoryRepo, diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index f6ed705..2cbf707 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -147,8 +147,10 @@ type Config struct { // DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。 // 未接线时对应路由返回 501,页面显示"列不出来"而不是空列表。 DingtalkSources dingtalkSourcesLister - NewID func() (string, error) - Auth *AuthConfig + // DingtalkDocuments 列在线文档/表格并校验可读(只读,dws)。 + DingtalkDocuments dingtalkDocLister + NewID func() (string, error) + Auth *AuthConfig } func NewHandler(config Config) (http.Handler, error) { @@ -426,6 +428,12 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("POST /api/v1/dingtalk-spaces", func(w http.ResponseWriter, r *http.Request) { registerDingtalkSpace(w, r, config) }) + mux.HandleFunc("GET /api/v1/source-catalog/available", func(w http.ResponseWriter, r *http.Request) { + listAvailableSources(w, r, config) + }) + mux.HandleFunc("POST /api/v1/source-catalog/validate", func(w http.ResponseWriter, r *http.Request) { + validateSourceSelection(w, r, config) + }) mux.HandleFunc("GET /api/v1/dingtalk-spaces/available", func(w http.ResponseWriter, r *http.Request) { listDingtalkSources(w, r, config) }) diff --git a/internal/httpapi/source_available.go b/internal/httpapi/source_available.go new file mode 100644 index 0000000..3660950 --- /dev/null +++ b/internal/httpapi/source_available.go @@ -0,0 +1,200 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "knowledge-governance/internal/dingtalkdiscovery" +) + +// dingtalkDocLister 是本路由需要的文档能力面:列在线文档/表格、并真实读一次确认可读。 +// 与知识库/钉盘共用同一份 dws 凭据,注入方在 cmd/api。 +type dingtalkDocLister interface { + ListDingtalkDocuments(ctx context.Context, query string, limit int) (dingtalkdiscovery.DocResult, error) + ValidateDingtalkDocument(ctx context.Context, node string) (dingtalkdiscovery.DocumentNode, error) +} + +// availableItem 是「添加数据源」向导第一步展示的候选条目,四类来源同一形状。 +type availableItem struct { + Ref string `json:"ref"` + Name string `json:"name"` + Kind string `json:"kind"` + ContentType string `json:"content_type,omitempty"` + URL string `json:"url,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +var availableSourceTypes = map[string]bool{"wiki": true, "drive": true, "doc": true, "pp": true} + +// listAvailableSources 处理 GET /api/v1/source-catalog/available?type=&q=&limit=: +// 向导用它列「可加入的数据源」。类型不同,形状相同——前端不需要为每类写一套。 +func listAvailableSources(w http.ResponseWriter, r *http.Request, config Config) { + if _, ok := requireSessionIdentity(w, r); !ok { + return + } + kind := strings.TrimSpace(r.URL.Query().Get("type")) + if !availableSourceTypes[kind] { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type 必须是 wiki、drive、doc 或 pp"}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + limit := 50 + if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 && parsed <= 200 { + limit = parsed + } + } + query := strings.TrimSpace(r.URL.Query().Get("q")) + items := make([]availableItem, 0, 16) + notice := "" + switch kind { + case "wiki", "drive": + lister := config.DingtalkSources + if lister == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk source discovery is not wired"}) + return + } + result, err := lister.ListAvailableDingtalkSources(ctx) + if err != nil { + writeDiscoveryFailure(w, err) + return + } + if kind == "wiki" { + for _, base := range result.KnowledgeBases { + items = append(items, availableItem{Ref: base.URL, Name: base.Name, Kind: "wiki", URL: base.URL}) + } + notice = result.KnowledgeNotice + } else { + for _, space := range result.DriveSpaces { + items = append(items, availableItem{Ref: dingtalkdiscovery.SpaceURL(space.SpaceID), Name: space.Name, Kind: "drive", URL: dingtalkdiscovery.SpaceURL(space.SpaceID)}) + } + notice = result.DriveNotice + } + case "doc": + lister := config.DingtalkDocuments + if lister == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk document discovery is not wired"}) + return + } + result, err := lister.ListDingtalkDocuments(ctx, query, limit) + if err != nil { + writeDiscoveryFailure(w, err) + return + } + for _, doc := range result.Documents { + items = append(items, availableItem{ + Ref: doc.NodeID, Name: doc.Name, Kind: "doc", + ContentType: doc.ContentType, URL: doc.URL, UpdatedAt: doc.UpdatedAt, + }) + } + notice = result.Notice + case "pp": + // PP 是公开只读目录:没有可枚举的账号级列表,向导要求直接给资料中心地址, + // 由第 4 步的校验去真打一次。这里返回空列表并说明原因,不伪装成"没权限"。 + notice = "PP 资料下载中心没有可列举的账号级列表,请在第 2 步填写资料中心地址。" + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{ + "type": kind, "items": items, "notice": notice, "fetched_at": time.Now().UTC(), + }}) +} + +// validateSourceSelection 处理 POST /api/v1/source-catalog/validate: +// 添加前真读一次,读不到就不让进下一步——避免加入一堆永远同步不了的来源。 +func validateSourceSelection(w http.ResponseWriter, r *http.Request, config Config) { + if _, ok := requireSessionIdentity(w, r); !ok { + return + } + var body struct { + Type string `json:"type"` + Refs []string `json:"refs"` + } + if decodeJSONStrict(r, &body) != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid_request_body"}) + return + } + kind := strings.TrimSpace(body.Type) + if !availableSourceTypes[kind] || len(body.Refs) == 0 || len(body.Refs) > 50 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type 无效,或一次校验的条目数不在 1–50 之间"}) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + switch kind { + case "doc": + lister := config.DingtalkDocuments + if lister == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk document discovery is not wired"}) + return + } + verified := make([]map[string]string, 0, len(body.Refs)) + for _, ref := range body.Refs { + doc, err := lister.ValidateDingtalkDocument(ctx, ref) + if err != nil { + writeDiscoveryFailure(w, err) + return + } + verified = append(verified, map[string]string{"ref": doc.NodeID, "name": doc.Name, "content_type": doc.ContentType}) + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{ + "ok": true, "type": kind, "items": verified, "discovered": len(verified), + }}) + case "wiki", "drive": + lister := config.DingtalkSources + if lister == nil { + writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk source discovery is not wired"}) + return + } + result, err := lister.ListAvailableDingtalkSources(ctx) + if err != nil { + writeDiscoveryFailure(w, err) + return + } + known := map[string]string{} + if kind == "wiki" { + for _, base := range result.KnowledgeBases { + known[base.URL] = base.Name + } + } else { + for _, space := range result.DriveSpaces { + known[dingtalkdiscovery.SpaceURL(space.SpaceID)] = space.Name + } + } + verified := make([]map[string]string, 0, len(body.Refs)) + for _, ref := range body.Refs { + name, ok := known[strings.TrimSpace(ref)] + if !ok { + writeJSON(w, http.StatusConflict, map[string]string{"error": "当前钉钉账号已看不到该来源,请刷新后再选"}) + return + } + verified = append(verified, map[string]string{"ref": ref, "name": name}) + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{ + "ok": true, "type": kind, "items": verified, "discovered": len(verified), + }}) + case "pp": + // PP 的可用性由真正的下载探测在登记流程里判定,这里只做形式校验。 + for _, ref := range body.Refs { + if !strings.HasPrefix(strings.TrimSpace(ref), "http") { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "PP 需要 http(s) 资料中心地址"}) + return + } + } + writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{"ok": true, "type": kind, "discovered": len(body.Refs)}}) + } +} + +func writeDiscoveryFailure(w http.ResponseWriter, err error) { + if errors.Is(err, dingtalkdiscovery.ErrUnavailable) { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{ + "error": "dingtalk_discovery_unavailable", + "hint": "dws 未登录或缺少读权限;请刷新钉钉会话,或让管理员检查 dws 凭据。", + }) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list sources"}) +} diff --git a/internal/httpapi/source_available_test.go b/internal/httpapi/source_available_test.go new file mode 100644 index 0000000..50f0fbb --- /dev/null +++ b/internal/httpapi/source_available_test.go @@ -0,0 +1,168 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "knowledge-governance/internal/dingtalkdiscovery" +) + +type fakeDingtalkDocuments struct { + docs dingtalkdiscovery.DocResult + listErr error + validated []string + validateAs dingtalkdiscovery.DocumentNode + validateEr error +} + +func (f *fakeDingtalkDocuments) ListDingtalkDocuments(_ context.Context, _ string, _ int) (dingtalkdiscovery.DocResult, error) { + return f.docs, f.listErr +} + +func (f *fakeDingtalkDocuments) ValidateDingtalkDocument(_ context.Context, node string) (dingtalkdiscovery.DocumentNode, error) { + f.validated = append(f.validated, node) + return f.validateAs, f.validateEr +} + +func availableRequest(role, url string) *http.Request { + request := httptest.NewRequest(http.MethodGet, url, nil) + if role == "" { + return request + } + return request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: role})) +} + +func decodeEnvelope(t *testing.T, rec *httptest.ResponseRecorder) map[string]any { + t.Helper() + var payload map[string]any + if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode body %s: %v", rec.Body.String(), err) + } + data, _ := payload["data"].(map[string]any) + return data +} + +func TestListAvailableSourcesKeepsOneShapeForEveryType(t *testing.T) { + docs := &fakeDingtalkDocuments{docs: dingtalkdiscovery.DocResult{Documents: []dingtalkdiscovery.DocumentNode{ + {NodeID: "node-a", Name: "售后周会纪要", ContentType: "ALIDOC", URL: "https://alidocs.dingtalk.com/i/nodes/node-a"}, + {NodeID: "node-b", Name: "服务工单统计", ContentType: "ALISHEET"}, + }}} + config := Config{DingtalkDocuments: docs} + rec := httptest.NewRecorder() + listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=doc"), config) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + data := decodeEnvelope(t, rec) + items, _ := data["items"].([]any) + if len(items) != 2 { + t.Fatalf("items = %d, want 2", len(items)) + } + first, _ := items[0].(map[string]any) + // 四类来源同一形状:ref/name/kind 一定在,类型差异只放在 content_type。 + for _, key := range []string{"ref", "name", "kind"} { + if _, ok := first[key]; !ok { + t.Fatalf("item missing %q: %v", key, first) + } + } + if first["kind"] != "doc" || first["content_type"] != "ALIDOC" { + t.Fatalf("doc item = %v", first) + } +} + +func TestListAvailableSourcesRejectsUnknownTypesAndReportsMissingWiring(t *testing.T) { + rec := httptest.NewRecorder() + listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=__nope__"), Config{}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("unknown type = %d, want 400", rec.Code) + } + + rec = httptest.NewRecorder() + listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=doc"), Config{}) + if rec.Code != http.StatusNotImplemented { + t.Fatalf("unwired doc = %d, want 501", rec.Code) + } + + rec = httptest.NewRecorder() + listAvailableSources(rec, availableRequest("", "/api/v1/source-catalog/available?type=doc"), Config{}) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("anonymous = %d, want 401", rec.Code) + } +} + +func TestListAvailableSourcesExplainsWhyPpHasNoAccountLevelList(t *testing.T) { + rec := httptest.NewRecorder() + listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=pp"), Config{}) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + data := decodeEnvelope(t, rec) + if items, _ := data["items"].([]any); len(items) != 0 { + t.Fatalf("pp items = %d, want 0", len(items)) + } + if notice, _ := data["notice"].(string); !strings.Contains(notice, "资料中心地址") { + t.Fatalf("notice = %q", notice) + } +} + +func TestValidateSourceSelectionProvesDocumentsAreReadable(t *testing.T) { + docs := &fakeDingtalkDocuments{validateAs: dingtalkdiscovery.DocumentNode{NodeID: "node-a", Name: "售后周会纪要", ContentType: "ALIDOC"}} + body, _ := json.Marshal(map[string]any{"type": "doc", "refs": []string{"node-a"}}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body)) + request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin})) + rec := httptest.NewRecorder() + validateSourceSelection(rec, request, Config{DingtalkDocuments: docs}) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + if len(docs.validated) != 1 || docs.validated[0] != "node-a" { + t.Fatalf("validated = %v", docs.validated) + } + if data := decodeEnvelope(t, rec); data["discovered"] != float64(1) { + t.Fatalf("discovered = %v", data["discovered"]) + } +} + +func TestValidateSourceSelectionFailsWhenTheDocumentIsGone(t *testing.T) { + docs := &fakeDingtalkDocuments{validateEr: dingtalkdiscovery.ErrUnavailable} + body, _ := json.Marshal(map[string]any{"type": "doc", "refs": []string{"node-gone"}}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body)) + request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin})) + rec := httptest.NewRecorder() + validateSourceSelection(rec, request, Config{DingtalkDocuments: docs}) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503", rec.Code) + } +} + +func TestValidateSourceSelectionRejectsSpacesTheAccountCanNoLongerSee(t *testing.T) { + config := Config{DingtalkSources: fakeDingtalkSources{result: dingtalkdiscovery.Result{ + KnowledgeBases: []dingtalkdiscovery.KnowledgeBase{{Name: "产品知识库", URL: "https://alidocs.dingtalk.com/i/spaces/keep/overview"}}, + }}} + body, _ := json.Marshal(map[string]any{"type": "wiki", "refs": []string{"https://alidocs.dingtalk.com/i/spaces/gone/overview"}}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body)) + request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin})) + rec := httptest.NewRecorder() + validateSourceSelection(rec, request, config) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d, want 409 body=%s", rec.Code, rec.Body.String()) + } +} + +func TestValidateSourceSelectionRejectsEmptyAndOversizedSelections(t *testing.T) { + for _, refs := range [][]string{{}, make([]string, 51)} { + body, _ := json.Marshal(map[string]any{"type": "doc", "refs": refs}) + request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body)) + request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin})) + rec := httptest.NewRecorder() + validateSourceSelection(rec, request, Config{}) + if rec.Code != http.StatusBadRequest { + t.Fatalf("refs=%d status = %d, want 400", len(refs), rec.Code) + } + } +} \ No newline at end of file