feat(api): 数据源统一列举与添加前校验
build / build (push) Failing after 16s

GET /source-catalog/available 四类同一形状(wiki/drive/doc/pp),类型差异只放在 content_type;
POST /source-catalog/validate 真读一次:文档表格校验可读、知识库钉盘校验账号仍可见,失败不让添加。
This commit is contained in:
hermes
2026-10-26 17:22:09 +08:00
parent b3992bfa0f
commit 017cbcb851
5 changed files with 423 additions and 2 deletions
+35
View File
@@ -1126,6 +1126,41 @@
}
}
},
"/api/v1/source-catalog/available": {
"get": {
"tags": ["Source catalog"],
"operationId": "listAvailableSources",
"summary": "列出可加入的数据源(wiki 知识库 / drive 钉盘 / doc 在线文档表格 / pp 资料中心);四类同一形状,类型差异只体现在 content_type",
"parameters": [
{"name": "type", "in": "query", "required": true, "schema": {"type": "string", "enum": ["wiki", "drive", "doc", "pp"]}},
{"name": "q", "in": "query", "schema": {"type": "string", "maxLength": 120}},
{"name": "limit", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": 200, "default": 50}}
],
"responses": {
"200": {"$ref": "#/components/responses/Envelope"},
"400": {"$ref": "#/components/responses/BadRequest"},
"401": {"description": "需要登录"},
"500": {"$ref": "#/components/responses/InternalError"},
"503": {"description": "dws 未登录或缺少读权限"}
}
}
},
"/api/v1/source-catalog/validate": {
"post": {
"tags": ["Source catalog"],
"operationId": "validateSourceSelection",
"summary": "添加前真读一次:文档/表格校验可读、知识库与钉盘校验账号仍可见;读不到返回 503/409,不让进入下一步",
"requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object", "additionalProperties": false, "required": ["type", "refs"], "properties": {"type": {"type": "string", "enum": ["wiki", "drive", "doc", "pp"]}, "refs": {"type": "array", "minItems": 1, "maxItems": 50, "items": {"type": "string", "maxLength": 512}}}}}}},
"responses": {
"200": {"$ref": "#/components/responses/Envelope"},
"400": {"$ref": "#/components/responses/BadRequest"},
"401": {"description": "需要登录"},
"409": {"description": "账号已看不到所选来源"},
"500": {"$ref": "#/components/responses/InternalError"},
"503": {"description": "dws 不可用或文档不可读"}
}
}
},
"/api/v1/source-catalog/sources": {
"get": {
"tags": [
+10
View File
@@ -37,6 +37,15 @@ func (dingtalkSourceLister) ListAvailableDingtalkSources(ctx context.Context) (d
return dingtalkdiscovery.Discover(ctx, dingtalkdiscovery.ExecRunner{})
}
// 同一份 dws 凭据也用来列在线文档/表格并校验可读。
func (dingtalkSourceLister) ListDingtalkDocuments(ctx context.Context, query string, limit int) (dingtalkdiscovery.DocResult, error) {
return dingtalkdiscovery.DiscoverDocs(ctx, dingtalkdiscovery.ExecRunner{}, query, limit)
}
func (dingtalkSourceLister) ValidateDingtalkDocument(ctx context.Context, node string) (dingtalkdiscovery.DocumentNode, error) {
return dingtalkdiscovery.ValidateDocument(ctx, dingtalkdiscovery.ExecRunner{}, node)
}
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
@@ -184,6 +193,7 @@ func run(ctx context.Context, getenv func(string) string) error {
ProductionReleaseAuthorizationToken: releaseAuthorizationToken,
// 数据源发现(系统设置 → 数据源配置):用与 worker 同一份 dws 状态做只读列举。
DingtalkSources: dingtalkSourceLister{},
DingtalkDocuments: dingtalkSourceLister{},
BackupSchedule: postgresstore.NewStore(pool),
VersionHistory: versionHistoryStore,
VersionHistoryRepo: versionHistoryRepo,
+10 -2
View File
@@ -147,8 +147,10 @@ type Config struct {
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
// 未接线时对应路由返回 501,页面显示"列不出来"而不是空列表。
DingtalkSources dingtalkSourcesLister
NewID func() (string, error)
Auth *AuthConfig
// DingtalkDocuments 列在线文档/表格并校验可读(只读,dws)。
DingtalkDocuments dingtalkDocLister
NewID func() (string, error)
Auth *AuthConfig
}
func NewHandler(config Config) (http.Handler, error) {
@@ -426,6 +428,12 @@ func NewHandler(config Config) (http.Handler, error) {
mux.HandleFunc("POST /api/v1/dingtalk-spaces", func(w http.ResponseWriter, r *http.Request) {
registerDingtalkSpace(w, r, config)
})
mux.HandleFunc("GET /api/v1/source-catalog/available", func(w http.ResponseWriter, r *http.Request) {
listAvailableSources(w, r, config)
})
mux.HandleFunc("POST /api/v1/source-catalog/validate", func(w http.ResponseWriter, r *http.Request) {
validateSourceSelection(w, r, config)
})
mux.HandleFunc("GET /api/v1/dingtalk-spaces/available", func(w http.ResponseWriter, r *http.Request) {
listDingtalkSources(w, r, config)
})
+200
View File
@@ -0,0 +1,200 @@
package httpapi
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"time"
"knowledge-governance/internal/dingtalkdiscovery"
)
// dingtalkDocLister 是本路由需要的文档能力面:列在线文档/表格、并真实读一次确认可读。
// 与知识库/钉盘共用同一份 dws 凭据,注入方在 cmd/api。
type dingtalkDocLister interface {
ListDingtalkDocuments(ctx context.Context, query string, limit int) (dingtalkdiscovery.DocResult, error)
ValidateDingtalkDocument(ctx context.Context, node string) (dingtalkdiscovery.DocumentNode, error)
}
// availableItem 是「添加数据源」向导第一步展示的候选条目,四类来源同一形状。
type availableItem struct {
Ref string `json:"ref"`
Name string `json:"name"`
Kind string `json:"kind"`
ContentType string `json:"content_type,omitempty"`
URL string `json:"url,omitempty"`
UpdatedAt string `json:"updated_at,omitempty"`
}
var availableSourceTypes = map[string]bool{"wiki": true, "drive": true, "doc": true, "pp": true}
// listAvailableSources 处理 GET /api/v1/source-catalog/available?type=&q=&limit=:
// 向导用它列「可加入的数据源」。类型不同,形状相同——前端不需要为每类写一套。
func listAvailableSources(w http.ResponseWriter, r *http.Request, config Config) {
if _, ok := requireSessionIdentity(w, r); !ok {
return
}
kind := strings.TrimSpace(r.URL.Query().Get("type"))
if !availableSourceTypes[kind] {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type 必须是 wiki、drive、doc 或 pp"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
limit := 50
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 && parsed <= 200 {
limit = parsed
}
}
query := strings.TrimSpace(r.URL.Query().Get("q"))
items := make([]availableItem, 0, 16)
notice := ""
switch kind {
case "wiki", "drive":
lister := config.DingtalkSources
if lister == nil {
writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk source discovery is not wired"})
return
}
result, err := lister.ListAvailableDingtalkSources(ctx)
if err != nil {
writeDiscoveryFailure(w, err)
return
}
if kind == "wiki" {
for _, base := range result.KnowledgeBases {
items = append(items, availableItem{Ref: base.URL, Name: base.Name, Kind: "wiki", URL: base.URL})
}
notice = result.KnowledgeNotice
} else {
for _, space := range result.DriveSpaces {
items = append(items, availableItem{Ref: dingtalkdiscovery.SpaceURL(space.SpaceID), Name: space.Name, Kind: "drive", URL: dingtalkdiscovery.SpaceURL(space.SpaceID)})
}
notice = result.DriveNotice
}
case "doc":
lister := config.DingtalkDocuments
if lister == nil {
writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk document discovery is not wired"})
return
}
result, err := lister.ListDingtalkDocuments(ctx, query, limit)
if err != nil {
writeDiscoveryFailure(w, err)
return
}
for _, doc := range result.Documents {
items = append(items, availableItem{
Ref: doc.NodeID, Name: doc.Name, Kind: "doc",
ContentType: doc.ContentType, URL: doc.URL, UpdatedAt: doc.UpdatedAt,
})
}
notice = result.Notice
case "pp":
// PP 是公开只读目录:没有可枚举的账号级列表,向导要求直接给资料中心地址,
// 由第 4 步的校验去真打一次。这里返回空列表并说明原因,不伪装成"没权限"。
notice = "PP 资料下载中心没有可列举的账号级列表,请在第 2 步填写资料中心地址。"
}
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{
"type": kind, "items": items, "notice": notice, "fetched_at": time.Now().UTC(),
}})
}
// validateSourceSelection 处理 POST /api/v1/source-catalog/validate:
// 添加前真读一次,读不到就不让进下一步——避免加入一堆永远同步不了的来源。
func validateSourceSelection(w http.ResponseWriter, r *http.Request, config Config) {
if _, ok := requireSessionIdentity(w, r); !ok {
return
}
var body struct {
Type string `json:"type"`
Refs []string `json:"refs"`
}
if decodeJSONStrict(r, &body) != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid_request_body"})
return
}
kind := strings.TrimSpace(body.Type)
if !availableSourceTypes[kind] || len(body.Refs) == 0 || len(body.Refs) > 50 {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type 无效,或一次校验的条目数不在 1–50 之间"})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
switch kind {
case "doc":
lister := config.DingtalkDocuments
if lister == nil {
writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk document discovery is not wired"})
return
}
verified := make([]map[string]string, 0, len(body.Refs))
for _, ref := range body.Refs {
doc, err := lister.ValidateDingtalkDocument(ctx, ref)
if err != nil {
writeDiscoveryFailure(w, err)
return
}
verified = append(verified, map[string]string{"ref": doc.NodeID, "name": doc.Name, "content_type": doc.ContentType})
}
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{
"ok": true, "type": kind, "items": verified, "discovered": len(verified),
}})
case "wiki", "drive":
lister := config.DingtalkSources
if lister == nil {
writeJSON(w, http.StatusNotImplemented, map[string]string{"error": "dingtalk source discovery is not wired"})
return
}
result, err := lister.ListAvailableDingtalkSources(ctx)
if err != nil {
writeDiscoveryFailure(w, err)
return
}
known := map[string]string{}
if kind == "wiki" {
for _, base := range result.KnowledgeBases {
known[base.URL] = base.Name
}
} else {
for _, space := range result.DriveSpaces {
known[dingtalkdiscovery.SpaceURL(space.SpaceID)] = space.Name
}
}
verified := make([]map[string]string, 0, len(body.Refs))
for _, ref := range body.Refs {
name, ok := known[strings.TrimSpace(ref)]
if !ok {
writeJSON(w, http.StatusConflict, map[string]string{"error": "当前钉钉账号已看不到该来源,请刷新后再选"})
return
}
verified = append(verified, map[string]string{"ref": ref, "name": name})
}
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{
"ok": true, "type": kind, "items": verified, "discovered": len(verified),
}})
case "pp":
// PP 的可用性由真正的下载探测在登记流程里判定,这里只做形式校验。
for _, ref := range body.Refs {
if !strings.HasPrefix(strings.TrimSpace(ref), "http") {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "PP 需要 http(s) 资料中心地址"})
return
}
}
writeJSON(w, http.StatusOK, map[string]any{"data": map[string]any{"ok": true, "type": kind, "discovered": len(body.Refs)}})
}
}
func writeDiscoveryFailure(w http.ResponseWriter, err error) {
if errors.Is(err, dingtalkdiscovery.ErrUnavailable) {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{
"error": "dingtalk_discovery_unavailable",
"hint": "dws 未登录或缺少读权限;请刷新钉钉会话,或让管理员检查 dws 凭据。",
})
return
}
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "could not list sources"})
}
+168
View File
@@ -0,0 +1,168 @@
package httpapi
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"knowledge-governance/internal/dingtalkdiscovery"
)
type fakeDingtalkDocuments struct {
docs dingtalkdiscovery.DocResult
listErr error
validated []string
validateAs dingtalkdiscovery.DocumentNode
validateEr error
}
func (f *fakeDingtalkDocuments) ListDingtalkDocuments(_ context.Context, _ string, _ int) (dingtalkdiscovery.DocResult, error) {
return f.docs, f.listErr
}
func (f *fakeDingtalkDocuments) ValidateDingtalkDocument(_ context.Context, node string) (dingtalkdiscovery.DocumentNode, error) {
f.validated = append(f.validated, node)
return f.validateAs, f.validateEr
}
func availableRequest(role, url string) *http.Request {
request := httptest.NewRequest(http.MethodGet, url, nil)
if role == "" {
return request
}
return request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: role}))
}
func decodeEnvelope(t *testing.T, rec *httptest.ResponseRecorder) map[string]any {
t.Helper()
var payload map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode body %s: %v", rec.Body.String(), err)
}
data, _ := payload["data"].(map[string]any)
return data
}
func TestListAvailableSourcesKeepsOneShapeForEveryType(t *testing.T) {
docs := &fakeDingtalkDocuments{docs: dingtalkdiscovery.DocResult{Documents: []dingtalkdiscovery.DocumentNode{
{NodeID: "node-a", Name: "售后周会纪要", ContentType: "ALIDOC", URL: "https://alidocs.dingtalk.com/i/nodes/node-a"},
{NodeID: "node-b", Name: "服务工单统计", ContentType: "ALISHEET"},
}}}
config := Config{DingtalkDocuments: docs}
rec := httptest.NewRecorder()
listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=doc"), config)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
data := decodeEnvelope(t, rec)
items, _ := data["items"].([]any)
if len(items) != 2 {
t.Fatalf("items = %d, want 2", len(items))
}
first, _ := items[0].(map[string]any)
// 四类来源同一形状:ref/name/kind 一定在,类型差异只放在 content_type。
for _, key := range []string{"ref", "name", "kind"} {
if _, ok := first[key]; !ok {
t.Fatalf("item missing %q: %v", key, first)
}
}
if first["kind"] != "doc" || first["content_type"] != "ALIDOC" {
t.Fatalf("doc item = %v", first)
}
}
func TestListAvailableSourcesRejectsUnknownTypesAndReportsMissingWiring(t *testing.T) {
rec := httptest.NewRecorder()
listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=__nope__"), Config{})
if rec.Code != http.StatusBadRequest {
t.Fatalf("unknown type = %d, want 400", rec.Code)
}
rec = httptest.NewRecorder()
listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=doc"), Config{})
if rec.Code != http.StatusNotImplemented {
t.Fatalf("unwired doc = %d, want 501", rec.Code)
}
rec = httptest.NewRecorder()
listAvailableSources(rec, availableRequest("", "/api/v1/source-catalog/available?type=doc"), Config{})
if rec.Code != http.StatusUnauthorized {
t.Fatalf("anonymous = %d, want 401", rec.Code)
}
}
func TestListAvailableSourcesExplainsWhyPpHasNoAccountLevelList(t *testing.T) {
rec := httptest.NewRecorder()
listAvailableSources(rec, availableRequest(roleAdmin, "/api/v1/source-catalog/available?type=pp"), Config{})
if rec.Code != http.StatusOK {
t.Fatalf("status = %d", rec.Code)
}
data := decodeEnvelope(t, rec)
if items, _ := data["items"].([]any); len(items) != 0 {
t.Fatalf("pp items = %d, want 0", len(items))
}
if notice, _ := data["notice"].(string); !strings.Contains(notice, "资料中心地址") {
t.Fatalf("notice = %q", notice)
}
}
func TestValidateSourceSelectionProvesDocumentsAreReadable(t *testing.T) {
docs := &fakeDingtalkDocuments{validateAs: dingtalkdiscovery.DocumentNode{NodeID: "node-a", Name: "售后周会纪要", ContentType: "ALIDOC"}}
body, _ := json.Marshal(map[string]any{"type": "doc", "refs": []string{"node-a"}})
request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body))
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
rec := httptest.NewRecorder()
validateSourceSelection(rec, request, Config{DingtalkDocuments: docs})
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if len(docs.validated) != 1 || docs.validated[0] != "node-a" {
t.Fatalf("validated = %v", docs.validated)
}
if data := decodeEnvelope(t, rec); data["discovered"] != float64(1) {
t.Fatalf("discovered = %v", data["discovered"])
}
}
func TestValidateSourceSelectionFailsWhenTheDocumentIsGone(t *testing.T) {
docs := &fakeDingtalkDocuments{validateEr: dingtalkdiscovery.ErrUnavailable}
body, _ := json.Marshal(map[string]any{"type": "doc", "refs": []string{"node-gone"}})
request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body))
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
rec := httptest.NewRecorder()
validateSourceSelection(rec, request, Config{DingtalkDocuments: docs})
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status = %d, want 503", rec.Code)
}
}
func TestValidateSourceSelectionRejectsSpacesTheAccountCanNoLongerSee(t *testing.T) {
config := Config{DingtalkSources: fakeDingtalkSources{result: dingtalkdiscovery.Result{
KnowledgeBases: []dingtalkdiscovery.KnowledgeBase{{Name: "产品知识库", URL: "https://alidocs.dingtalk.com/i/spaces/keep/overview"}},
}}}
body, _ := json.Marshal(map[string]any{"type": "wiki", "refs": []string{"https://alidocs.dingtalk.com/i/spaces/gone/overview"}})
request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body))
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
rec := httptest.NewRecorder()
validateSourceSelection(rec, request, config)
if rec.Code != http.StatusConflict {
t.Fatalf("status = %d, want 409 body=%s", rec.Code, rec.Body.String())
}
}
func TestValidateSourceSelectionRejectsEmptyAndOversizedSelections(t *testing.T) {
for _, refs := range [][]string{{}, make([]string, 51)} {
body, _ := json.Marshal(map[string]any{"type": "doc", "refs": refs})
request := httptest.NewRequest(http.MethodPost, "/api/v1/source-catalog/validate", bytes.NewReader(body))
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
rec := httptest.NewRecorder()
validateSourceSelection(rec, request, Config{})
if rec.Code != http.StatusBadRequest {
t.Fatalf("refs=%d status = %d, want 400", len(refs), rec.Code)
}
}
}