diff --git a/api/openapi.yaml b/api/openapi.yaml index 5e95bc8..b508a65 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -585,6 +585,34 @@ } } }, + "/api/v1/dingtalk-spaces": { + "post": { + "tags": [ + "Source connections" + ], + "operationId": "registerDingtalkSpace", + "summary": "解析并登记钉钉知识库空间(管理员)", + "parameters": [ + { + "$ref": "#/components/parameters/IdempotencyKey" + } + ], + "responses": { + "201": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "500": { + "$ref": "#/components/responses/InternalError" + } + } + } + }, "/api/v1/source-connections": { "get": { "tags": [ diff --git a/compose.yaml b/compose.yaml index 7d1c6f9..2d8c034 100644 --- a/compose.yaml +++ b/compose.yaml @@ -143,10 +143,12 @@ services: user: "${APP_UID:-1000}:${APP_GID:-1000}" environment: CAPTURE_FILE: /run/source-secrets/dingtalk_capture.json + DINGTALK_STATE_FILE: /sources/dingtalk/data/dingtalk-internal/inventory-state.json VERIFY_SPACE: "${DINGTALK_VERIFY_SPACE:-9JOGO6332vlBnm4Q}" QR_TTL_MS: "300000" volumes: - ./secrets:/run/source-secrets + - /home/zheromiku/dingtalk_kb_download:/sources/dingtalk - dingtalk-chromium-profile:/profile security_opt: - no-new-privileges:true diff --git a/internal/httpapi/auth.go b/internal/httpapi/auth.go index 6d01900..257e631 100644 --- a/internal/httpapi/auth.go +++ b/internal/httpapi/auth.go @@ -119,8 +119,9 @@ func classifyRole(method, path string) string { } // The DingTalk scan session replaces the shared browser credentials: // admin-only, like every other credential surface. The QR image is a - // login artifact, so even polling it stays admin-only. - if strings.HasPrefix(path, "/api/v1/dingtalk-session") { + // login artifact, so even polling it stays admin-only. Space + // registration writes new source connections: same boundary. + if strings.HasPrefix(path, "/api/v1/dingtalk-session") || strings.HasPrefix(path, "/api/v1/dingtalk-spaces") { return roleAdmin } // Binding the internal/external knowledge bases re-routes every future diff --git a/internal/httpapi/dingtalk_session.go b/internal/httpapi/dingtalk_session.go index 0607932..b0986c5 100644 --- a/internal/httpapi/dingtalk_session.go +++ b/internal/httpapi/dingtalk_session.go @@ -2,13 +2,19 @@ package httpapi import ( "bytes" + "context" "encoding/json" "io" "log" "net/http" "os" + "regexp" "strings" "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" ) // DingTalk login proxy. The scan-code login itself lives in the @@ -76,6 +82,97 @@ func deleteDingtalkSession(w http.ResponseWriter, r *http.Request) { writeProxyResult(w, status, data) } +// registerDingtalkSpace resolves a DingTalk knowledge-base space URL through +// the login sidecar and, on success, registers the returned space as a +// source connection (deterministic id, enabled, manual-only schedule until an +// admin sets an interval). Re-registering the same space re-syncs name/root. +func registerDingtalkSpace(w http.ResponseWriter, r *http.Request, config Config) { + if !requireDingtalkIdentity(w, r) { + return + } + operationID, ok := requireOperationID(w, r) + if !ok { + return + } + var input struct { + URL string `json:"url"` + } + if err := decodeJSONStrict(r, &input); err != nil || strings.TrimSpace(input.URL) == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "url is required"}) + return + } + payload, _ := json.Marshal(map[string]string{"url": strings.TrimSpace(input.URL)}) + status, data := proxyDingtalk(http.MethodPost, "/spaces/add", payload) + if status != http.StatusOK { + if status == 0 { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "dingtalk_login_unavailable"}) + return + } + // Relay the sidecar's error (login_required / space_not_found / + // name_conflict) verbatim; bodies are machine codes, no secrets. + writeProxyResult(w, status, data) + return + } + var resolved struct { + Key string `json:"key"` + Name string `json:"name"` + Root string `json:"root"` + } + if err := json.Unmarshal(data, &resolved); err != nil || resolved.Key == "" || resolved.Name == "" || resolved.Root == "" { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": "invalid sidecar registration response"}) + return + } + store, ok := config.Store.(interface { + RegisterDingtalkSpace(context.Context, postgresstore.RegisterDingtalkSpaceInput) (string, error) + }) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "registration_unavailable"}) + return + } + spaceID := extractDingtalkSpaceID(input.URL) + if spaceID == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid space url"}) + return + } + connectionID, err := store.RegisterDingtalkSpace(r.Context(), postgresstore.RegisterDingtalkSpaceInput{ + RootKey: resolved.Key, Name: resolved.Name, SpaceID: spaceID, RootID: resolved.Root, Now: config.Now(), + }) + if err != nil { + writeJSON(w, http.StatusConflict, map[string]string{"error": "space_registration_conflict"}) + return + } + // First scan is enqueued right away: the new connection has no catalog + // head yet, so the generic sources/{id}/refresh endpoint would 404 until + // the first snapshot exists. + if enqueuer, ok := config.Store.(interface { + EnqueueCatalogRefreshWithOperation(context.Context, governance.CatalogSource, string, time.Time) (jobs.Job, bool, error) + }); ok { + if _, _, enqueueErr := enqueuer.EnqueueCatalogRefreshWithOperation(r.Context(), governance.CatalogSource{ + ConnectionID: connectionID, SourceType: governance.SourceTypeDingTalk, Name: resolved.Name, + }, operationID+":first", config.Now().UTC()); enqueueErr != nil { + log.Printf("dingtalk space %s registered but first refresh enqueue failed: %v", connectionID, enqueueErr) + } + } + log.Printf("dingtalk space %q registered as connection %s by %s", resolved.Name, connectionID, actorFor(r)) + writeJSON(w, http.StatusCreated, map[string]any{"data": map[string]string{ + "connection_id": connectionID, "name": resolved.Name, + }}) +} + +var dingtalkSpacePattern = regexp.MustCompile(`/spaces/([A-Za-z0-9_-]+)`) + +func extractDingtalkSpaceID(raw string) string { + match := dingtalkSpacePattern.FindStringSubmatch(raw) + if len(match) == 2 { + return match[1] + } + trimmed := strings.TrimSpace(raw) + if regexp.MustCompile(`^[A-Za-z0-9_-]{8,64}$`).MatchString(trimmed) { + return trimmed + } + return "" +} + func safeSessionID(id string) string { id = strings.TrimSpace(id) if id == "" || len(id) > 64 { diff --git a/internal/httpapi/dingtalk_session_test.go b/internal/httpapi/dingtalk_session_test.go index 72eaac9..a3baf04 100644 --- a/internal/httpapi/dingtalk_session_test.go +++ b/internal/httpapi/dingtalk_session_test.go @@ -70,6 +70,15 @@ func TestDingtalkSessionRejectsUnauthenticatedAndHostileIDs(t *testing.T) { if got := safeSessionID("../etc/passwd"); got != "" { t.Fatalf("hostile id accepted: %q", got) } + for _, test := range []struct{ raw, want string }{ + {"https://alidocs.dingtalk.com/i/spaces/9JOGOQneRgkneX4Q/overview?rnd=1", "9JOGOQneRgkneX4Q"}, + {"9JOGOQneRgkneX4Q", "9JOGOQneRgkneX4Q"}, + {"https://example.com/not-a-space", ""}, + } { + if got := extractDingtalkSpaceID(test.raw); got != test.want { + t.Fatalf("extractDingtalkSpaceID(%q) = %q, want %q", test.raw, got, test.want) + } + } } func TestDingtalkSessionUnavailableWithoutSidecar(t *testing.T) { diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 8e36dc7..87f416d 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -258,6 +258,9 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("POST /api/v1/dingtalk-session", func(w http.ResponseWriter, r *http.Request) { startDingtalkSession(w, r) }) + mux.HandleFunc("POST /api/v1/dingtalk-spaces", func(w http.ResponseWriter, r *http.Request) { + registerDingtalkSpace(w, r, config) + }) mux.HandleFunc("GET /api/v1/dingtalk-session/{id}", func(w http.ResponseWriter, r *http.Request) { getDingtalkSession(w, r) }) diff --git a/internal/platform/postgres/dingtalk_space.go b/internal/platform/postgres/dingtalk_space.go new file mode 100644 index 0000000..f3456f5 --- /dev/null +++ b/internal/platform/postgres/dingtalk_space.go @@ -0,0 +1,92 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" +) + +// DingTalk space registration: a knowledge-base space discovered through the +// login sidecar becomes a source_connections row with a DETERMINISTIC id, +// derived the same way the catalog importer derives stable ids +// ('sc-' + sha256('dingtalk-'+root_key)[:32]). Re-registering the same space +// is an idempotent re-sync of name/root, never a duplicate row. New rows are +// enabled but carry no schedule (next_refresh_at stays NULL): automatic scans +// begin only after an admin sets an interval — same posture as the existing +// imported DingTalk sources. + +// RegisterDingtalkSpaceInput describes one space resolved by the sidecar. +type RegisterDingtalkSpaceInput struct { + RootKey string // sidecar inventory key, e.g. "atom" or "space-1f2e3d4c5b" + Name string // live space name from DingTalk spaceProfile + SpaceID string // alidocs space id from the URL + RootID string // root dentry uuid + Now time.Time +} + +func (in RegisterDingtalkSpaceInput) ConnectionID() string { + sum := sha256.Sum256([]byte("dingtalk-" + in.RootKey)) + return "sc-" + hex.EncodeToString(sum[:])[:32] +} + +func (in RegisterDingtalkSpaceInput) validate() error { + if in.Now.IsZero() { + return errors.New("dingtalk space registration: now is required") + } + if !validShortID(in.RootKey) || strings.TrimSpace(in.Name) == "" || len(in.Name) > 255 { + return errors.New("dingtalk space registration: invalid key or name") + } + for _, id := range []string{in.SpaceID, in.RootID} { + if id == "" || len(id) > 128 || strings.ContainsAny(id, " \t\r\n/") { + return errors.New("dingtalk space registration: invalid identifier") + } + } + return nil +} + +func (s *Store) RegisterDingtalkSpace(ctx context.Context, in RegisterDingtalkSpaceInput) (string, error) { + if err := in.validate(); err != nil { + return "", err + } + connectionID := in.ConnectionID() + var existingName, existingRoot string + err := s.db.QueryRow(ctx, `SELECT name, base_url FROM source_connections WHERE id = $1`, connectionID).Scan(&existingName, &existingRoot) + switch { + case errors.Is(err, pgx.ErrNoRows): + // fresh registration below + case err != nil: + return "", fmt.Errorf("load dingtalk connection: %w", err) + default: + // A different space resolved under the same key must not silently + // rewrite the existing connection. + if !strings.HasPrefix(existingRoot, "spaces/"+in.RootID+"?") { + return "", fmt.Errorf("dingtalk space registration: connection %s already serves another root", connectionID) + } + } + // base_url stores a stable, non-secret locator (the root dentry uuid plus + // the space id) so re-registration can verify identity and the UI can + // explain what a DingTalk connection points at. + locator := "spaces/" + in.RootID + "?space=" + in.SpaceID + _, err = s.db.Exec(ctx, ` +INSERT INTO source_connections ( + id, source_type, name, base_url, secret_ref, enabled, + schedule, last_health, created_at, updated_at, updated_by +) VALUES ($1, 'dingtalk', $2, $3, '', TRUE, '{}'::jsonb, '{}'::jsonb, $4, $4, $5) +ON CONFLICT (id) DO UPDATE +SET name = EXCLUDED.name, base_url = EXCLUDED.base_url, + updated_at = EXCLUDED.updated_at, updated_by = EXCLUDED.updated_by`, + connectionID, in.Name, locator, in.Now.UTC(), "dingtalk-registration") + if err != nil { + if isUniqueViolation(err) { + return "", fmt.Errorf("dingtalk space registration: name %q already used by another connection", in.Name) + } + return "", fmt.Errorf("register dingtalk space: %w", err) + } + return connectionID, nil +} diff --git a/internal/platform/postgres/dingtalk_space_test.go b/internal/platform/postgres/dingtalk_space_test.go new file mode 100644 index 0000000..62ac0a0 --- /dev/null +++ b/internal/platform/postgres/dingtalk_space_test.go @@ -0,0 +1,25 @@ +package postgres + +import ( + "testing" + "time" +) + +func TestRegisterDingtalkSpaceConnectionIDIsStableAndCompatibleWithImporter(t *testing.T) { + input := RegisterDingtalkSpaceInput{ + RootKey: "atom", Name: "ATOM", SpaceID: "9JOGOQneRgkneX4Q", + RootID: "root-atom", Now: time.Now(), + } + first := input.ConnectionID() + second := input.ConnectionID() + if first != second || len(first) != 35 || first[:3] != "sc-" { + t.Fatalf("unstable connection id: %q / %q", first, second) + } + if err := input.validate(); err != nil { + t.Fatalf("valid registration rejected: %v", err) + } + input.SpaceID = "../bad" + if err := input.validate(); err == nil { + t.Fatal("hostile space id accepted") + } +} diff --git a/scripts/live_catalog_refresh.py b/scripts/live_catalog_refresh.py index 7c8eccb..31e9ef3 100644 --- a/scripts/live_catalog_refresh.py +++ b/scripts/live_catalog_refresh.py @@ -4,7 +4,7 @@ Runs only inside the ordinary governance worker. It reads restricted credentials uses allowlisted read-only APIs, writes temporary files under /tmp, and never downloads AList content or invokes source write/delete APIs. """ -import argparse,csv,io,json,os,pathlib,subprocess,sys,tempfile +import argparse,csv,io,json,os,pathlib,re,subprocess,sys,tempfile from collections import deque sys.path.insert(0,'/app/scripts') from source_catalog_import import build_alist_source,build_dingtalk_sources,build_pp_live_source,catalog_sql,validate_source @@ -54,9 +54,19 @@ def refresh_pp(): emit('building',78,'本地内容清单暂不可用,将记录为新资料待校验') return build_pp_live_source(inventory,manifest) -def refresh_dingtalk(name): +def refresh_dingtalk(name,connection_id=''): root=pathlib.Path('/sources/dingtalk/data/dingtalk-internal/inventory-state.json');state=json.loads(root.read_text()) - matches=[(key,value) for key,value in state['roots'].items() if value.get('live_space_name')==name] + # Prefer the root registered on the connection row (spaces/?space=): + # name matching is the legacy fallback for the imported sources. + root_id='' + if connection_id and re.fullmatch(r'[A-Za-z0-9_.:-]{1,36}',connection_id): + try: + rows=subprocess.run(['psql','-X','-At','-d',os.environ.get('DATABASE_URL',''),'-c',"SELECT base_url FROM source_connections WHERE id='"+connection_id+"'"],capture_output=True,text=True,timeout=30) + locator=rows.stdout.strip() if rows.returncode==0 else '' + if locator.startswith('spaces/'):root_id=locator.split('?',1)[0][len('spaces/'):] + except Exception:root_id='' + matches=[(key,value) for key,value in state['roots'].items() if (root_id and value.get('configured_root_id')==root_id) or value.get('live_space_name')==name] + if root_id and len(matches)>1:matches=[m for m in matches if m[1].get('configured_root_id')==root_id] if len(matches)!=1:raise RuntimeError('DingTalk source root missing') key,current=matches[0];output=pathlib.Path('/tmp')/('dingtalk-'+key+'-live.json') if output.exists():output.unlink() @@ -72,7 +82,7 @@ def refresh_dingtalk(name): current_progress=10+(65*done//total if total else 65);last=max(last,current_progress) emit('scanning',last,f"正在扫描目录:已完成 {done} 个文件夹,待处理 {pending} 个",done,total,scanned_directories=done,discovered_directories=total,scanned_files=files,estimated_total_files=files,total_is_estimate=True) if process.wait()!=0 or not output.is_file():raise RefreshBlocked('钉钉登录会话已失效,请在来源链接重新扫码') - payload=json.loads(output.read_text());sources=build_dingtalk_sources(payload);match=[source for source in sources if source['name']==name] + payload=json.loads(output.read_text());sources=build_dingtalk_sources(payload);match=[source for source in sources if source.get('root_node_id')==current['configured_root_id'] or source['name']==name] if len(match)!=1:raise RuntimeError('DingTalk refreshed source missing') emit('building',80,'正在校验钉钉父子目录和文件数量') return match[0] @@ -114,11 +124,11 @@ def refresh_alist(name): return build_alist_source(slug,name,dir_buf,file_buf) def main(): - parser=argparse.ArgumentParser();parser.add_argument('--connection-id',required=True);parser.add_argument('--source-type',choices=('pp','dingtalk','alist'),required=True);parser.add_argument('--source-name',required=True);args=parser.parse_args() + parser=argparse.ArgumentParser();parser.add_argument('--connection-id',required=True);parser.add_argument('--source-type',choices=('pp','dingtalk','alist'),required=True);parser.add_argument('--source-name',required=True);parser.add_argument('--source-space-id',default='');args=parser.parse_args() global _source_name;_source_name=args.source_name try: if args.source_type=='pp':source=refresh_pp() - elif args.source_type=='dingtalk':source=refresh_dingtalk(args.source_name) + elif args.source_type=='dingtalk':source=refresh_dingtalk(args.source_name,args.connection_id) else:source=refresh_alist(args.source_name) validate_source(source);emit('activating',92,'正在写入不可变快照并切换当前版本',source['node_count'],source['node_count']) run_psql(catalog_sql(source,args.connection_id,apply=True));emit('completed',100,f"实时快照完成:{source['folder_count']-1} 个目录,{source['file_count']} 份文件",source['node_count'],source['node_count']) diff --git a/services/dingtalk-login/server.mjs b/services/dingtalk-login/server.mjs index 524ccda..337c0e3 100644 --- a/services/dingtalk-login/server.mjs +++ b/services/dingtalk-login/server.mjs @@ -1,18 +1,17 @@ // DingTalk login sidecar for the knowledge governance platform. -// Drives a persistent headless Chromium. A login session opens the target -// space page, screenshots the QR code when the browser session is dead, -// watches for a real authenticated XHR to fly, validates it, then atomically -// writes the scanner capture file. Tokens never leave this container except -// inside the written capture file. +// Drives a persistent headless Chromium for scan-code login, and uses the +// resulting capture (a-token + cookie headers, exactly what the scanner +// replays) to resolve and register knowledge-base spaces. Tokens never leave +// this container except inside the capture file. import { chromium } from "playwright"; import fs from "node:fs"; -import path from "node:path"; import http from "node:http"; import crypto from "node:crypto"; const PORT = Number(process.env.PORT || 8091); const PROFILE_DIR = process.env.PROFILE_DIR || "/profile"; const CAPTURE_FILE = process.env.CAPTURE_FILE || "/secrets/dingtalk_capture.json"; +const STATE_FILE = process.env.DINGTALK_STATE_FILE || ""; const VERIFY_SPACE = process.env.VERIFY_SPACE || "9JOGO6332vlBnm4Q"; const QR_TTL_MS = Number(process.env.QR_TTL_MS || 5 * 60 * 1000); const SCAN_URL = (space) => `https://alidocs.dingtalk.com/i/spaces/${space}/overview`; @@ -39,29 +38,19 @@ function closeSession(id) { s.context?.close().catch(() => {}); } -// Keep at most a couple of live login contexts. function evictOld() { const live = [...sessions.values()].sort((a, b) => a.createdAt - b.createdAt); while (live.length > 3) closeSession(live.shift().id); } -function validateHeadersSync(headers) { - return Boolean(headers["a-token"] && headers["cookie"]); -} +// ---------- capture ---------- -async function validateHeaders(page, headers) { +function loadCapture() { try { - const result = await page.evaluate(async (h) => { - const r = await fetch(`https://alidocs.dingtalk.com/box/api/v2/dentry/list?dentryUuid=${h.space}&withParentAncestors=true&orderType=SORT_KEY&sortType=desc&listDentrySource=2&pageSize=1`, { - headers: { accept: "application/json", "a-token": h.at, "x-xsrf-token": h.xsrf }, credentials: "include", - }); - const body = await r.text(); - return { status: r.status, ok: r.status === 200 && body.includes("spaceProfile"), name: (body.match(/"name":"([^"]{1,60})"/) || [])[1] }; - }, { space: VERIFY_SPACE, at: headers["a-token"], xsrf: headers["x-xsrf-token"] || "" }); - return result; - } catch (e) { - return { status: -1, ok: false, error: String(e) }; - } + const capture = JSON.parse(fs.readFileSync(CAPTURE_FILE, "utf8")); + const h = capture.headers || {}; + return h["a-token"] ? h : null; + } catch { return null; } } function writeCapture(headers) { @@ -80,6 +69,81 @@ function writeCapture(headers) { fs.closeSync(fd); } +// ---------- box API via the captured browser headers ---------- + +async function boxApi(headers, path) { + const res = await fetch("https://alidocs.dingtalk.com" + path, { + headers: { + accept: "application/json", + "a-token": headers["a-token"], + "x-xsrf-token": headers["x-xsrf-token"] || "", + cookie: headers.cookie || "", + "user-agent": headers["user-agent"] || "Mozilla/5.0", + referer: "https://alidocs.dingtalk.com/", + }, + }); + let body = null; + try { body = await res.json(); } catch { /* non-json */ } + return { status: res.status, body }; +} + +function extractSpace(value) { + const text = String(value || "").trim(); + const m = text.match(/\/spaces\/([A-Za-z0-9_-]+)/) || (/^[A-Za-z0-9_-]{8,64}$/.test(text) ? [null, text] : null); + return m ? m[1] : null; +} + +async function resolveSpace(spaceId) { + const headers = loadCapture(); + if (!headers) return { error: "login_required" }; + const { status, body } = await boxApi(headers, + `/box/api/v2/dentry/list?dentryUuid=${encodeURIComponent(spaceId)}&withParentAncestors=true&orderType=SORT_KEY&sortType=desc&listDentrySource=2&pageSize=5`); + if (status === 401 || (body && body.code === "52600001")) return { error: "login_required" }; + const sp = body && body.data && body.data.spaceProfile; + if (status !== 200 || !sp || !sp.rootDentryUuid || !sp.name) return { error: "space_not_found", http_status: status }; + return { space_id: spaceId, name: String(sp.name).slice(0, 120), root: String(sp.rootDentryUuid) }; +} + +function loadState() { + if (!STATE_FILE) throw new Error("state file not configured"); + return JSON.parse(fs.readFileSync(STATE_FILE, "utf8")); +} + +async function addSpace(spaceId, label) { + const resolved = await resolveSpace(spaceId); + if (resolved.error) return resolved; + const state = loadState(); + const roots = state.roots || (state.roots = {}); + const existing = Object.entries(roots).find(([, r]) => r.configured_root_id === resolved.root); + const rootKey = /^[a-z][a-z0-9_-]{1,31}$/.test(label || "") ? label : existing ? existing[0] : "space-" + crypto.createHash("sha256").update(spaceId).digest("hex").slice(0, 10); + const clash = Object.entries(roots).find(([k, r]) => r.live_space_name === resolved.name && k !== rootKey); + if (clash) return { error: "name_conflict", message: `已登记同名空间「${resolved.name}」` }; + if (!roots[rootKey]) { + roots[rootKey] = { + configured_root_id: resolved.root, + // Seed the live name so the refresh job matches this root immediately; + // the scanner confirms/overwrites it from the same spaceProfile. + live_space_name: resolved.name, + queue: [resolved.root], + scanned_folders: [], + nodes: {}, + errors: [], + }; + } else { + roots[rootKey].live_space_name = resolved.name; + roots[rootKey].configured_root_id = resolved.root; + } + // The inventory state is reached through a DIRECTORY bind mount: readers + // open it by path, so an atomic temp+rename swap is safe and avoids + // exposing half-written JSON to a refresh that starts mid-write. + const tmp = STATE_FILE + ".tmp"; + fs.writeFileSync(tmp, JSON.stringify(state, null, 2)); + fs.renameSync(tmp, STATE_FILE); + return { key: rootKey, name: resolved.name, root: resolved.root, reused: Boolean(existing) }; +} + +// ---------- login sessions ---------- + async function startSession(id, space) { const s = sessions.get(id); if (!s) return; @@ -87,22 +151,24 @@ async function startSession(id, space) { try { context = await chromium.launchPersistentContext(PROFILE_DIR, { headless: true, viewport: { width: 1280, height: 900 }, locale: "zh-CN", + deviceScaleFactor: 3, // crisp QR: canvas screenshots inherit DPR }); - } catch (e) { + } catch { s.state = "failed"; s.error = "browser_launch_failed"; return; } s.context = context; const page = context.pages()[0] ?? await context.newPage(); let capturedHeaders = null; - const finish = async (headers, via) => { - const v = await validateHeaders(page, headers); - if (v.ok) { + const finish = async (headers) => { + const probe = await boxApi(headers, `/box/api/v2/dentry/list?dentryUuid=${VERIFY_SPACE}&pageSize=1&orderType=SORT_KEY&sortType=desc&listDentrySource=2&withParentAncestors=true`); + if (probe.status === 200 && probe.body && probe.body.data && probe.body.data.spaceProfile) { writeCapture(headers); - s.state = "success"; s.spaceName = v.name || ""; s.error = null; + s.state = "success"; s.error = null; } else { s.state = "failed"; s.error = "capture_validation_failed"; } + s.expiresAt = Math.max(s.expiresAt, Date.now() + 30000); scheduleCleanup(s); }; @@ -110,67 +176,117 @@ async function startSession(id, space) { const url = request.url(); if (!url.includes("dingtalk.com")) return; const h = request.headers(); - if (!h["a-token"] || !h.cookie) return; - if (capturedHeaders) return; + if (!h["a-token"] || !h.cookie || capturedHeaders) return; capturedHeaders = { ...h }; delete capturedHeaders["content-length"]; delete capturedHeaders["accept-encoding"]; delete capturedHeaders["host"]; - if (validateHeadersSync(capturedHeaders)) { - s.state = s.state === "qr" ? "scanned" : "reusing"; - if (!s.tokenSeen) { s.tokenSeen = true; finish(capturedHeaders, "request"); } - } + s.state = s.state === "qr" ? "scanned" : "reusing"; + finish(capturedHeaders); }); + // Read the QR directly from DingTalk's canvas. Do NOT screenshot the DOM: + // Playwright element screenshots on a high-DPR page can apply the crop in + // device pixels twice, producing exactly the clipped page thumbnail the + // user reported. canvas.toDataURL() returns the complete PNG pixels. + const grabQR = async () => { + try { + return await page.evaluate(() => { + let best = null, bestDark = 0; + for (const el of document.querySelectorAll('canvas')) { + const r = el.getBoundingClientRect(); + if (r.width < 100 || r.width > 460 || Math.abs(r.width - r.height) > 24) continue; + try { + const pixels = el.getContext('2d').getImageData(0, 0, el.width, el.height).data; + let dark = 0; + for (let i = 0; i < pixels.length; i += 4) { + if (pixels[i + 3] > 0 && pixels[i] + pixels[i + 1] + pixels[i + 2] < 384) dark++; + } + if (dark > bestDark) { best = el; bestDark = dark; } + } catch { /* unreadable canvas: not suitable */ } + } + if (!best || bestDark < 500) return null; + // DingTalk's QR canvas has a transparent background and no guaranteed + // quiet zone. Composite it onto white with 20px padding so phone + // scanners and ordinary QR decoders get a complete code. + const padding = 20; + const out = document.createElement('canvas'); + out.width = best.width + padding * 2; + out.height = best.height + padding * 2; + const ctx = out.getContext('2d'); + ctx.fillStyle = '#fff'; + ctx.fillRect(0, 0, out.width, out.height); + ctx.drawImage(best, padding, padding); + const url = out.toDataURL('image/png'); + return url.startsWith('data:image/png;base64,') ? url.slice('data:image/png;base64,'.length) : null; + }); + } catch { return null; } + }; + await page.goto(SCAN_URL(space), { waitUntil: "domcontentloaded", timeout: 45000 }).catch(() => {}); if (page.url().includes("login.dingtalk.com") && !capturedHeaders) { - // Session is dead: expose the QR code for scanning. - try { - const qr = page.locator("canvas, [class*=qrcode], [class*=QR], img[src*=qrcode]").first(); - const png = (await qr.count()) ? await qr.screenshot({ timeout: 5000 }) : await page.screenshot({ clip: { x: 340, y: 60, width: 600, height: 640 } }); - s.qrPng = png.toString("base64"); - } catch { - try { s.qrPng = (await page.screenshot()).toString("base64"); } catch { /* keep state=qr anyway */ } - } + s.qrPng = await grabQR(); s.state = "qr"; - // Poll for the token to appear after a scan (request listener does the rest). const deadline = Date.now() + QR_TTL_MS; - const timer = setInterval(() => { - if (s.tokenSeen) { clearInterval(timer); return; } + const regrab = setInterval(async () => { + // Keep the freshest frame while waiting for the scan. + if (capturedHeaders) return; + try { const png = await grabQR(); if (png) s.qrPng = png; } catch { /* ignore */ } if (Date.now() > deadline) { - clearInterval(timer); - if (!s.tokenSeen) { s.state = "expired"; scheduleCleanup(s); } + clearInterval(regrab); + if (!capturedHeaders) { s.state = "expired"; scheduleCleanup(s); } } - }, 2000); - timer.unref?.(); + }, 8000); + regrab.unref?.(); } } +// ---------- HTTP ---------- + const server = http.createServer(async (req, res) => { const url = new URL(req.url, `http://127.0.0.1:${PORT}`); const send = (code, obj) => { res.writeHead(code, { "content-type": "application/json" }); res.end(JSON.stringify(obj)); }; - if (req.method === "GET" && url.pathname === "/healthz") return send(200, { status: "ok" }); - const sessionMatch = url.pathname.match(/^\/sessions\/([\w-]+)$/); - if (req.method === "POST" && url.pathname === "/sessions") { - evictOld(); + const readJson = async () => { let body = ""; - for await (const chunk of req) { body += chunk; if (body.length > 4096) break; } - let space = VERIFY_SPACE; - try { const raw = String(JSON.parse(body || "{}").space || "").replace(/[^\w]/g, ""); space = raw || VERIFY_SPACE; } catch { /* default */ } - const id = crypto.randomBytes(9).toString("hex"); - sessions.set(id, { id, state: "starting", space, createdAt: Date.now(), expiresAt: Date.now() + QR_TTL_MS + 60000 }); - startSession(id, space); - return send(201, redacted(sessions.get(id))); - } - if (req.method === "GET" && sessionMatch) { - const s = sessions.get(sessionMatch[1]); - if (!s) return send(404, { error: "session_not_found" }); - return send(200, redacted(s)); - } - if (req.method === "DELETE" && sessionMatch) { - closeSession(sessionMatch[1]); - return send(200, { status: "closed" }); + for await (const chunk of req) { body += chunk; if (body.length > 8192) break; } + try { return JSON.parse(body || "{}"); } catch { return {}; } + }; + try { + if (req.method === "GET" && url.pathname === "/healthz") return send(200, { status: "ok", state_configured: Boolean(STATE_FILE) }); + if (req.method === "POST" && url.pathname === "/sessions") { + evictOld(); + const body = await readJson(); + const space = extractSpace(body.space) || VERIFY_SPACE; + const id = crypto.randomBytes(9).toString("hex"); + sessions.set(id, { id, state: "starting", space, createdAt: Date.now(), expiresAt: Date.now() + QR_TTL_MS + 60000 }); + startSession(id, space); + return send(201, redacted(sessions.get(id))); + } + const sessionMatch = url.pathname.match(/^\/sessions\/([\w-]+)$/); + if (sessionMatch) { + const s = sessions.get(sessionMatch[1]); + if (!s) return send(404, { error: "session_not_found" }); + if (req.method === "GET") return send(200, redacted(s)); + if (req.method === "DELETE") { closeSession(sessionMatch[1]); return send(200, { status: "closed" }); } + } + if (req.method === "POST" && (url.pathname === "/spaces/resolve" || url.pathname === "/spaces/add")) { + const body = await readJson(); + const space = extractSpace(body.url || body.space); + if (!space) return send(400, { error: "invalid_space_url" }); + if (url.pathname.endsWith("resolve")) { + const r = await resolveSpace(space); + return r.error ? send(r.error === "login_required" ? 409 : 404, r) : send(200, r); + } + const r = await addSpace(space, body.label); + if (r.error) { + const code = r.error === "login_required" ? 409 : r.error === "name_conflict" ? 409 : 404; + return send(code, r); + } + return send(200, r); + } + } catch (e) { + return send(500, { error: "sidecar_internal" }); } send(404, { error: "not_found" }); }); diff --git a/web/src/DingtalkSpaceRegistrar.test.tsx b/web/src/DingtalkSpaceRegistrar.test.tsx new file mode 100644 index 0000000..6826a19 --- /dev/null +++ b/web/src/DingtalkSpaceRegistrar.test.tsx @@ -0,0 +1,13 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe,expect,it,vi } from 'vitest' +import { DingtalkSpaceRegistrar } from './DingtalkSpaceRegistrar' + +describe('DingtalkSpaceRegistrar',()=>{ + it('shows an explicit add-source action without making a request',()=>{ + const fetchMock=vi.fn(); vi.stubGlobal('fetch',fetchMock) + const markup=renderToStaticMarkup(undefined}/>) + expect(markup).toContain('新增钉钉知识库') + expect(fetchMock).not.toHaveBeenCalled() + vi.unstubAllGlobals() + }) +}) diff --git a/web/src/DingtalkSpaceRegistrar.tsx b/web/src/DingtalkSpaceRegistrar.tsx new file mode 100644 index 0000000..0b60c50 --- /dev/null +++ b/web/src/DingtalkSpaceRegistrar.tsx @@ -0,0 +1,49 @@ +import { useState } from 'react' + +import { registerDingtalkSpace } from './dingtalkSession' + +// Admin-only entry in 系统设置 → 数据源配置: paste a DingTalk knowledge-base +// space URL (or space id) and register it as a governed source. The sidecar +// resolves name+root with the current scan session; 409 login_required tells +// the operator to refresh the QR first. + +export function DingtalkSpaceRegistrar({ onRegistered }: { onRegistered: () => void }) { + const [open, setOpen] = useState(false) + const [url, setUrl] = useState('') + const [busy, setBusy] = useState(false) + const [failure, setFailure] = useState(null) + const [done, setDone] = useState(null) + + const submit = async () => { + setBusy(true); setFailure(null) + try { + const registered = await registerDingtalkSpace(url.trim()) + setDone(`「${registered.name}」已登记为钉钉来源,首次目录扫描已在后台排队。`) + setUrl('') + onRegistered() + } catch (cause) { + setFailure(cause instanceof Error ? cause.message : '登记钉钉知识库失败') + } finally { + setBusy(false) + } + } + + return
+ + {open ?
+
+

新增钉钉知识库

+

粘贴知识库空间链接(形如 https://alidocs.dingtalk.com/i/spaces/…/overview),平台会解析空间名称与根目录并登记为新来源;首次扫描自动排队。

+ + {done ?

{done}

: null} + {failure ?

{failure}

: null} +
+ + +
+
+
: null} +
+} diff --git a/web/src/SystemSettings.tsx b/web/src/SystemSettings.tsx index 0228e58..dc96540 100644 --- a/web/src/SystemSettings.tsx +++ b/web/src/SystemSettings.tsx @@ -2,6 +2,7 @@ import { useCallback, useEffect, useState } from 'react' import { MembersSection } from './MembersSection' import { DingtalkSessionControl } from './DingtalkSessionControl' +import { DingtalkSpaceRegistrar } from './DingtalkSpaceRegistrar' import { RoutingBindingsSection } from './RoutingBindingsSection' import { useOptionalAuth } from './auth' import { fetchMonitorOverview } from './monitor' @@ -166,7 +167,7 @@ export function SystemSettingsView({ sources, connections, monitor, settings, pr
文档工程师可调整:来源调度周期、刷新间隔、停滞判断管理员设置已锁定:端点、凭据、映射、生产开关
设置服务已启用当前活动版本 · 启用人 {settings.updated_by} · 启用时间 {settings.activated_at ? new Date(settings.activated_at).toLocaleString('zh-CN') : '待激活'};新设置先验证,再原子切换。
{section === 'sources' ?
-

数据源配置

逐来源设置自动刷新周期或保持仅手动;保存会先生成修订、验证后原子启用,旧版本保留用于审计。

+

数据源配置

逐来源设置自动刷新周期或保持仅手动;保存会先生成修订、验证后原子启用,旧版本保留用于审计。

{identity?.role === 'admin' ? : null}
{connections.length ?
{connections.map((connection) => )} {sources.length ? null : 尚无快照数据;快照将在来源首次刷新完成后出现。} diff --git a/web/src/dingtalkSession.ts b/web/src/dingtalkSession.ts index 934de04..2cc7e19 100644 --- a/web/src/dingtalkSession.ts +++ b/web/src/dingtalkSession.ts @@ -41,6 +41,41 @@ export async function cancelDingtalkSession(id: string): Promise { await fetch(`/api/v1/dingtalk-session/${encodeURIComponent(id)}`, { method: 'DELETE', headers: { Accept: 'application/json' } }).catch(() => undefined) } +export interface RegisteredDingtalkSpace { + connection_id: string + name: string +} + +// Register a DingTalk knowledge-base space (URL or bare space id) as a new +// source connection. Errors carry the sidecar's machine codes so the UI can +// explain them: login_required, space_not_found, name_conflict. +export async function registerDingtalkSpace(url: string): Promise { + const response = await fetch('/api/v1/dingtalk-spaces', { + method: 'POST', + headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-Governance-Action': 'dingtalk-space', 'Idempotency-Key': newOperationID() }, + body: JSON.stringify({ url }), + }) + if (!response.ok) { + let code = '' + try { + const parsed = await response.json() as unknown + if (isRecord(parsed) && typeof parsed.error === 'string') code = parsed.error + } catch { /* status-only */ } + const friendly: Record = { + login_required: '钉钉扫码会话无效或已过期,请先在钉钉来源卡片上「扫码刷新会话」', + space_not_found: '未找到该知识库空间:请确认空间链接正确且当前账号有阅读权限', + name_conflict: '该空间与已登记的来源同名', + invalid_space_url: '链接不是有效的钉钉知识库空间地址(应形如 alidocs.dingtalk.com/i/spaces/…)', + } + throw new Error(friendly[code] || `登记钉钉知识库失败(HTTP ${response.status}${code ? ':' + code : ''})`) + } + const value = await response.json() as unknown + if (!isRecord(value) || !isRecord(value.data)) throw new Error('invalid dingtalk space response') + const data = value.data + if (typeof data.connection_id !== 'string' || typeof data.name !== 'string') throw new Error('invalid dingtalk space response') + return { connection_id: data.connection_id, name: data.name } +} + function parseSession(value: unknown): DingtalkSessionView { if (!isRecord(value) || typeof value.id !== 'string' || typeof value.state !== 'string' || !sessionStates.has(value.state)) { throw new Error('invalid dingtalk session response') diff --git a/web/src/systemSettings.css b/web/src/systemSettings.css index cb3ae41..8067173 100644 --- a/web/src/systemSettings.css +++ b/web/src/systemSettings.css @@ -28,7 +28,7 @@ button.link{background:none;border:none;color:var(--c-run);cursor:pointer;text-d @media (max-width:900px){.settings-layout{grid-template-columns:1fr}.settings-nav{position:static;flex-direction:row;flex-wrap:wrap}} .kb-binding-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin:12px 0}.kb-binding-side{display:grid;gap:4px;border:1px solid var(--border-base);border-radius:9px;padding:11px;background:var(--bg-soft)}.kb-binding-side.bound{border-color:var(--c-done)}.kb-binding-side.warn{border-color:var(--c-fail)}.kb-binding-side b{color:var(--ink-base);font-size:11px}.kb-binding-side span{color:var(--c-done);font-size:11px;word-break:break-all}.kb-binding-side.warn span{color:var(--c-fail)}.kb-binding-side small{color:var(--ink-mute);font-size:9px}.kb-binding-editor{display:grid;gap:10px;border-top:1px dashed var(--border-base);margin-top:8px;padding-top:12px}.kb-binding-editor label{display:grid;gap:6px;color:var(--ink-mute);font-size:10px}.kb-binding-editor select,.kb-binding-editor input{border:1px solid var(--border-strong);border-radius:7px;padding:8px;background:var(--bg-panel);color:var(--ink-base)}.kb-binding-editor button{justify-self:start;padding:7px 10px}@media(max-width:650px){.kb-binding-grid{grid-template-columns:1fr}}.hint{color:var(--ink-mute);font-size:10px;margin:4px 0} .dingtalk-session-control{display:flex;flex-direction:column;gap:4px;align-self:end} -.dingtalk-qr-card{max-width:360px;text-align:center} +.dingtalk-qr-card{max-width:440px;text-align:center} .dingtalk-qr-hint{color:var(--ink-mute);font-size:13px;min-height:20px} -.dingtalk-qr-image{display:block;margin:10px auto;width:240px;height:240px;image-rendering:pixelated;border:1px solid var(--border-base);border-radius:10px;background:#fff} +.dingtalk-qr-image{display:block;margin:12px auto;width:320px;height:320px;image-rendering:pixelated;border:12px solid #fff;border-radius:4px;background:#fff;box-sizing:content-box} .dingtalk-qr-card .modal-actions{justify-content:center}