From d29e04e647fda81681a9735b2daee7bc43f1ee1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9A=E7=AB=A0=E6=B5=A9?= Date: Sat, 10 Oct 2026 09:57:41 +0800 Subject: [PATCH] =?UTF-8?q?fix(artifact):=20=E7=99=BD=E5=90=8D=E5=8D=95?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E6=BC=8F=E4=BC=A0=20dws=20=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E9=92=89=E9=92=89=E5=8F=96=E4=BB=B6=E6=95=B4?= =?UTF-8?q?=E4=BD=93=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 端到端验收实测(2026-10-10):标注后取件以 artifact_fetch_rejected: credentials 终态失败。 根因:artifactfetch runner 给子进程的是白名单环境(防凭据外泄),但没有透传 dws 取件器 必需的 ARTIFACT_DINGTALK_FETCHER/DWS_BIN/DWS_CONFIG_DIR/DWS_HOME/DWS_DISABLE_KEYCHAIN → dws 找不到配置目录与文件型 keychain($HOME/.local/share/dws-cli 落空)→ 报未登录 → 映射成 credentials 终态(不可重试)→ 且不会回落老路径。脚本单跑正常,因为绕过了白名单。 - handler.go: childEnv() 显式透传 dws 变量(仅非空值);失败时把 stderr 尾部脱敏后带进错误, 否则只能看到裸 code 无法定位(本次排查就卡在这) - 新增 3 个回归测试:环境透传 / 空值不透传 / stderr 上报与敏感内容抑制 - 实测:修复后重新入队 → artifact.fetch completed,产物 sn-753f….docx 4799B(adoc→docx 导出), content_objects 记为 docx/verified --- internal/workflows/artifactfetch/handler.go | 72 ++++++++++++--- .../artifactfetch/handler_env_test.go | 91 +++++++++++++++++++ 2 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 internal/workflows/artifactfetch/handler_env_test.go diff --git a/internal/workflows/artifactfetch/handler.go b/internal/workflows/artifactfetch/handler.go index db9bd1d..109b217 100644 --- a/internal/workflows/artifactfetch/handler.go +++ b/internal/workflows/artifactfetch/handler.go @@ -10,6 +10,7 @@ import ( "context" "encoding/json" "errors" + "os" "os/exec" "path/filepath" "strconv" @@ -39,6 +40,9 @@ type Result struct { Retryable bool `json:"retryable"` Code string `json:"code"` Error string `json:"error"` + // StderrTail 是脚本 stderr 的(脱敏、截断后的)尾部,只在失败时用于报告原因; + // json:"-" 保证它不参与结果 JSON 契约(解码侧是 DisallowUnknownFields)。 + StderrTail string `json:"-"` } type Manifest struct { @@ -114,13 +118,13 @@ func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error { if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil { return err } - return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error))) + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)+stderrDetail(result.StderrTail))) case !result.OK: _ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now) if result.Retryable { - return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code)) + return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code) + stderrDetail(result.StderrTail)) } - return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code))) + return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)+stderrDetail(result.StderrTail))) } if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" { _ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now) @@ -170,6 +174,47 @@ func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, max return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil } +// dwsEnvNames 是 dws 取件器必需的运行环境。子进程用的是白名单环境(为了不把任何 +// 凭据带进子进程),因此这几项必须显式透传:漏传时 dws 会找不到配置目录与文件型 +// keychain($HOME/.local/share/dws-cli 落空),报"未登录"并被映射成 credentials 终态 +// ——钉钉取件在真实链路上会整体失败(2026-10-10 端到端验收实测;脚本单跑正常, +// 因为那只绕过了这个白名单)。 +var dwsEnvNames = []string{ + "ARTIFACT_DINGTALK_FETCHER", "DWS_BIN", "DWS_CONFIG_DIR", "DWS_HOME", "DWS_DISABLE_KEYCHAIN", +} + +func forwardedDWSEnv() []string { + forwarded := make([]string, 0, len(dwsEnvNames)) + for _, name := range dwsEnvNames { + if value := strings.TrimSpace(os.Getenv(name)); value != "" { + forwarded = append(forwarded, name+"="+value) + } + } + return forwarded +} + +// childEnv 是取件子进程的完整环境:固定白名单 + dws 透传项。 +func (runner *CommandRunner) childEnv() []string { + env := []string{ + "PATH=/usr/bin:/bin", + "PYTHONUNBUFFERED=1", + "INGESTION_SPOOL_ROOT=" + runner.SpoolRoot, + "SPOOL_REF_PREFIX=" + runner.SpoolPrefix, + "ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10), + "TZ=UTC", + } + return append(env, forwardedDWSEnv()...) +} + +// stderrDetail 把脚本 stderr 尾部附到错误里便于定位(先脱敏,再截断)。 +func stderrDetail(tail string) string { + detail := safeMessage(tail) + if detail == "" { + return "" + } + return " [" + detail + "]" +} + func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) { manifestJSON, err := json.Marshal(map[string]any{ "spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID, @@ -182,20 +227,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result } command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON)) // The literal guard keeps secrets out of the child environment: the script - // only ever reads pre-mounted files. - command.Env = []string{ - "PATH=/usr/bin:/bin", - "PYTHONUNBUFFERED=1", - "INGESTION_SPOOL_ROOT=" + runner.SpoolRoot, - "SPOOL_REF_PREFIX=" + runner.SpoolPrefix, - "ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10), - "TZ=UTC", - } + // only ever reads pre-mounted files. dws 取件器的运行环境在白名单里显式透传(见 childEnv)。 + command.Env = runner.childEnv() stdout, err := command.StdoutPipe() if err != nil { return Result{}, errors.New("artifact_fetch_start_failed") } - command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096} + stderr := &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096} + command.Stderr = stderr if err := command.Start(); err != nil { return Result{}, errors.New("artifact_fetch_start_failed") } @@ -220,11 +259,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result } waitErr := command.Wait() if !decoded { + tail := stderr.buffer.String() if waitErr != nil { - return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed") + return Result{OK: false, StderrTail: tail, Error: "artifact fetch exited without a result"}, + errors.New("artifact_fetch_failed" + stderrDetail(tail)) } - return Result{}, errors.New("artifact_fetch_no_output") + return Result{}, errors.New("artifact_fetch_no_output" + stderrDetail(tail)) } + result.StderrTail = stderr.buffer.String() return result, nil } diff --git a/internal/workflows/artifactfetch/handler_env_test.go b/internal/workflows/artifactfetch/handler_env_test.go new file mode 100644 index 0000000..49057ae --- /dev/null +++ b/internal/workflows/artifactfetch/handler_env_test.go @@ -0,0 +1,91 @@ +package artifactfetch + +import ( + "context" + "os/exec" + "path/filepath" + "strings" + "testing" + + "os" +) + +// 回归(2026-10-10 端到端验收实测):取件子进程用的是白名单环境(为防凭据外泄), +// dws 取件器的运行变量必须显式透传。漏传时 dws 找不到配置目录与文件型 keychain, +// 报"未登录"→ 被映射成 credentials 终态 → 钉钉取件在真实链路上整体失败 +// (脚本单跑却正常,因为那只绕过了这个白名单)。 +func TestChildEnvForwardsDWSVariables(t *testing.T) { + t.Setenv("DWS_HOME", "/data/home") + t.Setenv("DWS_CONFIG_DIR", "/data/dws") + t.Setenv("DWS_DISABLE_KEYCHAIN", "1") + t.Setenv("ARTIFACT_DINGTALK_FETCHER", "auto") + t.Setenv("DWS_BIN", "/usr/local/bin/dws") + t.Setenv("PGPASSWORD", "should-not-leak") + runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(runner.childEnv(), "\n") + for _, want := range []string{ + "DWS_HOME=/data/home", "DWS_CONFIG_DIR=/data/dws", "DWS_DISABLE_KEYCHAIN=1", + "ARTIFACT_DINGTALK_FETCHER=auto", "DWS_BIN=/usr/local/bin/dws", + "INGESTION_SPOOL_ROOT=/spool", "TZ=UTC", + } { + if !strings.Contains(joined, want) { + t.Errorf("childEnv 缺少 %q", want) + } + } + if strings.Contains(joined, "PGPASSWORD") { + t.Fatal("白名单之外的变量不得进入子进程环境") + } +} + +// 未设置时不透传空值(让 dws 用自己的默认值)。 +func TestChildEnvOmitsUnsetDWSVariables(t *testing.T) { + for _, name := range dwsEnvNames { + t.Setenv(name, "") + } + runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024) + if err != nil { + t.Fatal(err) + } + if joined := strings.Join(runner.childEnv(), "\n"); strings.Contains(joined, "DWS_HOME=") { + t.Fatalf("空值不应透传:%s", joined) + } +} + +// 失败时把脚本 stderr 尾部脱敏后带进错误,否则操作者只能看到裸错误码(本次排查就卡在这)。 +func TestRunSurfacesSanitizedStderrOnFailure(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 not available") + } + dir := t.TempDir() + script := filepath.Join(dir, "fake_fetch.py") + payload := "import json,sys\n" + + "sys.stderr.write('dws: 未登录,请先执行 dws auth login\\n')\n" + + "print(json.dumps({'ok': False, 'code': 'credentials', 'error': 'not authenticated', 'node_id': 'sn-x'}))\n" + if err := os.WriteFile(script, []byte(payload), 0o644); err != nil { + t.Fatal(err) + } + runner, err := NewCommandRunner(python, script, dir, "", 4096) + if err != nil { + t.Fatal(err) + } + result, err := runner.Run(context.Background(), Manifest{Request: Request{NodeID: "sn-x"}}) + if err != nil { + t.Fatal(err) + } + if result.OK || result.Code != "credentials" { + t.Fatalf("应返回失败结果:%+v", result) + } + if !strings.Contains(result.StderrTail, "dws") { + t.Fatalf("应带回 stderr 尾部,实际 %q", result.StderrTail) + } + if detail := stderrDetail(result.StderrTail); !strings.Contains(detail, "未登录") { + t.Fatalf("stderrDetail 应含原因,实际 %q", detail) + } + if suppressed := stderrDetail("Authorization: Bearer abc https://example/x"); strings.Contains(suppressed, "Bearer") { + t.Fatalf("敏感内容必须被抑制,实际 %q", suppressed) + } +}