fix(artifact): 白名单环境漏传 dws 变量导致钉钉取件整体失败

端到端验收实测(2026-10-10):标注后取件以 artifact_fetch_rejected: credentials 终态失败。
根因:artifactfetch runner 给子进程的是白名单环境(防凭据外泄),但没有透传 dws 取件器
必需的 ARTIFACT_DINGTALK_FETCHER/DWS_BIN/DWS_CONFIG_DIR/DWS_HOME/DWS_DISABLE_KEYCHAIN
→ dws 找不到配置目录与文件型 keychain($HOME/.local/share/dws-cli 落空)→ 报未登录 →
映射成 credentials 终态(不可重试)→ 且不会回落老路径。脚本单跑正常,因为绕过了白名单。

- handler.go: childEnv() 显式透传 dws 变量(仅非空值);失败时把 stderr 尾部脱敏后带进错误,
  否则只能看到裸 code 无法定位(本次排查就卡在这)
- 新增 3 个回归测试:环境透传 / 空值不透传 / stderr 上报与敏感内容抑制
- 实测:修复后重新入队 → artifact.fetch completed,产物 sn-753f….docx 4799B(adoc→docx 导出),
  content_objects 记为 docx/verified
This commit is contained in:
姚章浩
2026-10-10 09:57:41 +08:00
parent a1d04f07ec
commit d29e04e647
2 changed files with 148 additions and 15 deletions
+57 -15
View File
@@ -10,6 +10,7 @@ import (
"context"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"strconv"
@@ -39,6 +40,9 @@ type Result struct {
Retryable bool `json:"retryable"`
Code string `json:"code"`
Error string `json:"error"`
// StderrTail 是脚本 stderr 的(脱敏、截断后的)尾部,只在失败时用于报告原因;
// json:"-" 保证它不参与结果 JSON 契约(解码侧是 DisallowUnknownFields)。
StderrTail string `json:"-"`
}
type Manifest struct {
@@ -114,13 +118,13 @@ func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil {
return err
}
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)))
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)+stderrDetail(result.StderrTail)))
case !result.OK:
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now)
if result.Retryable {
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code))
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code) + stderrDetail(result.StderrTail))
}
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)))
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)+stderrDetail(result.StderrTail)))
}
if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" {
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now)
@@ -170,6 +174,47 @@ func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, max
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil
}
// dwsEnvNames 是 dws 取件器必需的运行环境。子进程用的是白名单环境(为了不把任何
// 凭据带进子进程),因此这几项必须显式透传:漏传时 dws 会找不到配置目录与文件型
// keychain($HOME/.local/share/dws-cli 落空),报"未登录"并被映射成 credentials 终态
// ——钉钉取件在真实链路上会整体失败(2026-10-10 端到端验收实测;脚本单跑正常,
// 因为那只绕过了这个白名单)。
var dwsEnvNames = []string{
"ARTIFACT_DINGTALK_FETCHER", "DWS_BIN", "DWS_CONFIG_DIR", "DWS_HOME", "DWS_DISABLE_KEYCHAIN",
}
func forwardedDWSEnv() []string {
forwarded := make([]string, 0, len(dwsEnvNames))
for _, name := range dwsEnvNames {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
forwarded = append(forwarded, name+"="+value)
}
}
return forwarded
}
// childEnv 是取件子进程的完整环境:固定白名单 + dws 透传项。
func (runner *CommandRunner) childEnv() []string {
env := []string{
"PATH=/usr/bin:/bin",
"PYTHONUNBUFFERED=1",
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
"TZ=UTC",
}
return append(env, forwardedDWSEnv()...)
}
// stderrDetail 把脚本 stderr 尾部附到错误里便于定位(先脱敏,再截断)。
func stderrDetail(tail string) string {
detail := safeMessage(tail)
if detail == "" {
return ""
}
return " [" + detail + "]"
}
func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) {
manifestJSON, err := json.Marshal(map[string]any{
"spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID,
@@ -182,20 +227,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result
}
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON))
// The literal guard keeps secrets out of the child environment: the script
// only ever reads pre-mounted files.
command.Env = []string{
"PATH=/usr/bin:/bin",
"PYTHONUNBUFFERED=1",
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
"TZ=UTC",
}
// only ever reads pre-mounted files. dws 取件器的运行环境在白名单里显式透传(见 childEnv)。
command.Env = runner.childEnv()
stdout, err := command.StdoutPipe()
if err != nil {
return Result{}, errors.New("artifact_fetch_start_failed")
}
command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
stderr := &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
command.Stderr = stderr
if err := command.Start(); err != nil {
return Result{}, errors.New("artifact_fetch_start_failed")
}
@@ -220,11 +259,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result
}
waitErr := command.Wait()
if !decoded {
tail := stderr.buffer.String()
if waitErr != nil {
return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed")
return Result{OK: false, StderrTail: tail, Error: "artifact fetch exited without a result"},
errors.New("artifact_fetch_failed" + stderrDetail(tail))
}
return Result{}, errors.New("artifact_fetch_no_output")
return Result{}, errors.New("artifact_fetch_no_output" + stderrDetail(tail))
}
result.StderrTail = stderr.buffer.String()
return result, nil
}
@@ -0,0 +1,91 @@
package artifactfetch
import (
"context"
"os/exec"
"path/filepath"
"strings"
"testing"
"os"
)
// 回归(2026-10-10 端到端验收实测):取件子进程用的是白名单环境(为防凭据外泄),
// dws 取件器的运行变量必须显式透传。漏传时 dws 找不到配置目录与文件型 keychain,
// 报"未登录"→ 被映射成 credentials 终态 → 钉钉取件在真实链路上整体失败
// (脚本单跑却正常,因为那只绕过了这个白名单)。
func TestChildEnvForwardsDWSVariables(t *testing.T) {
t.Setenv("DWS_HOME", "/data/home")
t.Setenv("DWS_CONFIG_DIR", "/data/dws")
t.Setenv("DWS_DISABLE_KEYCHAIN", "1")
t.Setenv("ARTIFACT_DINGTALK_FETCHER", "auto")
t.Setenv("DWS_BIN", "/usr/local/bin/dws")
t.Setenv("PGPASSWORD", "should-not-leak")
runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024)
if err != nil {
t.Fatal(err)
}
joined := strings.Join(runner.childEnv(), "\n")
for _, want := range []string{
"DWS_HOME=/data/home", "DWS_CONFIG_DIR=/data/dws", "DWS_DISABLE_KEYCHAIN=1",
"ARTIFACT_DINGTALK_FETCHER=auto", "DWS_BIN=/usr/local/bin/dws",
"INGESTION_SPOOL_ROOT=/spool", "TZ=UTC",
} {
if !strings.Contains(joined, want) {
t.Errorf("childEnv 缺少 %q", want)
}
}
if strings.Contains(joined, "PGPASSWORD") {
t.Fatal("白名单之外的变量不得进入子进程环境")
}
}
// 未设置时不透传空值(让 dws 用自己的默认值)。
func TestChildEnvOmitsUnsetDWSVariables(t *testing.T) {
for _, name := range dwsEnvNames {
t.Setenv(name, "")
}
runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024)
if err != nil {
t.Fatal(err)
}
if joined := strings.Join(runner.childEnv(), "\n"); strings.Contains(joined, "DWS_HOME=") {
t.Fatalf("空值不应透传:%s", joined)
}
}
// 失败时把脚本 stderr 尾部脱敏后带进错误,否则操作者只能看到裸错误码(本次排查就卡在这)。
func TestRunSurfacesSanitizedStderrOnFailure(t *testing.T) {
python, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 not available")
}
dir := t.TempDir()
script := filepath.Join(dir, "fake_fetch.py")
payload := "import json,sys\n" +
"sys.stderr.write('dws: 未登录,请先执行 dws auth login\\n')\n" +
"print(json.dumps({'ok': False, 'code': 'credentials', 'error': 'not authenticated', 'node_id': 'sn-x'}))\n"
if err := os.WriteFile(script, []byte(payload), 0o644); err != nil {
t.Fatal(err)
}
runner, err := NewCommandRunner(python, script, dir, "", 4096)
if err != nil {
t.Fatal(err)
}
result, err := runner.Run(context.Background(), Manifest{Request: Request{NodeID: "sn-x"}})
if err != nil {
t.Fatal(err)
}
if result.OK || result.Code != "credentials" {
t.Fatalf("应返回失败结果:%+v", result)
}
if !strings.Contains(result.StderrTail, "dws") {
t.Fatalf("应带回 stderr 尾部,实际 %q", result.StderrTail)
}
if detail := stderrDetail(result.StderrTail); !strings.Contains(detail, "未登录") {
t.Fatalf("stderrDetail 应含原因,实际 %q", detail)
}
if suppressed := stderrDetail("Authorization: Bearer abc https://example/x"); strings.Contains(suppressed, "Bearer") {
t.Fatalf("敏感内容必须被抑制,实际 %q", suppressed)
}
}