fix(artifact): 白名单环境漏传 dws 变量导致钉钉取件整体失败
端到端验收实测(2026-10-10):标注后取件以 artifact_fetch_rejected: credentials 终态失败。 根因:artifactfetch runner 给子进程的是白名单环境(防凭据外泄),但没有透传 dws 取件器 必需的 ARTIFACT_DINGTALK_FETCHER/DWS_BIN/DWS_CONFIG_DIR/DWS_HOME/DWS_DISABLE_KEYCHAIN → dws 找不到配置目录与文件型 keychain($HOME/.local/share/dws-cli 落空)→ 报未登录 → 映射成 credentials 终态(不可重试)→ 且不会回落老路径。脚本单跑正常,因为绕过了白名单。 - handler.go: childEnv() 显式透传 dws 变量(仅非空值);失败时把 stderr 尾部脱敏后带进错误, 否则只能看到裸 code 无法定位(本次排查就卡在这) - 新增 3 个回归测试:环境透传 / 空值不透传 / stderr 上报与敏感内容抑制 - 实测:修复后重新入队 → artifact.fetch completed,产物 sn-753f….docx 4799B(adoc→docx 导出), content_objects 记为 docx/verified
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -39,6 +40,9 @@ type Result struct {
|
||||
Retryable bool `json:"retryable"`
|
||||
Code string `json:"code"`
|
||||
Error string `json:"error"`
|
||||
// StderrTail 是脚本 stderr 的(脱敏、截断后的)尾部,只在失败时用于报告原因;
|
||||
// json:"-" 保证它不参与结果 JSON 契约(解码侧是 DisallowUnknownFields)。
|
||||
StderrTail string `json:"-"`
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
@@ -114,13 +118,13 @@ func (handler *Handler) Handle(ctx context.Context, job jobs.Job) error {
|
||||
if err := handler.store.MarkArtifactFetchMissing(ctx, request.ConnectionID, request.NodeID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)))
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_missing_source: "+safeMessage(result.Error)+stderrDetail(result.StderrTail)))
|
||||
case !result.OK:
|
||||
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, safeMessage(result.Error), now)
|
||||
if result.Retryable {
|
||||
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code))
|
||||
return errors.New("artifact_fetch_retryable: " + safeMessage(result.Code) + stderrDetail(result.StderrTail))
|
||||
}
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)))
|
||||
return jobs.Permanent(jobs.ErrCodeArtifactInvalid, errors.New("artifact_fetch_rejected: "+safeMessage(result.Code)+stderrDetail(result.StderrTail)))
|
||||
}
|
||||
if result.SHA256 == "" || result.SpoolRef == "" || result.SizeBytes <= 0 || result.NodeID != request.NodeID || result.ValidatedFormat == "" {
|
||||
_ = handler.store.MarkArtifactFetchFailed(ctx, request.ConnectionID, request.NodeID, "artifact fetch returned an inconsistent result", now)
|
||||
@@ -170,6 +174,47 @@ func NewCommandRunner(pythonPath, scriptPath, spoolRoot, spoolPrefix string, max
|
||||
return &CommandRunner{PythonPath: pythonPath, ScriptPath: scriptPath, SpoolRoot: spoolRoot, SpoolPrefix: spoolPrefix, MaxBytes: maxBytes}, nil
|
||||
}
|
||||
|
||||
// dwsEnvNames 是 dws 取件器必需的运行环境。子进程用的是白名单环境(为了不把任何
|
||||
// 凭据带进子进程),因此这几项必须显式透传:漏传时 dws 会找不到配置目录与文件型
|
||||
// keychain($HOME/.local/share/dws-cli 落空),报"未登录"并被映射成 credentials 终态
|
||||
// ——钉钉取件在真实链路上会整体失败(2026-10-10 端到端验收实测;脚本单跑正常,
|
||||
// 因为那只绕过了这个白名单)。
|
||||
var dwsEnvNames = []string{
|
||||
"ARTIFACT_DINGTALK_FETCHER", "DWS_BIN", "DWS_CONFIG_DIR", "DWS_HOME", "DWS_DISABLE_KEYCHAIN",
|
||||
}
|
||||
|
||||
func forwardedDWSEnv() []string {
|
||||
forwarded := make([]string, 0, len(dwsEnvNames))
|
||||
for _, name := range dwsEnvNames {
|
||||
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||
forwarded = append(forwarded, name+"="+value)
|
||||
}
|
||||
}
|
||||
return forwarded
|
||||
}
|
||||
|
||||
// childEnv 是取件子进程的完整环境:固定白名单 + dws 透传项。
|
||||
func (runner *CommandRunner) childEnv() []string {
|
||||
env := []string{
|
||||
"PATH=/usr/bin:/bin",
|
||||
"PYTHONUNBUFFERED=1",
|
||||
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
|
||||
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
|
||||
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
|
||||
"TZ=UTC",
|
||||
}
|
||||
return append(env, forwardedDWSEnv()...)
|
||||
}
|
||||
|
||||
// stderrDetail 把脚本 stderr 尾部附到错误里便于定位(先脱敏,再截断)。
|
||||
func stderrDetail(tail string) string {
|
||||
detail := safeMessage(tail)
|
||||
if detail == "" {
|
||||
return ""
|
||||
}
|
||||
return " [" + detail + "]"
|
||||
}
|
||||
|
||||
func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result, error) {
|
||||
manifestJSON, err := json.Marshal(map[string]any{
|
||||
"spec_id": manifest.VersionID, "node_id": manifest.NodeID, "connection_id": manifest.ConnectionID,
|
||||
@@ -182,20 +227,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result
|
||||
}
|
||||
command := exec.CommandContext(ctx, runner.PythonPath, runner.ScriptPath, string(manifestJSON))
|
||||
// The literal guard keeps secrets out of the child environment: the script
|
||||
// only ever reads pre-mounted files.
|
||||
command.Env = []string{
|
||||
"PATH=/usr/bin:/bin",
|
||||
"PYTHONUNBUFFERED=1",
|
||||
"INGESTION_SPOOL_ROOT=" + runner.SpoolRoot,
|
||||
"SPOOL_REF_PREFIX=" + runner.SpoolPrefix,
|
||||
"ARTIFACT_FETCH_MAX_BYTES=" + strconv.FormatInt(runner.MaxBytes, 10),
|
||||
"TZ=UTC",
|
||||
}
|
||||
// only ever reads pre-mounted files. dws 取件器的运行环境在白名单里显式透传(见 childEnv)。
|
||||
command.Env = runner.childEnv()
|
||||
stdout, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return Result{}, errors.New("artifact_fetch_start_failed")
|
||||
}
|
||||
command.Stderr = &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
|
||||
stderr := &limitedWriter{buffer: bytes.NewBuffer(nil), limit: 4096}
|
||||
command.Stderr = stderr
|
||||
if err := command.Start(); err != nil {
|
||||
return Result{}, errors.New("artifact_fetch_start_failed")
|
||||
}
|
||||
@@ -220,11 +259,14 @@ func (runner *CommandRunner) Run(ctx context.Context, manifest Manifest) (Result
|
||||
}
|
||||
waitErr := command.Wait()
|
||||
if !decoded {
|
||||
tail := stderr.buffer.String()
|
||||
if waitErr != nil {
|
||||
return Result{OK: false, Error: "artifact fetch exited without a result"}, errors.New("artifact_fetch_failed")
|
||||
return Result{OK: false, StderrTail: tail, Error: "artifact fetch exited without a result"},
|
||||
errors.New("artifact_fetch_failed" + stderrDetail(tail))
|
||||
}
|
||||
return Result{}, errors.New("artifact_fetch_no_output")
|
||||
return Result{}, errors.New("artifact_fetch_no_output" + stderrDetail(tail))
|
||||
}
|
||||
result.StderrTail = stderr.buffer.String()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package artifactfetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"os"
|
||||
)
|
||||
|
||||
// 回归(2026-10-10 端到端验收实测):取件子进程用的是白名单环境(为防凭据外泄),
|
||||
// dws 取件器的运行变量必须显式透传。漏传时 dws 找不到配置目录与文件型 keychain,
|
||||
// 报"未登录"→ 被映射成 credentials 终态 → 钉钉取件在真实链路上整体失败
|
||||
// (脚本单跑却正常,因为那只绕过了这个白名单)。
|
||||
func TestChildEnvForwardsDWSVariables(t *testing.T) {
|
||||
t.Setenv("DWS_HOME", "/data/home")
|
||||
t.Setenv("DWS_CONFIG_DIR", "/data/dws")
|
||||
t.Setenv("DWS_DISABLE_KEYCHAIN", "1")
|
||||
t.Setenv("ARTIFACT_DINGTALK_FETCHER", "auto")
|
||||
t.Setenv("DWS_BIN", "/usr/local/bin/dws")
|
||||
t.Setenv("PGPASSWORD", "should-not-leak")
|
||||
runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(runner.childEnv(), "\n")
|
||||
for _, want := range []string{
|
||||
"DWS_HOME=/data/home", "DWS_CONFIG_DIR=/data/dws", "DWS_DISABLE_KEYCHAIN=1",
|
||||
"ARTIFACT_DINGTALK_FETCHER=auto", "DWS_BIN=/usr/local/bin/dws",
|
||||
"INGESTION_SPOOL_ROOT=/spool", "TZ=UTC",
|
||||
} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("childEnv 缺少 %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(joined, "PGPASSWORD") {
|
||||
t.Fatal("白名单之外的变量不得进入子进程环境")
|
||||
}
|
||||
}
|
||||
|
||||
// 未设置时不透传空值(让 dws 用自己的默认值)。
|
||||
func TestChildEnvOmitsUnsetDWSVariables(t *testing.T) {
|
||||
for _, name := range dwsEnvNames {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
runner, err := NewCommandRunner("/usr/bin/python3", "/app/scripts/artifact_fetch.py", "/spool", "", 1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if joined := strings.Join(runner.childEnv(), "\n"); strings.Contains(joined, "DWS_HOME=") {
|
||||
t.Fatalf("空值不应透传:%s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
// 失败时把脚本 stderr 尾部脱敏后带进错误,否则操作者只能看到裸错误码(本次排查就卡在这)。
|
||||
func TestRunSurfacesSanitizedStderrOnFailure(t *testing.T) {
|
||||
python, err := exec.LookPath("python3")
|
||||
if err != nil {
|
||||
t.Skip("python3 not available")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
script := filepath.Join(dir, "fake_fetch.py")
|
||||
payload := "import json,sys\n" +
|
||||
"sys.stderr.write('dws: 未登录,请先执行 dws auth login\\n')\n" +
|
||||
"print(json.dumps({'ok': False, 'code': 'credentials', 'error': 'not authenticated', 'node_id': 'sn-x'}))\n"
|
||||
if err := os.WriteFile(script, []byte(payload), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner, err := NewCommandRunner(python, script, dir, "", 4096)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := runner.Run(context.Background(), Manifest{Request: Request{NodeID: "sn-x"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.OK || result.Code != "credentials" {
|
||||
t.Fatalf("应返回失败结果:%+v", result)
|
||||
}
|
||||
if !strings.Contains(result.StderrTail, "dws") {
|
||||
t.Fatalf("应带回 stderr 尾部,实际 %q", result.StderrTail)
|
||||
}
|
||||
if detail := stderrDetail(result.StderrTail); !strings.Contains(detail, "未登录") {
|
||||
t.Fatalf("stderrDetail 应含原因,实际 %q", detail)
|
||||
}
|
||||
if suppressed := stderrDetail("Authorization: Bearer abc https://example/x"); strings.Contains(suppressed, "Bearer") {
|
||||
t.Fatalf("敏感内容必须被抑制,实际 %q", suppressed)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user