refactor(cli): --limit/--all-pages, Go 1.26, internal/agent → aiclient

Cross-cutting cleanup that lands alongside the new feature surface:

- `--limit / -L` and `--all-pages` on every list command. Default
  --limit 30 (gh-parity); --all-pages drains every server page
  client-side, capped by --limit. Closes the audit finding that the
  old "1000 max per call" implicit cap was undiscoverable.
- `auth token` emits a TTY-only stderr advisory when stdout is a
  terminal (the credential just got displayed in scrollback) plus an
  api-key-mode rotation hint.
- Comment + doc discipline pass: drop external project name
  references from in-code comments (we reference them in design notes,
  not inline).
- Bump `go` directive to 1.26.0 and CI matrix to 1.26.x to align with
  the main module's go.mod.
- Rename `cli/internal/agent` → `cli/internal/aiclient` to
  disambiguate from the new `cli/cmd/agent` resource subtree. The
  package handles AI coding-agent env detection + per-command --help
  annotations; the new name reflects that more precisely.
This commit is contained in:
nullkey
2026-05-14 23:28:14 +08:00
committed by lyingbug
parent 9bb83b47fd
commit bdc589e1c0
59 changed files with 837 additions and 275 deletions
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/agent"
"github.com/Tencent/WeKnora/cli/internal/aiclient"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/format"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
@@ -70,7 +70,7 @@ without the user's explicit go-ahead.`,
},
}
cmdutil.AddJSONFlags(cmd, docDeleteFields)
agent.SetAgentHelp(cmd, "Destructively deletes one document by id. ALWAYS pass -y/--yes in agent mode (no TTY ⇒ confirm prompt fails). Returns data: {id, deleted:true}.")
aiclient.SetAgentHelp(cmd, "Destructively deletes one document by id. ALWAYS pass -y/--yes in agent mode (no TTY ⇒ confirm prompt fails). Returns data: {id, deleted:true}.")
return cmd
}
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/agent"
"github.com/Tencent/WeKnora/cli/internal/aiclient"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
)
@@ -57,7 +57,7 @@ Existing files are NOT overwritten unless --clobber is passed.`,
}
cmd.Flags().StringVarP(&opts.Output, "output", "O", "", `Output path; "-" for stdout. Defaults to the server-suggested filename.`)
cmd.Flags().BoolVar(&opts.Clobber, "clobber", false, "Overwrite the output file if it already exists")
agent.SetAgentHelp(cmd, "Downloads a document. Agents that pipe the bytes should pass --output - (stdout). The exit code is 0 on success, 1 on resource.not_found, 5 on missing --output when the server didn't supply a filename.")
aiclient.SetAgentHelp(cmd, "Downloads a document. Agents that pipe the bytes should pass --output - (stdout). The exit code is 0 on success, 1 on resource.not_found, 5 on missing --output when the server didn't supply a filename.")
return cmd
}
+78 -35
View File
@@ -11,7 +11,7 @@ import (
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/agent"
"github.com/Tencent/WeKnora/cli/internal/aiclient"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/format"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
@@ -31,9 +31,15 @@ var docListFields = []string{
}
type ListOptions struct {
Page int
PageSize int
PageSize int // Items per server batch. With --all-pages, controls
// per-request load. Without, controls the single page size.
Status string // --status: filter by parse_status (server-side query param)
// Limit caps the returned items client-side (default 30; 0 = no cap).
// Applied after pagination / --all-pages accumulation and sort.
Limit int
// AllPages walks server pages internally, accumulating items until
// total exhausted or --limit hit.
AllPages bool
}
// docListStatusValues mirrors internal/types/knowledge.go ParseStatus*
@@ -48,17 +54,12 @@ type ListService interface {
ListKnowledgeWithFilter(ctx context.Context, kbID string, page, pageSize int, filter sdk.KnowledgeListFilter) ([]sdk.Knowledge, int64, error)
}
// listResult is the typed payload emitted under data on success.
//
// Items is non-nil even when empty (json:"[]" not "null") so agents can iterate
// without nil-checks. Page metadata is duplicated here (and not in _meta) to
// keep the payload self-describing for downstream consumers that strip _meta.
// listResult is the typed payload emitted under data. Pagination
// metadata (page / page_size / total) lives in envelope `_meta`, and
// kb_id is in `_meta.kb_id`, so this stays a single-shape `{items}`
// payload consistent with every other list command.
type listResult struct {
Items []sdk.Knowledge `json:"items"`
Page int `json:"page"`
PageSize int `json:"page_size"`
Total int64 `json:"total"`
KBID string `json:"kb_id"`
Items []sdk.Knowledge `json:"items"`
}
// NewCmdList builds `weknora doc list`.
@@ -77,7 +78,7 @@ backend storage order is not guaranteed and varies between deployments.`,
Example: ` weknora doc list # uses project link / env
weknora doc list --kb a32a63ff-fb36-4874-bcaa-30f48570a694 # explicit UUID
weknora doc list --kb my-kb # resolved by name
weknora doc list --page 2 --json # paginated envelope output`,
weknora doc list --all-pages --json # walk every page`,
Args: cobra.NoArgs,
RunE: func(c *cobra.Command, _ []string) error {
jopts, err := cmdutil.CheckJSONFlags(c)
@@ -98,27 +99,28 @@ backend storage order is not guaranteed and varies between deployments.`,
// --kb is read by Factory.ResolveKB; declare it here so cobra parses the
// value into the command's flag set.
cmd.Flags().String("kb", "", "Knowledge base UUID or name (overrides env / project link)")
cmd.Flags().IntVar(&opts.Page, "page", 1, "Page number (1-based)")
cmd.Flags().IntVar(&opts.PageSize, "page-size", 20, "Items per page (1..1000)")
cmd.Flags().IntVar(&opts.PageSize, "page-size", 50, "Items per server batch (1..1000)")
cmd.Flags().IntVarP(&opts.Limit, "limit", "L", 30, "Maximum results to return (0 = no cap, 1..10000 = explicit)")
cmd.Flags().BoolVar(&opts.AllPages, "all-pages", false, "Walk all server pages until exhausted (or --limit hit)")
cmd.Flags().StringVar(&opts.Status, "status", "", "Filter by parse status: pending | processing | completed | failed")
cmdutil.AddJSONFlags(cmd, docListFields)
agent.SetAgentHelp(cmd, "Lists docs in the resolved KB. Returns data: {items, page, page_size, total, kb_id}; pass --kb when not running inside a project. --status (server-side filter) restricts to a parse-pipeline state — `failed` is the practical one for surfacing ingestion errors.")
aiclient.SetAgentHelp(cmd, "Lists docs in the resolved KB. data.{items}; pagination + kb_id in _meta.{page, page_size, total, kb_id}. --status filters server-side; `failed` surfaces ingestion errors. --all-pages walks every server page until exhausted (capped by --limit), useful for one-shot exports.")
return cmd
}
func runList(ctx context.Context, opts *ListOptions, jopts *cmdutil.JSONOptions, svc ListService, kbID string) error {
if opts.Page < 1 {
return &cmdutil.Error{
Code: cmdutil.CodeInputInvalidArgument,
Message: fmt.Sprintf("--page must be >= 1, got %d", opts.Page),
}
}
if opts.PageSize < 1 || opts.PageSize > 1000 {
return &cmdutil.Error{
Code: cmdutil.CodeInputInvalidArgument,
Message: fmt.Sprintf("--page-size must be in 1..1000, got %d", opts.PageSize),
}
}
if opts.Limit < 0 || opts.Limit > 10000 {
return &cmdutil.Error{
Code: cmdutil.CodeInputInvalidArgument,
Message: fmt.Sprintf("--limit must be in 0..10000 (0 = no cap), got %d", opts.Limit),
}
}
if opts.Status != "" && !validDocListStatus(opts.Status) {
return &cmdutil.Error{
Code: cmdutil.CodeInputInvalidArgument,
@@ -126,10 +128,41 @@ func runList(ctx context.Context, opts *ListOptions, jopts *cmdutil.JSONOptions,
strings.Join(docListStatusValues, " | "), opts.Status),
}
}
items, total, err := svc.ListKnowledgeWithFilter(ctx, kbID, opts.Page, opts.PageSize,
sdk.KnowledgeListFilter{ParseStatus: opts.Status})
if err != nil {
return cmdutil.WrapHTTP(err, "list documents")
filter := sdk.KnowledgeListFilter{ParseStatus: opts.Status}
// Pagination is always 1-indexed internally. --all-pages walks; the
// non-walking path returns the first page only.
var (
items []sdk.Knowledge
total int64
)
if opts.AllPages {
accum := make([]sdk.Knowledge, 0)
page := 1
for {
chunk, t, err := svc.ListKnowledgeWithFilter(ctx, kbID, page, opts.PageSize, filter)
if err != nil {
return cmdutil.WrapHTTP(err, "list documents")
}
total = t
accum = append(accum, chunk...)
if opts.Limit > 0 && len(accum) >= opts.Limit {
accum = accum[:opts.Limit]
break
}
if int64(page*opts.PageSize) >= total || len(chunk) == 0 {
break
}
page++
}
items = accum
} else {
chunk, t, err := svc.ListKnowledgeWithFilter(ctx, kbID, 1, opts.PageSize, filter)
if err != nil {
return cmdutil.WrapHTTP(err, "list documents")
}
items = chunk
total = t
}
if items == nil {
items = []sdk.Knowledge{} // ensure JSON [] not null
@@ -140,16 +173,26 @@ func runList(ctx context.Context, opts *ListOptions, jopts *cmdutil.JSONOptions,
sort.Slice(items, func(i, j int) bool {
return items[i].UpdatedAt.After(items[j].UpdatedAt)
})
r := listResult{
Items: items,
Page: opts.Page,
PageSize: opts.PageSize,
Total: total,
KBID: kbID,
// --limit applies after sort so users get the top-N most-recent items
// when combined with a single-page fetch where page_size > limit.
if opts.Limit > 0 && len(items) > opts.Limit {
items = items[:opts.Limit]
}
r := listResult{Items: items}
if jopts.Enabled() {
return format.WriteEnvelopeFiltered(iostreams.IO.Out, format.Success(r, &format.Meta{KBID: kbID}), jopts.Fields, jopts.JQ)
// --all-pages collapses pagination into a single conceptual page,
// so the meta reflects "you got everything" semantics.
meta := &format.Meta{
KBID: kbID,
Page: 1,
PageSize: opts.PageSize,
Total: total,
}
if !opts.AllPages {
meta.HasMore = int64(opts.PageSize) < total
}
return format.WriteEnvelopeFiltered(iostreams.IO.Out, format.Success(r, meta), jopts.Fields, jopts.JQ)
}
if len(items) == 0 {
+102 -9
View File
@@ -3,6 +3,7 @@ package doc
import (
"context"
"errors"
"fmt"
"os"
"strings"
"testing"
@@ -55,7 +56,7 @@ func TestList_Success_Human(t *testing.T) {
{ID: "doc2", FileName: "beta.md", FileSize: 0, ParseStatus: "pending", UpdatedAt: now.Add(-2 * 24 * time.Hour)},
}
svc := &fakeListSvc{items: items, total: 2}
opts := &ListOptions{Page: 1, PageSize: 20}
opts := &ListOptions{PageSize: 20}
require.NoError(t, runList(context.Background(), opts, nil, svc, "kb_xxx"))
assert.Equal(t, "kb_xxx", svc.got.kbID)
@@ -74,7 +75,7 @@ func TestList_Success_Human(t *testing.T) {
func TestList_Success_JSON(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &fakeListSvc{items: []sdk.Knowledge{{ID: "doc1", FileName: "a.pdf"}}, total: 1}
opts := &ListOptions{Page: 1, PageSize: 20}
opts := &ListOptions{PageSize: 20}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
got := out.String()
@@ -90,7 +91,7 @@ func TestList_Success_JSON(t *testing.T) {
func TestList_Empty_Human(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &fakeListSvc{items: nil, total: 0}
opts := &ListOptions{Page: 1, PageSize: 20}
opts := &ListOptions{PageSize: 20}
require.NoError(t, runList(context.Background(), opts, nil, svc, "kb_xxx"))
assert.Contains(t, out.String(), "(no documents)")
}
@@ -98,7 +99,7 @@ func TestList_Empty_Human(t *testing.T) {
func TestList_Empty_JSON(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &fakeListSvc{items: nil, total: 0}
opts := &ListOptions{Page: 1, PageSize: 20}
opts := &ListOptions{PageSize: 20}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
got := out.String()
@@ -109,7 +110,7 @@ func TestList_Empty_JSON(t *testing.T) {
func TestList_HTTPError_500(t *testing.T) {
_, _ = iostreams.SetForTest(t)
svc := &fakeListSvc{err: errors.New("HTTP error 500: internal")}
opts := &ListOptions{Page: 1, PageSize: 20}
opts := &ListOptions{PageSize: 20}
err := runList(context.Background(), opts, nil, svc, "kb_xxx")
require.Error(t, err)
@@ -188,7 +189,7 @@ func TestList_SortByUpdatedDesc(t *testing.T) {
{ID: "new", FileName: "new.pdf", UpdatedAt: now.Add(-1 * time.Hour)},
}
svc := &fakeListSvc{items: items, total: 2}
require.NoError(t, runList(context.Background(), &ListOptions{Page: 1, PageSize: 20}, nil, svc, "kb_xxx"))
require.NoError(t, runList(context.Background(), &ListOptions{PageSize: 20}, nil, svc, "kb_xxx"))
got := out.String()
newIdx := strings.Index(got, "new.pdf")
@@ -221,7 +222,7 @@ func TestList_StatusFilter_ForwardedToSDK(t *testing.T) {
chdirIsolated(t)
_, _ = iostreams.SetForTest(t)
svc := &fakeListSvc{}
opts := &ListOptions{Page: 1, PageSize: 20, Status: "failed"}
opts := &ListOptions{PageSize: 20, Status: "failed"}
require.NoError(t, runList(context.Background(), opts, nil, svc, "kb_xxx"))
assert.Equal(t, "failed", svc.got.filter.ParseStatus,
"--status must be forwarded as filter.ParseStatus for server-side filtering")
@@ -231,7 +232,7 @@ func TestList_StatusFilter_RejectsUnknownValue(t *testing.T) {
chdirIsolated(t)
_, _ = iostreams.SetForTest(t)
svc := &fakeListSvc{}
opts := &ListOptions{Page: 1, PageSize: 20, Status: "bogus"}
opts := &ListOptions{PageSize: 20, Status: "bogus"}
err := runList(context.Background(), opts, nil, svc, "kb_xxx")
require.Error(t, err)
var typed *cmdutil.Error
@@ -246,9 +247,101 @@ func TestList_StatusFilter_AcceptsAllEnumValues(t *testing.T) {
for _, v := range docListStatusValues {
_, _ = iostreams.SetForTest(t)
svc := &fakeListSvc{}
opts := &ListOptions{Page: 1, PageSize: 20, Status: v}
opts := &ListOptions{PageSize: 20, Status: v}
require.NoError(t, runList(context.Background(), opts, nil, svc, "kb_xxx"),
"status=%q should be accepted", v)
}
}
// makeDocs returns N Knowledge records with distinct IDs and descending
// UpdatedAt timestamps, useful for limit / pagination tests.
func makeDocs(n int) []sdk.Knowledge {
base := time.Now()
out := make([]sdk.Knowledge, n)
for i := 0; i < n; i++ {
out[i] = sdk.Knowledge{
ID: fmt.Sprintf("doc_%02d", i),
FileName: fmt.Sprintf("f_%02d.pdf", i),
UpdatedAt: base.Add(-time.Duration(i) * time.Hour),
}
}
return out
}
// pagedDocSvc returns server-paginated Knowledge results from a flat slice.
// Records the page numbers requested for assertion.
type pagedDocSvc struct {
all []sdk.Knowledge
calls []int // 1-based page numbers received
pageSize int
}
func (p *pagedDocSvc) ListKnowledgeWithFilter(_ context.Context, _ string, page, pageSize int, _ sdk.KnowledgeListFilter) ([]sdk.Knowledge, int64, error) {
p.calls = append(p.calls, page)
p.pageSize = pageSize
start := (page - 1) * pageSize
if start >= len(p.all) {
return []sdk.Knowledge{}, int64(len(p.all)), nil
}
end := start + pageSize
if end > len(p.all) {
end = len(p.all)
}
return p.all[start:end], int64(len(p.all)), nil
}
func TestList_Limit_LessThanPageSize_SlicesToLimit(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &fakeListSvc{items: makeDocs(20), total: 20}
opts := &ListOptions{PageSize: 20, Limit: 5}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
body := out.String()
// Count occurrences of "id":"doc_" — should be exactly 5.
got := strings.Count(body, `"id":"doc_`)
assert.Equal(t, 5, got, "--limit 5 must slice 20 returned items to 5; body=\n%s", body)
}
func TestList_Limit_GreaterThanPageSize_NoCap(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &fakeListSvc{items: makeDocs(10), total: 10}
opts := &ListOptions{PageSize: 10, Limit: 50}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
got := strings.Count(out.String(), `"id":"doc_`)
assert.Equal(t, 10, got, "--limit 50 with page-size 10 + 10 items returns all 10")
}
func TestList_Limit_Negative_Rejected(t *testing.T) {
_, _ = iostreams.SetForTest(t)
opts := &ListOptions{PageSize: 20, Limit: -1}
err := runList(context.Background(), opts, nil, &fakeListSvc{}, "kb_xxx")
require.Error(t, err)
var typed *cmdutil.Error
require.ErrorAs(t, err, &typed)
assert.Equal(t, cmdutil.CodeInputInvalidArgument, typed.Code)
}
func TestList_AllPages_WalksAllServerPages(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &pagedDocSvc{all: makeDocs(45)}
opts := &ListOptions{PageSize: 20, AllPages: true}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
// 45 items / page_size 20 = 3 pages: 20 + 20 + 5.
assert.Equal(t, []int{1, 2, 3}, svc.calls)
got := strings.Count(out.String(), `"id":"doc_`)
assert.Equal(t, 45, got)
}
func TestList_AllPages_WithLimit_StopsAtLimit(t *testing.T) {
out, _ := iostreams.SetForTest(t)
svc := &pagedDocSvc{all: makeDocs(200)}
opts := &ListOptions{PageSize: 20, AllPages: true, Limit: 50}
require.NoError(t, runList(context.Background(), opts, &cmdutil.JSONOptions{}, svc, "kb_xxx"))
got := strings.Count(out.String(), `"id":"doc_`)
assert.Equal(t, 50, got, "--limit 50 with --all-pages should stop after 50 items")
// Should have called pages 1..3 (60 items) then capped at 50.
assert.LessOrEqual(t, len(svc.calls), 3, "should not walk past the page that fills --limit")
}
// (test removed: --page flag was dropped in v0.4 mainstream-alignment;
// --all-pages now walks unconditionally, no mutex case to test.)
+2 -2
View File
@@ -8,7 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/agent"
"github.com/Tencent/WeKnora/cli/internal/aiclient"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/format"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
@@ -125,7 +125,7 @@ Use --recursive --glob to upload a directory tree (see Examples).`,
cmd.Flags().StringVar(&opts.Glob, "glob", "*", "Filename pattern to filter when --recursive (e.g. '*.pdf')")
cmd.Flags().StringVar(&opts.FromURL, "from-url", "", "Ingest a remote `URL` (HTTP/HTTPS) instead of a local file")
cmdutil.AddJSONFlags(cmd, docUploadFields)
agent.SetAgentHelp(cmd, "Uploads to the resolved KB. Default: one local regular file. With --recursive: walks the directory, filters by --glob, uploads each match. With --from-url: ingests a remote URL (mutually exclusive with the positional file / --recursive / --glob). Returns data: single Knowledge object (file or URL) or aggregate report (recursive). Errors: resource.already_exists when the URL was previously ingested.")
aiclient.SetAgentHelp(cmd, "Three modes (mutually exclusive): local file (positional), --recursive directory walk + --glob, or --from-url remote ingest. Returns data: Knowledge object (file/URL) or aggregate report (recursive). Errors: resource.already_exists (URL previously ingested) / local.upload_file_not_found.")
return cmd
}
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/agent"
"github.com/Tencent/WeKnora/cli/internal/aiclient"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/format"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
@@ -53,7 +53,7 @@ func NewCmdView(f *cmdutil.Factory) *cobra.Command {
},
}
cmdutil.AddJSONFlags(cmd, docViewFields)
agent.SetAgentHelp(cmd, "Returns the full Knowledge object for one document: id, kb_id, file metadata, parse_status, processed_at, error_message. Use this instead of paging through `doc list` when you have the doc id.")
aiclient.SetAgentHelp(cmd, "Returns the full Knowledge object for one document: id, kb_id, file metadata, parse_status, processed_at, error_message. Use this instead of paging through `doc list` when you have the doc id.")
return cmd
}