feat(cli): doc delete multi-id + api --paginate + paginate fixes + batch deletes

Batch-write surface and pagination consistency:

* weknora doc delete <doc-id> [<doc-id>...] — positional multi-id, default
  keep-going on failure. Single -y confirms entire batch.
* weknora session delete <session-id> [<session-id>...] — same shape.
* weknora chunk delete <chunk-id> [<chunk-id>...] --doc <doc-id> — multi-id
  with shared --doc parent.
* Multi-id partial-failure rolls up as operation.failed (exit 1), not
  server.error (exit 7) — failures are operation outcomes, not transient
  transport issues, and the retry-with-backoff hint for server.* would
  mislead callers.
* weknora api <path> --paginate — auto-walks offset pagination and merges
  pages into a single {data, total} JSON response.
* Paginate truncation fix across 6 list/follower call sites.
* All doc / search / chunk / session / kb list commands migrated to
  FormatOptions API.

Multi-id RunE only emits the {ok, failed} envelope when the operation
actually ran — pre-flight failures (e.g. confirmation_required) leave
stdout empty per the wire contract.

doc upload's missing-positional-or-flag check is wrapped as FlagError so
the exit code (2) matches the convention used by other commands that
require a positional argument directly.
This commit is contained in:
nullkey
2026-05-18 01:38:30 +08:00
committed by lyingbug
parent 0e081aec5c
commit 34bb0b5096
33 changed files with 1259 additions and 406 deletions
+143 -13
View File
@@ -3,7 +3,7 @@
// Shape: one positional (path) + `-X/--method` flag, default GET (auto-
// promoted to POST when a body is supplied via --data or --input). The two
// body-source flags are mutually exclusive. Default raw response body to
// stdout; --json emits a {status, headers, body} object. Reuses sdk.Client.Raw which already
// stdout; --format json emits a {status, headers, body} object. Reuses sdk.Client.Raw which already
// applies tenant + auth headers.
package api
@@ -13,7 +13,9 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"github.com/spf13/cobra"
@@ -25,8 +27,8 @@ import (
)
// apiFields is intentionally a marker - api wraps arbitrary HTTP responses
// whose schema the CLI doesn't know, so the `--json=id,name` field-filter
// is a no-op here. The marker shows up in --help so users can tell.
// whose schema the CLI doesn't know, so field hints are meaningless here.
// The marker shows up in --help so users can tell.
var apiFields = []string{"<response-shape-varies>"}
type Options struct {
@@ -58,7 +60,7 @@ POST. Use -X/--method to override (DELETE / PUT / PATCH / HEAD).
Auth, tenant, and request-id headers are applied automatically from the
active context. The response body is written to stdout by default; use
--json to emit a {status, headers, body} JSON object.
--format json to emit a {status, headers, body} JSON object.
Examples:
weknora api /api/v1/knowledge-bases # GET
@@ -67,15 +69,16 @@ Examples:
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
opts.Yes, _ = c.Flags().GetBool("yes")
jopts, err := cmdutil.CheckJSONFlags(c)
fopts, err := cmdutil.CheckFormatFlag(c)
if err != nil {
return err
}
fopts.ResolveDefault(iostreams.IO.IsStdoutTTY())
method := resolveMethod(opts)
// Escape-hatch DELETE through `weknora api` is just as destructive
// as `weknora kb delete` - exit-10 protocol must apply (cli/README.md).
if method == http.MethodDelete {
if err := cmdutil.ConfirmDestructive(f.Prompter(), opts.Yes, jopts.Enabled(), "endpoint", args[0]); err != nil {
if err := cmdutil.ConfirmDestructive(f.Prompter(), opts.Yes, fopts.WantsJSON(), "endpoint", args[0]); err != nil {
return err
}
}
@@ -83,13 +86,15 @@ Examples:
if err != nil {
return err
}
return runAPI(c.Context(), opts, jopts, cli, method, args[0])
paginate, _ := c.Flags().GetBool("paginate")
return runAPI(c.Context(), opts, fopts, cli, method, args[0], paginate)
},
}
cmd.Flags().StringVarP(&opts.Method, "method", "X", "", "HTTP method (default: GET, or POST when a body is supplied)")
cmd.Flags().StringVarP(&opts.Data, "data", "d", "", "Request body as raw string (e.g. JSON)")
cmd.Flags().StringVar(&opts.Input, "input", "", "Read request body from file (use `-` for stdin)")
cmdutil.AddJSONFlags(cmd, apiFields)
cmd.Flags().Bool("paginate", false, "Follow offset-based pagination (?page=N&page_size=M), merging all pages into a single {data, total} JSON response.")
cmdutil.AddFormatFlag(cmd, apiFields...)
cmd.MarkFlagsMutuallyExclusive("data", "input")
return cmd
}
@@ -133,7 +138,19 @@ func resolveMethod(opts *Options) string {
// caller is responsible for resolving the method (defaults / auto-POST)
// and uppercasing it; runAPI guards against unsupported values like
// `-X PATCH-INVALID` reaching the wire.
func runAPI(ctx context.Context, opts *Options, jopts *cmdutil.JSONOptions, svc Service, method, path string) error {
//
// When paginate is true and method is GET, all offset-based pages are
// fetched and merged into a single {data, total} JSON response. For
// non-GET methods paginate is silently ignored (no offset semantic).
func runAPI(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, svc Service, method, path string, paginate bool) error {
if paginate && method == http.MethodGet {
return runAPIPaginated(ctx, opts, fopts, svc, path)
}
return runAPISingle(ctx, opts, fopts, svc, method, path)
}
// runAPISingle is the original single-call implementation of runAPI.
func runAPISingle(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, svc Service, method, path string) error {
switch method {
case http.MethodGet, http.MethodPost, http.MethodPut,
http.MethodPatch, http.MethodDelete, http.MethodHead:
@@ -177,7 +194,7 @@ func runAPI(ctx context.Context, opts *Options, jopts *cmdutil.JSONOptions, svc
}
out := iostreams.IO.Out
if jopts.Enabled() {
if fopts.WantsJSON() {
// Best-effort decode: if response body is valid JSON, surface the
// parsed structure under .body so JSON consumers can drill
// in; otherwise fall back to the raw string.
@@ -193,13 +210,14 @@ func runAPI(ctx context.Context, opts *Options, jopts *cmdutil.JSONOptions, svc
hdrs[k] = v[0]
}
}
// --json field-filter is ignored (response shape unknown to the
// CLI); --jq runs over the full {status, headers, body} object.
// --jq runs over the full {status, headers, body} object. Per-field
// projection isn't meaningful here since the response schema is opaque
// to the CLI.
return format.WriteJSONFiltered(out, map[string]any{
"status": resp.StatusCode,
"headers": hdrs,
"body": bodyAny,
}, nil, jopts.JQ)
}, nil, fopts.JQ)
}
if _, err := out.Write(respBody); err != nil {
@@ -211,5 +229,117 @@ func runAPI(ctx context.Context, opts *Options, jopts *cmdutil.JSONOptions, svc
return nil
}
// runAPIPaginated fetches all offset-based pages for a GET request and writes
// a single merged {data, total} JSON object to stdout. If the first page
// response does not carry pagination metadata (total + page_size), the raw
// response is passed through unchanged (single-call fallback).
func runAPIPaginated(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, svc Service, path string) error {
if !strings.HasPrefix(path, "/") {
return cmdutil.NewError(cmdutil.CodeInputInvalidArgument, fmt.Sprintf("path must start with /: %s", path))
}
pageSize := extractPageSize(path)
if pageSize == 0 {
pageSize = 50
}
var allData []json.RawMessage
var lastTotal int64
page := 1
for {
curPath := setPageParam(path, page, pageSize)
resp, err := svc.Raw(ctx, http.MethodGet, curPath, nil)
if err != nil {
return cmdutil.WrapHTTP(err, "GET %s", curPath)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
code := cmdutil.ClassifyHTTPStatus(resp.StatusCode)
return cmdutil.NewError(code, fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))))
}
var pageResp struct {
Data []json.RawMessage `json:"data"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
}
if err := json.Unmarshal(body, &pageResp); err != nil {
// Non-JSON response on first page — pass through verbatim.
if page == 1 {
return passThroughRaw(body)
}
return cmdutil.NewError(cmdutil.CodeInputInvalidArgument,
fmt.Sprintf("--paginate: page %d response not in expected shape: %v", page, err))
}
// Heuristic: if the first page lacks pagination metadata, treat the
// response as non-paginated and pass through verbatim.
if page == 1 && pageResp.Total == 0 && pageResp.PageSize == 0 {
return passThroughRaw(body)
}
allData = append(allData, pageResp.Data...)
lastTotal = pageResp.Total
// Termination: accumulated count (not page*pageSize) handles server-capped page sizes.
if int64(len(allData)) >= pageResp.Total || len(pageResp.Data) == 0 {
break
}
page++
}
merged := map[string]any{
"data": allData,
"total": lastTotal,
}
return fopts.Emit(iostreams.IO.Out, merged)
}
// passThroughRaw writes body verbatim to stdout (appending a newline if
// absent), mirroring the single-call passthrough path.
func passThroughRaw(body []byte) error {
out := iostreams.IO.Out
if _, err := out.Write(body); err != nil {
return cmdutil.Wrapf(cmdutil.CodeLocalFileIO, err, "write response body")
}
if len(body) > 0 && body[len(body)-1] != '\n' {
_, _ = out.Write([]byte{'\n'})
}
return nil
}
// extractPageSize parses the page_size query parameter from path, returning 0
// if absent or unparseable.
func extractPageSize(path string) int {
u, err := url.Parse(path)
if err != nil {
return 0
}
if v := u.Query().Get("page_size"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
}
return 0
}
// setPageParam rewrites the page and page_size query parameters in path,
// preserving all other query parameters.
func setPageParam(path string, page, pageSize int) string {
u, err := url.Parse(path)
if err != nil {
return path
}
q := u.Query()
q.Set("page", strconv.Itoa(page))
q.Set("page_size", strconv.Itoa(pageSize))
u.RawQuery = q.Encode()
return u.String()
}
// compile-time check: the production SDK client implements Service.
var _ Service = (*sdk.Client)(nil)
+156 -8
View File
@@ -1,8 +1,10 @@
package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
@@ -19,6 +21,16 @@ import (
sdk "github.com/Tencent/WeKnora/client"
)
// fakeAPISvc is a test double for Service that delegates each call to a
// caller-supplied do function, giving full control over per-call responses.
type fakeAPISvc struct {
do func(method, path string, body any) (*http.Response, error)
}
func (f *fakeAPISvc) Raw(_ context.Context, method, path string, body any) (*http.Response, error) {
return f.do(method, path, body)
}
// newTestClient stands up an httptest server with the supplied handler and
// returns an *sdk.Client targeting it plus a teardown closure. The real SDK is
// used so we exercise the same Raw() code path as production (header
@@ -40,7 +52,7 @@ func TestAPI_GetSuccess(t *testing.T) {
})
defer stop()
if err := runAPI(context.Background(), &Options{}, nil, cli, "GET", "/api/v1/foo"); err != nil {
if err := runAPI(context.Background(), &Options{}, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, cli, "GET", "/api/v1/foo", false); err != nil {
t.Fatalf("runAPI: %v", err)
}
got := out.String()
@@ -62,7 +74,7 @@ func TestAPI_GetSuccess_JSON(t *testing.T) {
})
defer stop()
if err := runAPI(context.Background(), &Options{}, &cmdutil.JSONOptions{}, cli, "GET", "/api/v1/foo"); err != nil {
if err := runAPI(context.Background(), &Options{}, &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}, cli, "GET", "/api/v1/foo", false); err != nil {
t.Fatalf("runAPI: %v", err)
}
var got struct {
@@ -98,7 +110,7 @@ func TestAPI_PostWithData(t *testing.T) {
defer stop()
opts := &Options{Data: `{"name":"foo"}`}
if err := runAPI(context.Background(), opts, nil, cli, "POST", "/api/v1/things"); err != nil {
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, cli, "POST", "/api/v1/things", false); err != nil {
t.Fatalf("runAPI: %v", err)
}
if seenMethod != http.MethodPost || seenPath != "/api/v1/things" {
@@ -127,7 +139,7 @@ func TestAPI_InputFile(t *testing.T) {
defer stop()
opts := &Options{Input: tmp}
if err := runAPI(context.Background(), opts, nil, cli, "POST", "/api/v1/x"); err != nil {
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, cli, "POST", "/api/v1/x", false); err != nil {
t.Fatalf("runAPI: %v", err)
}
if string(seenBody) != payload {
@@ -148,7 +160,7 @@ func TestAPI_InputDash_Stdin(t *testing.T) {
payload := `{"k":"from-stdin"}`
opts := &Options{Input: "-", StdinReader: strings.NewReader(payload)}
if err := runAPI(context.Background(), opts, nil, cli, "POST", "/api/v1/x"); err != nil {
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, cli, "POST", "/api/v1/x", false); err != nil {
t.Fatalf("runAPI: %v", err)
}
if string(seenBody) != payload {
@@ -164,7 +176,7 @@ func TestAPI_NotFound(t *testing.T) {
})
defer stop()
err := runAPI(context.Background(), &Options{}, nil, cli, "GET", "/api/v1/missing")
err := runAPI(context.Background(), &Options{}, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, cli, "GET", "/api/v1/missing", false)
if err == nil {
t.Fatal("expected error for 404")
}
@@ -176,7 +188,7 @@ func TestAPI_NotFound(t *testing.T) {
func TestAPI_InvalidMethod(t *testing.T) {
_, _ = iostreams.SetForTest(t)
// No server needed: validation should fail before dispatch.
err := runAPI(context.Background(), &Options{}, nil, nil, "FOO", "/api/v1/things")
err := runAPI(context.Background(), &Options{}, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, nil, "FOO", "/api/v1/things", false)
if err == nil {
t.Fatal("expected error for unsupported method")
}
@@ -188,7 +200,7 @@ func TestAPI_InvalidMethod(t *testing.T) {
func TestAPI_PathWithoutSlash(t *testing.T) {
_, _ = iostreams.SetForTest(t)
err := runAPI(context.Background(), &Options{}, nil, nil, "GET", "api/v1/things")
err := runAPI(context.Background(), &Options{}, &cmdutil.FormatOptions{Mode: cmdutil.FormatText}, nil, "GET", "api/v1/things", false)
if err == nil {
t.Fatal("expected error for missing leading slash")
}
@@ -281,3 +293,139 @@ func asTypedError(err error, dst **cmdutil.Error) bool {
}
return false
}
func TestAPI_PaginateMergesPages(t *testing.T) {
pages := [][]byte{
[]byte(`{"success":true,"data":[{"id":"1"},{"id":"2"}],"total":5,"page":1,"page_size":2}`),
[]byte(`{"success":true,"data":[{"id":"3"},{"id":"4"}],"total":5,"page":2,"page_size":2}`),
[]byte(`{"success":true,"data":[{"id":"5"}],"total":5,"page":3,"page_size":2}`),
}
idx := 0
svc := &fakeAPISvc{do: func(method, path string, _ any) (*http.Response, error) {
if idx >= len(pages) {
return nil, fmt.Errorf("too many calls; idx=%d", idx)
}
body := pages[idx]
idx++
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewReader(body)),
Header: make(http.Header),
}, nil
}}
out, _ := iostreams.SetForTest(t)
opts := &Options{}
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}, svc, "GET", "/api/v1/knowledge-base?page=1&page_size=2", true); err != nil {
t.Fatalf("runAPI: %v", err)
}
var got struct {
Data []map[string]string `json:"data"`
Total int `json:"total"`
}
if err := json.Unmarshal(out.Bytes(), &got); err != nil {
t.Fatalf("unmarshal: %v\n%s", err, out.String())
}
if len(got.Data) != 5 || got.Total != 5 {
t.Errorf("got %d records (total %d), want 5/5", len(got.Data), got.Total)
}
if idx != 3 {
t.Errorf("called %d times, want 3", idx)
}
}
func TestAPI_PaginateIgnoredForPOST(t *testing.T) {
// --paginate should be a no-op for non-GET methods (no pagination
// semantic for POST/PUT/DELETE). Single call expected.
called := 0
svc := &fakeAPISvc{do: func(method, path string, _ any) (*http.Response, error) {
called++
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewReader([]byte(`{"success":true,"data":[],"total":5,"page":1,"page_size":2}`))),
Header: make(http.Header),
}, nil
}}
_, _ = iostreams.SetForTest(t)
opts := &Options{Data: `{"name":"foo"}`}
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}, svc, "POST", "/api/v1/knowledge-base", true); err != nil {
t.Fatalf("runAPI: %v", err)
}
if called != 1 {
t.Errorf("called %d times, want 1 (POST should not paginate)", called)
}
}
func TestAPI_PaginateNoMetadataPassesThrough(t *testing.T) {
// If response doesn't look paginated (no total/page/page_size), --paginate
// should fall back to single-call behavior (don't crash).
called := 0
svc := &fakeAPISvc{do: func(method, path string, _ any) (*http.Response, error) {
called++
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewReader([]byte(`{"hello":"world"}`))),
Header: make(http.Header),
}, nil
}}
_, _ = iostreams.SetForTest(t)
opts := &Options{}
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}, svc, "GET", "/api/v1/whoami", true); err != nil {
t.Fatalf("runAPI: %v", err)
}
if called != 1 {
t.Errorf("called %d times, want 1 (non-paginated response)", called)
}
}
// TestAPI_PaginateServerCapsPageSize covers the case where the user
// requests --page_size=50 but the server caps page_size at a smaller
// value (e.g. 2). Termination must count actually-collected records
// (len(allData)) not requested-page-count (page*pageSize) — otherwise
// we'd break early and silently truncate results.
func TestAPI_PaginateServerCapsPageSize(t *testing.T) {
// User asks page_size=10; server only ever returns 2 per page (cap).
// Total = 5 records; should make 3 calls (2+2+1) and return all 5.
pages := [][]byte{
[]byte(`{"success":true,"data":[{"id":"1"},{"id":"2"}],"total":5,"page":1,"page_size":2}`),
[]byte(`{"success":true,"data":[{"id":"3"},{"id":"4"}],"total":5,"page":2,"page_size":2}`),
[]byte(`{"success":true,"data":[{"id":"5"}],"total":5,"page":3,"page_size":2}`),
}
idx := 0
svc := &fakeAPISvc{do: func(_, _ string, _ any) (*http.Response, error) {
if idx >= len(pages) {
return nil, fmt.Errorf("too many calls; idx=%d", idx)
}
body := pages[idx]
idx++
return &http.Response{
StatusCode: 200,
Body: io.NopCloser(bytes.NewReader(body)),
Header: make(http.Header),
}, nil
}}
var stdout bytes.Buffer
iostreams.IO.Out = &stdout
defer func() { iostreams.IO.Out = os.Stdout }()
opts := &Options{}
// User requests page_size=10; server caps at 2 each response.
if err := runAPI(context.Background(), opts, &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}, svc, "GET", "/api/v1/items?page=1&page_size=10", true); err != nil {
t.Fatalf("runAPI: %v", err)
}
var got struct {
Data []map[string]string `json:"data"`
Total int `json:"total"`
}
if err := json.Unmarshal(stdout.Bytes(), &got); err != nil {
t.Fatalf("unmarshal: %v\n%s", err, stdout.String())
}
if len(got.Data) != 5 {
t.Errorf("got %d records, want 5 (server-capped page_size should not cause truncation)", len(got.Data))
}
}