diff --git a/cli/internal/mcp/skill_tool_parity_test.go b/cli/internal/mcp/skill_tool_parity_test.go new file mode 100644 index 00000000..0103afaa --- /dev/null +++ b/cli/internal/mcp/skill_tool_parity_test.go @@ -0,0 +1,71 @@ +package mcp + +import ( + "context" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" +) + +// TestSkillReferencedMCPToolsExist is part of the K6 drift guard: the MCP tool +// names the bundled `weknora-shared` skill advertises (in its "CLI vs MCP" +// section) must all be registered by registerTools. Guards against a skill +// naming a renamed/removed tool (e.g. the agent_invoke→session_ask rename). +func TestSkillReferencedMCPToolsExist(t *testing.T) { + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + skillPath := filepath.Join(filepath.Dir(file), "..", "..", "skills", "weknora-shared", "SKILL.md") + raw, err := os.ReadFile(skillPath) + if err != nil { + t.Fatalf("read skill: %v", err) + } + body := string(raw) + + // Slice the "CLI vs MCP" section (heading → next "## " heading). + start := strings.Index(body, "CLI vs MCP") + if start < 0 { + t.Fatal("weknora-shared SKILL.md has no 'CLI vs MCP' section to validate") + } + section := body[start:] + if end := strings.Index(section, "\n## "); end >= 0 { + section = section[:end] + } + + // Tool names are backtick-quoted snake_case identifiers in that section. + tokenRe := regexp.MustCompile("`([a-z][a-z_]*)`") + referenced := map[string]bool{} + for _, m := range tokenRe.FindAllStringSubmatch(section, -1) { + tok := m[1] + if strings.Contains(tok, "_") || tok == "chat" { // tool-shaped (skip prose like `cli`) + referenced[tok] = true + } + } + if len(referenced) == 0 { + t.Fatal("found no MCP tool names in the skill's CLI vs MCP section") + } + + // Live registered tool set. + c, _ := newTestServer(t, &fakeSvc{}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + res, err := c.ListTools(ctx, nil) + if err != nil { + t.Fatalf("ListTools: %v", err) + } + registered := map[string]bool{} + for _, tool := range res.Tools { + registered[tool.Name] = true + } + + for name := range referenced { + if !registered[name] { + t.Errorf("weknora-shared skill references MCP tool %q which is NOT registered (renamed/removed?)", name) + } + } +} diff --git a/cli/internal/skillparity/parity_test.go b/cli/internal/skillparity/parity_test.go new file mode 100644 index 00000000..4688f363 --- /dev/null +++ b/cli/internal/skillparity/parity_test.go @@ -0,0 +1,171 @@ +// Package skillparity contains the K6 drift guard: every weknora command and +// long flag referenced in a bundled Agent Skill (cli/skills/**) must still +// exist in the live cobra command tree. A skill that references a renamed or +// removed flag/command is worse than no skill, so this fails CI on drift. +package skillparity + +import ( + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/Tencent/WeKnora/cli/cmd" + "github.com/Tencent/WeKnora/cli/internal/cmdutil" +) + +var ( + codeFence = regexp.MustCompile("(?s)```[a-zA-Z]*\\n(.*?)```") + longFlag = regexp.MustCompile(`--[a-zA-Z][a-zA-Z0-9-]*`) +) + +func skillsRoot(t *testing.T) string { + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Join(filepath.Dir(file), "..", "..", "skills") +} + +// allFlagNames collects every long flag name reachable in the command tree +// (each command's local + persistent + inherited flags, plus root persistent +// and the cobra-added help/version flags). +func allFlagNames(root *cobra.Command) map[string]bool { + set := map[string]bool{"help": true, "version": true} + add := func(fs *pflag.FlagSet) { + fs.VisitAll(func(f *pflag.Flag) { set[f.Name] = true }) + } + var walk func(c *cobra.Command) + walk = func(c *cobra.Command) { + add(c.LocalFlags()) + add(c.PersistentFlags()) + add(c.InheritedFlags()) + for _, sub := range c.Commands() { + walk(sub) + } + } + walk(root) + return set +} + +// globalTakesValue reports, for each root persistent flag (long + short), whether +// it consumes the following token as a value (non-bool). +func globalTakesValue(root *cobra.Command) map[string]bool { + m := map[string]bool{} + root.PersistentFlags().VisitAll(func(f *pflag.Flag) { + takes := f.Value.Type() != "bool" + m["--"+f.Name] = takes + if f.Shorthand != "" { + m["-"+f.Shorthand] = takes + } + }) + return m +} + +func TestSkillsReferenceLiveCommandsAndFlags(t *testing.T) { + root := cmd.NewRootCmd(&cmdutil.Factory{}) + flags := allFlagNames(root) + globals := globalTakesValue(root) + + files := skillMarkdownFiles(t, skillsRoot(t)) + if len(files) == 0 { + t.Fatal("no skill markdown files found under cli/skills/") + } + + checkedAny := false + for _, path := range files { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + rel, _ := filepath.Rel(skillsRoot(t), path) + for _, block := range codeFence.FindAllStringSubmatch(string(raw), -1) { + for _, line := range strings.Split(block[1], "\n") { + idx := strings.Index(line, "weknora ") + if idx < 0 { + continue + } + inv := line[idx+len("weknora"):] + toks := strings.Fields(inv) + if len(toks) == 0 { + continue + } + // Skip leading global flags (and their value tokens). + i := 0 + for i < len(toks) && strings.HasPrefix(toks[i], "-") { + tok := toks[i] + if eq := strings.Index(tok, "="); eq >= 0 { + tok = tok[:eq] + } + i++ + if globals[tok] && i < len(toks) { + i++ // consume the value + } + } + // Placeholder command (e.g. `weknora --help`) → skip. + if i < len(toks) && strings.HasPrefix(toks[i], "<") { + continue + } + // Greedily descend subcommands while the next token is one. + curr := root + for i < len(toks) { + sub := findSub(curr, toks[i]) + if sub == nil { + break + } + curr = sub + i++ + } + if curr != root { + checkedAny = true + } + // Every long flag in the invocation must exist somewhere in the tree. + for _, m := range longFlag.FindAllString(inv, -1) { + name := strings.TrimPrefix(m, "--") + if !flags[name] { + t.Errorf("%s: skill references unknown flag --%s (renamed/removed? line: %q)", rel, name, strings.TrimSpace(line)) + } + } + } + } + } + if !checkedAny { + t.Fatal("parser resolved no weknora commands from skills — check tokenizer") + } +} + +func findSub(c *cobra.Command, name string) *cobra.Command { + for _, sub := range c.Commands() { + if sub.Name() == name { + return sub + } + for _, a := range sub.Aliases { + if a == name { + return sub + } + } + } + return nil +} + +func skillMarkdownFiles(t *testing.T, root string) []string { + var out []string + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(p, ".md") { + out = append(out, p) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + return out +} diff --git a/cli/skills/weknora-rag-search/SKILL.md b/cli/skills/weknora-rag-search/SKILL.md new file mode 100644 index 00000000..d094ac31 --- /dev/null +++ b/cli/skills/weknora-rag-search/SKILL.md @@ -0,0 +1,72 @@ +--- +name: weknora-rag-search +description: Use when retrieving from or asking questions against a WeKnora knowledge base via the `weknora` CLI — and especially when unsure whether to use `chat`, `session ask`, or `search chunks` for a given goal. +metadata: + tested_against: v0.9 +--- + +# WeKnora — retrieval & RAG queries + +**REQUIRED BACKGROUND:** read the `weknora-shared` skill first (auth, `--kb` +resolution, the JSON envelope, exit codes, streaming/NDJSON output). + +WeKnora gives you several ways to "ask about a knowledge base." Picking the wrong +one wastes turns or returns the wrong shape. Use the decision table. + +## Pick the command by your goal + +| Your goal | Command | LLM synthesis? | Returns | +|---|---|---|---| +| Natural-language **answer** grounded in a KB | `chat "" --kb ` | yes | streaming answer + references | +| Answer via a **custom agent** (its own KB scope, tools, web search) | `session ask --agent ""` | yes (+ tools) | streaming answer + tool events | +| **Raw context chunks** to reason over yourself (no answer) | `search chunks "" --kb ` | no | ranked chunk list | +| Which **documents** match a keyword (title/filename) | `search docs "" --kb ` | no | document list | +| Find a **knowledge base** by name | `search kb ""` | no | KB list | +| Find a past **session** by title | `search sessions ""` | no | session list | + +### The three decisions that matter + +1. **Answer vs raw context.** Want a written answer → `chat` / `session ask`. + Want chunks to feed into your *own* reasoning (e.g. you'll synthesize across + sources) → `search chunks`. Don't call `chat` just to read source text. +2. **`chat` vs `session ask`.** `chat` = plain KB RAG Q&A. `session ask --agent + ` = invoke a *configured custom agent* (it may scope its own KBs, call + tools, do web search). If the user set up an agent for this, prefer it + (`weknora agent list` to find ids); otherwise `chat`. +3. **One-shot vs multi-turn.** Both `chat` and `session ask` print an `init` + event with a `session_id`. Pass `--session ` on the next call to continue + the conversation. See `references/chat.md`. + +## Safety / Gotchas + +- `--kb` is **required** for `chat`, `search chunks`, `search docs` (no implicit + project-link fallback) — an unset `--kb` is exit 2, a bad name is exit 1 + (`local.kb_not_found`). Resolve names with `weknora kb list` / `search kb`. +- `chat` / `session ask` stream **NDJSON** by default. Parse line-by-line; keep + the `init` event's `session_id` **and** `message_id`. Use `--format text` only + for a human transcript. +- A stalled stream is not stopped by Ctrl-C (that just drops your local + connection; the server keeps generating + billing). Stop it server-side: + `weknora session stop --message ` (ids from `init`). + Re-attach to a stream with `weknora session continue-stream + --message `. +- `search chunks --limit` defaults to **8** (tuned for an LLM context window); + the `search docs/kb/sessions` lists default to 30. Tune retrieval with + `--vector-threshold` / `--keyword-threshold`, or `--no-vector`/`--no-keyword` + to disable a channel. Details: `references/search-chunks.md`. + +## Quick examples + +```bash +# raw retrieval to reason over +weknora search chunks "retry backoff policy" --kb engineering --limit 12 + +# grounded answer (human transcript) +weknora chat "How do we handle retries?" --kb engineering --format text + +# continue the conversation (session id from the prior init event) +weknora chat "And the max attempts?" --kb engineering --session sess_abc + +# answer via a custom agent +weknora session ask --agent ag_123 "Summarize this quarter's incidents" +``` diff --git a/cli/skills/weknora-rag-search/references/chat.md b/cli/skills/weknora-rag-search/references/chat.md new file mode 100644 index 00000000..af578555 --- /dev/null +++ b/cli/skills/weknora-rag-search/references/chat.md @@ -0,0 +1,49 @@ +# `chat` and `session ask` — streaming RAG + +Both stream an NDJSON event log (one JSON object per line) by default. They share +the same event vocabulary; `chat` does plain KB RAG, `session ask` invokes a +custom agent. + +## Commands & flags + +``` +weknora chat "" --kb [--session ] +weknora session ask "" --agent [--session ] +``` + +- `--kb` (chat) is required name-or-id. `--agent` (session ask) is required. +- `--session ` continues an existing conversation; omit to start a new one. +- `--format text` renders a human transcript instead of NDJSON; `--format json` + / `--format ndjson` both produce the NDJSON stream (identical here). + +## Event stream + +The CLI emits an `init` line first, then passes SDK events through verbatim: + +```jsonc +{"event":"init","session_id":"sess_abc","message_id":"msg_xyz","profile":"prod"} +{"response_type":"thinking","content":"…"} +{"response_type":"tool_call","tool_calls":[…]} // agent only +{"response_type":"tool_result","content":"…"} // agent only +{"response_type":"references","knowledge_references":[…]} +{"response_type":"answer","content":"partial text…"} // streamed in pieces +{"response_type":"complete","done":true} +``` + +- Accumulate `response_type:"answer"` `content` pieces for the final answer. +- `knowledge_references` carry the grounding chunks (source attribution). +- **Keep `init.session_id` and `init.message_id`**: `session_id` continues the + chat (`--session`); `message_id` is needed to `session stop` or + `session continue-stream` that specific message. +- On failure mid-stream you get `response_type:"error"`; a transport/HTTP error + surfaces as the normal error envelope on stderr with a typed code. + +## Recovery + +- **Stop server-side generation:** `weknora session stop --message + `. Ctrl-C only closes your local connection — the server keeps + generating (and billing) until told to stop. +- **Re-attach after a dropped connection:** `weknora session continue-stream + --message `. The server replays the event log from + index 0 then tails new events, so **dedupe by message_id** if you already + consumed some events. Buffer TTL is ~1h (redis) or process-lifetime (memory). diff --git a/cli/skills/weknora-rag-search/references/search-chunks.md b/cli/skills/weknora-rag-search/references/search-chunks.md new file mode 100644 index 00000000..6e8c15ce --- /dev/null +++ b/cli/skills/weknora-rag-search/references/search-chunks.md @@ -0,0 +1,46 @@ +# `search chunks` — hybrid retrieval (no LLM) + +Raw vector + keyword retrieval against one knowledge base. Returns ranked chunks +for you to reason over; it does NOT synthesize an answer (use `chat` for that). + +## Command & flags + +``` +weknora search chunks "" --kb [flags] +``` + +| Flag | Default | Meaning | +|---|---|---| +| `--kb` | (required) | KB name or UUID | +| `--limit`, `-L` | 8 | max chunks returned (1..1000); 8 is tuned for an LLM context window | +| `--vector-threshold` | 0 (off) | min vector similarity, per-channel pre-fusion | +| `--keyword-threshold` | 0 (off) | min keyword score, per-channel pre-fusion | +| `--no-vector` | false | disable the vector channel (keyword-only) | +| `--no-keyword` | false | disable the keyword channel (vector-only) | + +You cannot disable both channels. `--limit` is a hard cap on returned chunks +applied client-side (the server may internally retrieve a larger pool for recall, +then the CLI trims). + +## Output (`--format json`) + +`data` is an array of chunk objects; `meta.count` is the number returned. + +```jsonc +{"ok":true,"meta":{"count":3},"data":[ + {"id":"chunk_…","content":"…","knowledge_id":"doc_…","knowledge_title":"…", + "chunk_index":4,"score":0.82,"match_type":"hybrid","chunk_type":"text"} +]} +``` + +- `score` is the fused rank; `match_type` indicates which channel(s) hit. +- `knowledge_id` / `knowledge_title` attribute the chunk to its source document. +- Project just what you need with `--jq`, e.g. + `weknora search chunks "q" --kb eng --jq '.data[] | {score,content}'`. + +## When to use vs alternatives + +- Need an **answer** → `chat` (it does retrieval + synthesis internally). +- Building your **own** prompt/context from sources → `search chunks` (this). +- Finding **which documents** exist by keyword → `search docs --kb `. +- Inspecting/debugging a specific document's chunks → `chunk list --doc `. diff --git a/cli/skills/weknora-shared/SKILL.md b/cli/skills/weknora-shared/SKILL.md new file mode 100644 index 00000000..c09edfd9 --- /dev/null +++ b/cli/skills/weknora-shared/SKILL.md @@ -0,0 +1,169 @@ +--- +name: weknora-shared +description: Use when driving a WeKnora RAG server through the `weknora` CLI as an agent — authenticating, managing knowledge bases / documents / sessions / agents, running search or chat, or interpreting the CLI's JSON envelopes and exit codes. Read this before any other weknora-* skill. +metadata: + tested_against: v0.9 +--- + +# WeKnora CLI — shared base + +`weknora` is the agent-first CLI for a WeKnora RAG server. Every command prints +a JSON envelope and uses a typed exit code, so you branch on machine-readable +output, not prose. Read this skill before any task-specific `weknora-*` skill. + +## 1. Authenticate (do this first — order matters) + +Authentication is a **two-step sequence**. `weknora auth login` operates on the +*active profile*, so the profile must exist first: + +```bash +# 1. register a connection target and make it active +weknora profile add prod --host https://kb.example.com --use +# 2a. API key (agent default — pipe the key on stdin, non-interactive): +echo "$WEKNORA_API_KEY" | weknora auth login --with-token +# 2b. OR email+password (interactive prompt only — no flags; not for agents) +weknora auth login + +weknora auth status # verify: who am I, which tenant +``` + +- Target a *non-active* profile for one command with the global `--profile NAME` + (e.g. `weknora --profile staging auth refresh`). There is no per-command + `--name`/`--host` on auth commands. +- Get the raw token for scripting with `WEKNORA_TOKEN=$(weknora auth token)` + (raw token by default; `--format json` gives the `{token, mode, profile}` envelope). +- `weknora doctor` runs 4 health checks (reachability, credential, version, storage). + +## 2. Selecting a knowledge base (`--kb`) + +`--kb` accepts a **name or a UUID** (resolved server-side). Resolution order: +`--kb` flag → `WEKNORA_KB_ID` env → directory link (`weknora link --kb X` binds +the cwd) → error. Read/create commands that operate "inside a project" inherit +the link; **`search *` and destructive `--all` operations always require an +explicit `--kb`** (so an agent never silently hits the wrong corpus). + +## 3. Output contract — every command + +Default output is `--format json`: a single envelope. + +| Field | Meaning | +|---|---| +| `ok` | `true`/`false` — branch on this first (see batch/wait caveat below) | +| `data` | success payload (object or array; absent on mutation-only success) | +| `meta` | `count` / `has_more` / `dry_run` / `plan` … | +| `error` | on failure: `{type, message, hint?, retry_command?, risk?}` | +| `profile` | active profile name | + +- `error.type` is a **stable typed code** (e.g. `local.kb_not_found`, + `input.invalid_argument`, `input.confirmation_required`, `server.error`). + Branch on it; `error.hint` usually tells you the next action. +- `--format text` = human output. Streaming commands (`chat`, `session ask`, + `session continue-stream`) emit **NDJSON event lines** by default — expect a + stream, not one blob; use `--format text` for a rendered transcript. +- `--jq ''` filters the envelope (e.g. `weknora kb list --jq '.data[].id'`). +- Exception: `weknora auth token` emits the **raw token** by default (it's a + scripting helper); pass `--format json` for the `{token, mode, profile}` envelope. +- **Batch / wait caveat:** for multi-item commands (`doc/chunk/session delete` + with several ids, `doc wait`), `ok:true` means *the command ran to completion*, + **not** that every item succeeded — per-item failures live in `data` and the + **exit code** carries the aggregate verdict (e.g. `doc wait` exits 1 if any doc + failed, 124 on timeout, while still printing the `{completed,failed,timeout}` + partition with `ok:true`). For these, branch on the **exit code**, then read + `data` for which items failed. + +## 4. Exit codes (branch on these) + +| Code | Meaning (typed code class) | Agent action | +|---|---|---| +| 0 | success (incl. `--dry-run`) | proceed | +| 1 | `local.*` / unclassified (incl. `local.kb_not_found`) | read `error`, decide retry/abort | +| 2 | flag / argument validation (bad/unknown/missing-required flag) | re-check `weknora --help` | +| 3 | `auth.*` (missing / expired / forbidden) | re-auth, then retry | +| 4 | `resource.not_found` (a server resource id) | verify the id | +| 5 | `input.*` (other than confirmation_required) | adjust args, retry | +| 6 | `server.rate_limited` | back off, retry | +| 7 | `server.*` / `network.*` | transient — retry with backoff | +| **10** | **`input.confirmation_required` (destructive)** | **see §5 — never auto-bypass** | +| 124 | `operation.timeout` | raise `--timeout` or check the job | +| 130 | `operation.cancelled` (SIGINT/SIGTERM) | stop, don't retry | + +Note: a server resource id that doesn't exist → exit 4 (`resource.not_found`); +a `--kb` *name* that doesn't resolve client-side → exit 1 (`local.kb_not_found`). +`input.invalid_argument` spans two exit codes — a malformed *invocation* +(unknown/missing flag, wrong arg count) exits **2** (fix the command, re-check +`--help`), while a value rejected *after* parsing (e.g. a bad `--jq` expression) +exits **5** (adjust the value, retry). Branch on the exit code to tell them apart. + +## 5. Destructive writes (exit 10) — hard rule + +Destructive commands (`kb/doc/chunk/session/agent delete`, `kb/agent edit`, +`auth logout`, `doc delete --all`, …) without `-y` exit **10** with +`error.type = input.confirmation_required` and `error.risk = {level, action}`: + +```jsonc +{"ok":false,"error":{"type":"input.confirmation_required", + "message":"delete knowledge base X requires explicit confirmation: re-run with -y", + "retry_command":"weknora kb delete X -y","risk":{"level":"destructive","action":"kb.delete"}}} +``` + +**Surface this to the user and get explicit approval. Re-run with `-y` ONLY +after they approve. Never add `-y` on your own initiative.** + +## 6. Preview before acting — `--dry-run` + +Any mutation accepts `--dry-run`: it resolves the request, prints +`meta.plan = {action, args}`, makes **zero** server/side-effect changes, exits 0. +Use it to confirm a command is well-formed (and what it would do) before running +it for real — especially before destructive or bulk operations. + +## 7. Chaining (mutation → id) + +Mutations return the new resource id in `data.id`. Chain with it: + +```bash +KB=$(weknora kb create "Docs" --jq '.data.id' --format json | tr -d '"') +weknora doc upload ./manual.pdf --kb "$KB" +``` + +## 8. Resource model & command map + +``` +kb knowledge bases list/view/create/edit/delete/pin/unpin/status/check +doc documents in a KB list/view/create/upload/fetch/download/delete/wait +chunk retrieval units list/view/delete (RAG debug; not search) +session conversations list/view/delete/ask/stop/continue-stream +agent custom agents list/view/create/edit/delete/status/check +search retrieval chunks / docs / kb / sessions +chat one-shot KB RAG Q&A (streaming) +api raw HTTP passthrough to any server endpoint (escape hatch) +link/unlink bind cwd to a KB mcp serve expose weknora as MCP tools +``` + +**chat vs session ask vs search** — the most common confusion: see the +`weknora-rag-search` skill for the decision table. Briefly: `chat` = one-shot +KB Q&A with an LLM; `session ask --agent ` = invoke a *custom agent*; +`search chunks` = raw hybrid retrieval (no LLM). + +## 9. CLI vs MCP + +For your own scripted control, use the CLI (richer: dry-run, exit-10, all verbs). +For an IDE/host agent that speaks MCP, `weknora mcp serve` exposes a curated +read+chat tool set: `kb_list`, `kb_view`, `doc_list`, `doc_view`, `doc_download`, +`search_chunks`, `chunk_list`, `agent_list`, `chat`, `session_ask`. MCP tools +take raw ids (no name resolution); resolve names via `kb_list` first. + +## 10. Agent self-help + +Run `weknora --help`. With `WEKNORA_AGENT_HELP=1` set, `--help` emits a +JSON blob (`used_for` / `required_flags` / `examples`) instead of prose — parse +that to learn a command without scraping the human help table. + +## Common mistakes + +| Mistake | Fix | +|---|---| +| `weknora auth login` before any profile exists | `profile add --host --use` first (§1) | +| Expecting `auth login --host/--name` flags | Gone — host comes from the profile; use global `--profile` | +| Auto-adding `-y` to clear an exit-10 | Never; get user approval first (§5) | +| `weknora chat "..."` prints "garbled" stream | NDJSON by default; use `--format text` for a transcript | +| `search chunks "q"` → exit 2 | `--kb` is required on search; pass `--kb ` |