feat(cli): bundled Agent Skills + drift guards

Ship weknora-shared and weknora-rag-search Agent Skills (SKILL.md plus
references) so an agent driving the CLI learns the auth flow, output contract,
exit codes, and the chat vs session-ask vs search distinction without scraping
help text.

Add CI drift guards that keep the skills honest: every ```weknora``` block must
reference a live command/flag (skillparity), and every MCP tool named in a
skill must exist in the registry (skill_tool_parity).
This commit is contained in:
nullkey
2026-05-29 20:06:41 +08:00
committed by lyingbug
parent c90f4e895e
commit 5f98df530c
6 changed files with 578 additions and 0 deletions
@@ -0,0 +1,71 @@
package mcp
import (
"context"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"testing"
"time"
)
// TestSkillReferencedMCPToolsExist is part of the K6 drift guard: the MCP tool
// names the bundled `weknora-shared` skill advertises (in its "CLI vs MCP"
// section) must all be registered by registerTools. Guards against a skill
// naming a renamed/removed tool (e.g. the agent_invoke→session_ask rename).
func TestSkillReferencedMCPToolsExist(t *testing.T) {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
skillPath := filepath.Join(filepath.Dir(file), "..", "..", "skills", "weknora-shared", "SKILL.md")
raw, err := os.ReadFile(skillPath)
if err != nil {
t.Fatalf("read skill: %v", err)
}
body := string(raw)
// Slice the "CLI vs MCP" section (heading → next "## " heading).
start := strings.Index(body, "CLI vs MCP")
if start < 0 {
t.Fatal("weknora-shared SKILL.md has no 'CLI vs MCP' section to validate")
}
section := body[start:]
if end := strings.Index(section, "\n## "); end >= 0 {
section = section[:end]
}
// Tool names are backtick-quoted snake_case identifiers in that section.
tokenRe := regexp.MustCompile("`([a-z][a-z_]*)`")
referenced := map[string]bool{}
for _, m := range tokenRe.FindAllStringSubmatch(section, -1) {
tok := m[1]
if strings.Contains(tok, "_") || tok == "chat" { // tool-shaped (skip prose like `cli`)
referenced[tok] = true
}
}
if len(referenced) == 0 {
t.Fatal("found no MCP tool names in the skill's CLI vs MCP section")
}
// Live registered tool set.
c, _ := newTestServer(t, &fakeSvc{})
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
res, err := c.ListTools(ctx, nil)
if err != nil {
t.Fatalf("ListTools: %v", err)
}
registered := map[string]bool{}
for _, tool := range res.Tools {
registered[tool.Name] = true
}
for name := range referenced {
if !registered[name] {
t.Errorf("weknora-shared skill references MCP tool %q which is NOT registered (renamed/removed?)", name)
}
}
}
+171
View File
@@ -0,0 +1,171 @@
// Package skillparity contains the K6 drift guard: every weknora command and
// long flag referenced in a bundled Agent Skill (cli/skills/**) must still
// exist in the live cobra command tree. A skill that references a renamed or
// removed flag/command is worse than no skill, so this fails CI on drift.
package skillparity
import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"testing"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/Tencent/WeKnora/cli/cmd"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
)
var (
codeFence = regexp.MustCompile("(?s)```[a-zA-Z]*\\n(.*?)```")
longFlag = regexp.MustCompile(`--[a-zA-Z][a-zA-Z0-9-]*`)
)
func skillsRoot(t *testing.T) string {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("runtime.Caller failed")
}
return filepath.Join(filepath.Dir(file), "..", "..", "skills")
}
// allFlagNames collects every long flag name reachable in the command tree
// (each command's local + persistent + inherited flags, plus root persistent
// and the cobra-added help/version flags).
func allFlagNames(root *cobra.Command) map[string]bool {
set := map[string]bool{"help": true, "version": true}
add := func(fs *pflag.FlagSet) {
fs.VisitAll(func(f *pflag.Flag) { set[f.Name] = true })
}
var walk func(c *cobra.Command)
walk = func(c *cobra.Command) {
add(c.LocalFlags())
add(c.PersistentFlags())
add(c.InheritedFlags())
for _, sub := range c.Commands() {
walk(sub)
}
}
walk(root)
return set
}
// globalTakesValue reports, for each root persistent flag (long + short), whether
// it consumes the following token as a value (non-bool).
func globalTakesValue(root *cobra.Command) map[string]bool {
m := map[string]bool{}
root.PersistentFlags().VisitAll(func(f *pflag.Flag) {
takes := f.Value.Type() != "bool"
m["--"+f.Name] = takes
if f.Shorthand != "" {
m["-"+f.Shorthand] = takes
}
})
return m
}
func TestSkillsReferenceLiveCommandsAndFlags(t *testing.T) {
root := cmd.NewRootCmd(&cmdutil.Factory{})
flags := allFlagNames(root)
globals := globalTakesValue(root)
files := skillMarkdownFiles(t, skillsRoot(t))
if len(files) == 0 {
t.Fatal("no skill markdown files found under cli/skills/")
}
checkedAny := false
for _, path := range files {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
rel, _ := filepath.Rel(skillsRoot(t), path)
for _, block := range codeFence.FindAllStringSubmatch(string(raw), -1) {
for _, line := range strings.Split(block[1], "\n") {
idx := strings.Index(line, "weknora ")
if idx < 0 {
continue
}
inv := line[idx+len("weknora"):]
toks := strings.Fields(inv)
if len(toks) == 0 {
continue
}
// Skip leading global flags (and their value tokens).
i := 0
for i < len(toks) && strings.HasPrefix(toks[i], "-") {
tok := toks[i]
if eq := strings.Index(tok, "="); eq >= 0 {
tok = tok[:eq]
}
i++
if globals[tok] && i < len(toks) {
i++ // consume the value
}
}
// Placeholder command (e.g. `weknora <command> --help`) → skip.
if i < len(toks) && strings.HasPrefix(toks[i], "<") {
continue
}
// Greedily descend subcommands while the next token is one.
curr := root
for i < len(toks) {
sub := findSub(curr, toks[i])
if sub == nil {
break
}
curr = sub
i++
}
if curr != root {
checkedAny = true
}
// Every long flag in the invocation must exist somewhere in the tree.
for _, m := range longFlag.FindAllString(inv, -1) {
name := strings.TrimPrefix(m, "--")
if !flags[name] {
t.Errorf("%s: skill references unknown flag --%s (renamed/removed? line: %q)", rel, name, strings.TrimSpace(line))
}
}
}
}
}
if !checkedAny {
t.Fatal("parser resolved no weknora commands from skills — check tokenizer")
}
}
func findSub(c *cobra.Command, name string) *cobra.Command {
for _, sub := range c.Commands() {
if sub.Name() == name {
return sub
}
for _, a := range sub.Aliases {
if a == name {
return sub
}
}
}
return nil
}
func skillMarkdownFiles(t *testing.T, root string) []string {
var out []string
err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() && strings.HasSuffix(p, ".md") {
out = append(out, p)
}
return nil
})
if err != nil {
t.Fatalf("walk %s: %v", root, err)
}
return out
}
+72
View File
@@ -0,0 +1,72 @@
---
name: weknora-rag-search
description: Use when retrieving from or asking questions against a WeKnora knowledge base via the `weknora` CLI — and especially when unsure whether to use `chat`, `session ask`, or `search chunks` for a given goal.
metadata:
tested_against: v0.9
---
# WeKnora — retrieval & RAG queries
**REQUIRED BACKGROUND:** read the `weknora-shared` skill first (auth, `--kb`
resolution, the JSON envelope, exit codes, streaming/NDJSON output).
WeKnora gives you several ways to "ask about a knowledge base." Picking the wrong
one wastes turns or returns the wrong shape. Use the decision table.
## Pick the command by your goal
| Your goal | Command | LLM synthesis? | Returns |
|---|---|---|---|
| Natural-language **answer** grounded in a KB | `chat "<q>" --kb <kb>` | yes | streaming answer + references |
| Answer via a **custom agent** (its own KB scope, tools, web search) | `session ask --agent <id> "<q>"` | yes (+ tools) | streaming answer + tool events |
| **Raw context chunks** to reason over yourself (no answer) | `search chunks "<q>" --kb <kb>` | no | ranked chunk list |
| Which **documents** match a keyword (title/filename) | `search docs "<q>" --kb <kb>` | no | document list |
| Find a **knowledge base** by name | `search kb "<q>"` | no | KB list |
| Find a past **session** by title | `search sessions "<q>"` | no | session list |
### The three decisions that matter
1. **Answer vs raw context.** Want a written answer → `chat` / `session ask`.
Want chunks to feed into your *own* reasoning (e.g. you'll synthesize across
sources) → `search chunks`. Don't call `chat` just to read source text.
2. **`chat` vs `session ask`.** `chat` = plain KB RAG Q&A. `session ask --agent
<id>` = invoke a *configured custom agent* (it may scope its own KBs, call
tools, do web search). If the user set up an agent for this, prefer it
(`weknora agent list` to find ids); otherwise `chat`.
3. **One-shot vs multi-turn.** Both `chat` and `session ask` print an `init`
event with a `session_id`. Pass `--session <id>` on the next call to continue
the conversation. See `references/chat.md`.
## Safety / Gotchas
- `--kb` is **required** for `chat`, `search chunks`, `search docs` (no implicit
project-link fallback) — an unset `--kb` is exit 2, a bad name is exit 1
(`local.kb_not_found`). Resolve names with `weknora kb list` / `search kb`.
- `chat` / `session ask` stream **NDJSON** by default. Parse line-by-line; keep
the `init` event's `session_id` **and** `message_id`. Use `--format text` only
for a human transcript.
- A stalled stream is not stopped by Ctrl-C (that just drops your local
connection; the server keeps generating + billing). Stop it server-side:
`weknora session stop <session-id> --message <message-id>` (ids from `init`).
Re-attach to a stream with `weknora session continue-stream <session-id>
--message <message-id>`.
- `search chunks --limit` defaults to **8** (tuned for an LLM context window);
the `search docs/kb/sessions` lists default to 30. Tune retrieval with
`--vector-threshold` / `--keyword-threshold`, or `--no-vector`/`--no-keyword`
to disable a channel. Details: `references/search-chunks.md`.
## Quick examples
```bash
# raw retrieval to reason over
weknora search chunks "retry backoff policy" --kb engineering --limit 12
# grounded answer (human transcript)
weknora chat "How do we handle retries?" --kb engineering --format text
# continue the conversation (session id from the prior init event)
weknora chat "And the max attempts?" --kb engineering --session sess_abc
# answer via a custom agent
weknora session ask --agent ag_123 "Summarize this quarter's incidents"
```
@@ -0,0 +1,49 @@
# `chat` and `session ask` — streaming RAG
Both stream an NDJSON event log (one JSON object per line) by default. They share
the same event vocabulary; `chat` does plain KB RAG, `session ask` invokes a
custom agent.
## Commands & flags
```
weknora chat "<query>" --kb <name-or-id> [--session <id>]
weknora session ask "<query>" --agent <agent-id> [--session <id>]
```
- `--kb` (chat) is required name-or-id. `--agent` (session ask) is required.
- `--session <id>` continues an existing conversation; omit to start a new one.
- `--format text` renders a human transcript instead of NDJSON; `--format json`
/ `--format ndjson` both produce the NDJSON stream (identical here).
## Event stream
The CLI emits an `init` line first, then passes SDK events through verbatim:
```jsonc
{"event":"init","session_id":"sess_abc","message_id":"msg_xyz","profile":"prod"}
{"response_type":"thinking","content":"…"}
{"response_type":"tool_call","tool_calls":[…]} // agent only
{"response_type":"tool_result","content":"…"} // agent only
{"response_type":"references","knowledge_references":[…]}
{"response_type":"answer","content":"partial text…"} // streamed in pieces
{"response_type":"complete","done":true}
```
- Accumulate `response_type:"answer"` `content` pieces for the final answer.
- `knowledge_references` carry the grounding chunks (source attribution).
- **Keep `init.session_id` and `init.message_id`**: `session_id` continues the
chat (`--session`); `message_id` is needed to `session stop` or
`session continue-stream` that specific message.
- On failure mid-stream you get `response_type:"error"`; a transport/HTTP error
surfaces as the normal error envelope on stderr with a typed code.
## Recovery
- **Stop server-side generation:** `weknora session stop <session-id> --message
<message-id>`. Ctrl-C only closes your local connection — the server keeps
generating (and billing) until told to stop.
- **Re-attach after a dropped connection:** `weknora session continue-stream
<session-id> --message <message-id>`. The server replays the event log from
index 0 then tails new events, so **dedupe by message_id** if you already
consumed some events. Buffer TTL is ~1h (redis) or process-lifetime (memory).
@@ -0,0 +1,46 @@
# `search chunks` — hybrid retrieval (no LLM)
Raw vector + keyword retrieval against one knowledge base. Returns ranked chunks
for you to reason over; it does NOT synthesize an answer (use `chat` for that).
## Command & flags
```
weknora search chunks "<query>" --kb <name-or-id> [flags]
```
| Flag | Default | Meaning |
|---|---|---|
| `--kb` | (required) | KB name or UUID |
| `--limit`, `-L` | 8 | max chunks returned (1..1000); 8 is tuned for an LLM context window |
| `--vector-threshold` | 0 (off) | min vector similarity, per-channel pre-fusion |
| `--keyword-threshold` | 0 (off) | min keyword score, per-channel pre-fusion |
| `--no-vector` | false | disable the vector channel (keyword-only) |
| `--no-keyword` | false | disable the keyword channel (vector-only) |
You cannot disable both channels. `--limit` is a hard cap on returned chunks
applied client-side (the server may internally retrieve a larger pool for recall,
then the CLI trims).
## Output (`--format json`)
`data` is an array of chunk objects; `meta.count` is the number returned.
```jsonc
{"ok":true,"meta":{"count":3},"data":[
{"id":"chunk_…","content":"…","knowledge_id":"doc_…","knowledge_title":"…",
"chunk_index":4,"score":0.82,"match_type":"hybrid","chunk_type":"text"}
]}
```
- `score` is the fused rank; `match_type` indicates which channel(s) hit.
- `knowledge_id` / `knowledge_title` attribute the chunk to its source document.
- Project just what you need with `--jq`, e.g.
`weknora search chunks "q" --kb eng --jq '.data[] | {score,content}'`.
## When to use vs alternatives
- Need an **answer** → `chat` (it does retrieval + synthesis internally).
- Building your **own** prompt/context from sources → `search chunks` (this).
- Finding **which documents** exist by keyword → `search docs --kb <kb>`.
- Inspecting/debugging a specific document's chunks → `chunk list --doc <id>`.
+169
View File
@@ -0,0 +1,169 @@
---
name: weknora-shared
description: Use when driving a WeKnora RAG server through the `weknora` CLI as an agent — authenticating, managing knowledge bases / documents / sessions / agents, running search or chat, or interpreting the CLI's JSON envelopes and exit codes. Read this before any other weknora-* skill.
metadata:
tested_against: v0.9
---
# WeKnora CLI — shared base
`weknora` is the agent-first CLI for a WeKnora RAG server. Every command prints
a JSON envelope and uses a typed exit code, so you branch on machine-readable
output, not prose. Read this skill before any task-specific `weknora-*` skill.
## 1. Authenticate (do this first — order matters)
Authentication is a **two-step sequence**. `weknora auth login` operates on the
*active profile*, so the profile must exist first:
```bash
# 1. register a connection target and make it active
weknora profile add prod --host https://kb.example.com --use
# 2a. API key (agent default — pipe the key on stdin, non-interactive):
echo "$WEKNORA_API_KEY" | weknora auth login --with-token
# 2b. OR email+password (interactive prompt only — no flags; not for agents)
weknora auth login
weknora auth status # verify: who am I, which tenant
```
- Target a *non-active* profile for one command with the global `--profile NAME`
(e.g. `weknora --profile staging auth refresh`). There is no per-command
`--name`/`--host` on auth commands.
- Get the raw token for scripting with `WEKNORA_TOKEN=$(weknora auth token)`
(raw token by default; `--format json` gives the `{token, mode, profile}` envelope).
- `weknora doctor` runs 4 health checks (reachability, credential, version, storage).
## 2. Selecting a knowledge base (`--kb`)
`--kb` accepts a **name or a UUID** (resolved server-side). Resolution order:
`--kb` flag → `WEKNORA_KB_ID` env → directory link (`weknora link --kb X` binds
the cwd) → error. Read/create commands that operate "inside a project" inherit
the link; **`search *` and destructive `--all` operations always require an
explicit `--kb`** (so an agent never silently hits the wrong corpus).
## 3. Output contract — every command
Default output is `--format json`: a single envelope.
| Field | Meaning |
|---|---|
| `ok` | `true`/`false` — branch on this first (see batch/wait caveat below) |
| `data` | success payload (object or array; absent on mutation-only success) |
| `meta` | `count` / `has_more` / `dry_run` / `plan` … |
| `error` | on failure: `{type, message, hint?, retry_command?, risk?}` |
| `profile` | active profile name |
- `error.type` is a **stable typed code** (e.g. `local.kb_not_found`,
`input.invalid_argument`, `input.confirmation_required`, `server.error`).
Branch on it; `error.hint` usually tells you the next action.
- `--format text` = human output. Streaming commands (`chat`, `session ask`,
`session continue-stream`) emit **NDJSON event lines** by default — expect a
stream, not one blob; use `--format text` for a rendered transcript.
- `--jq '<expr>'` filters the envelope (e.g. `weknora kb list --jq '.data[].id'`).
- Exception: `weknora auth token` emits the **raw token** by default (it's a
scripting helper); pass `--format json` for the `{token, mode, profile}` envelope.
- **Batch / wait caveat:** for multi-item commands (`doc/chunk/session delete`
with several ids, `doc wait`), `ok:true` means *the command ran to completion*,
**not** that every item succeeded — per-item failures live in `data` and the
**exit code** carries the aggregate verdict (e.g. `doc wait` exits 1 if any doc
failed, 124 on timeout, while still printing the `{completed,failed,timeout}`
partition with `ok:true`). For these, branch on the **exit code**, then read
`data` for which items failed.
## 4. Exit codes (branch on these)
| Code | Meaning (typed code class) | Agent action |
|---|---|---|
| 0 | success (incl. `--dry-run`) | proceed |
| 1 | `local.*` / unclassified (incl. `local.kb_not_found`) | read `error`, decide retry/abort |
| 2 | flag / argument validation (bad/unknown/missing-required flag) | re-check `weknora <cmd> --help` |
| 3 | `auth.*` (missing / expired / forbidden) | re-auth, then retry |
| 4 | `resource.not_found` (a server resource id) | verify the id |
| 5 | `input.*` (other than confirmation_required) | adjust args, retry |
| 6 | `server.rate_limited` | back off, retry |
| 7 | `server.*` / `network.*` | transient — retry with backoff |
| **10** | **`input.confirmation_required` (destructive)** | **see §5 — never auto-bypass** |
| 124 | `operation.timeout` | raise `--timeout` or check the job |
| 130 | `operation.cancelled` (SIGINT/SIGTERM) | stop, don't retry |
Note: a server resource id that doesn't exist → exit 4 (`resource.not_found`);
a `--kb` *name* that doesn't resolve client-side → exit 1 (`local.kb_not_found`).
`input.invalid_argument` spans two exit codes — a malformed *invocation*
(unknown/missing flag, wrong arg count) exits **2** (fix the command, re-check
`--help`), while a value rejected *after* parsing (e.g. a bad `--jq` expression)
exits **5** (adjust the value, retry). Branch on the exit code to tell them apart.
## 5. Destructive writes (exit 10) — hard rule
Destructive commands (`kb/doc/chunk/session/agent delete`, `kb/agent edit`,
`auth logout`, `doc delete --all`, …) without `-y` exit **10** with
`error.type = input.confirmation_required` and `error.risk = {level, action}`:
```jsonc
{"ok":false,"error":{"type":"input.confirmation_required",
"message":"delete knowledge base X requires explicit confirmation: re-run with -y",
"retry_command":"weknora kb delete X -y","risk":{"level":"destructive","action":"kb.delete"}}}
```
**Surface this to the user and get explicit approval. Re-run with `-y` ONLY
after they approve. Never add `-y` on your own initiative.**
## 6. Preview before acting — `--dry-run`
Any mutation accepts `--dry-run`: it resolves the request, prints
`meta.plan = {action, args}`, makes **zero** server/side-effect changes, exits 0.
Use it to confirm a command is well-formed (and what it would do) before running
it for real — especially before destructive or bulk operations.
## 7. Chaining (mutation → id)
Mutations return the new resource id in `data.id`. Chain with it:
```bash
KB=$(weknora kb create "Docs" --jq '.data.id' --format json | tr -d '"')
weknora doc upload ./manual.pdf --kb "$KB"
```
## 8. Resource model & command map
```
kb knowledge bases list/view/create/edit/delete/pin/unpin/status/check
doc documents in a KB list/view/create/upload/fetch/download/delete/wait
chunk retrieval units list/view/delete (RAG debug; not search)
session conversations list/view/delete/ask/stop/continue-stream
agent custom agents list/view/create/edit/delete/status/check
search retrieval chunks / docs / kb / sessions
chat one-shot KB RAG Q&A (streaming)
api raw HTTP passthrough to any server endpoint (escape hatch)
link/unlink bind cwd to a KB mcp serve expose weknora as MCP tools
```
**chat vs session ask vs search** — the most common confusion: see the
`weknora-rag-search` skill for the decision table. Briefly: `chat` = one-shot
KB Q&A with an LLM; `session ask --agent <id>` = invoke a *custom agent*;
`search chunks` = raw hybrid retrieval (no LLM).
## 9. CLI vs MCP
For your own scripted control, use the CLI (richer: dry-run, exit-10, all verbs).
For an IDE/host agent that speaks MCP, `weknora mcp serve` exposes a curated
read+chat tool set: `kb_list`, `kb_view`, `doc_list`, `doc_view`, `doc_download`,
`search_chunks`, `chunk_list`, `agent_list`, `chat`, `session_ask`. MCP tools
take raw ids (no name resolution); resolve names via `kb_list` first.
## 10. Agent self-help
Run `weknora <command> --help`. With `WEKNORA_AGENT_HELP=1` set, `--help` emits a
JSON blob (`used_for` / `required_flags` / `examples`) instead of prose — parse
that to learn a command without scraping the human help table.
## Common mistakes
| Mistake | Fix |
|---|---|
| `weknora auth login` before any profile exists | `profile add <n> --host <url> --use` first (§1) |
| Expecting `auth login --host/--name` flags | Gone — host comes from the profile; use global `--profile` |
| Auto-adding `-y` to clear an exit-10 | Never; get user approval first (§5) |
| `weknora chat "..."` prints "garbled" stream | NDJSON by default; use `--format text` for a transcript |
| `search chunks "q"` → exit 2 | `--kb` is required on search; pass `--kb <name-or-id>` |