From 1c735ebb9e40732abb4f5ce062b7fb463cf1fa76 Mon Sep 17 00:00:00 2001 From: adawang Date: Tue, 16 Jun 2026 16:41:07 +0800 Subject: [PATCH] feat(wiki): front-load static rules in chunk-citation prompt for prefix cache WikiChunkCitationPrompt placed the per-batch block before the static rules, output schema and the per-document-stable . Within one document only ChunksXML changes between batches, so the stable content trailing the chunks could not form a long common prefix and providers re-billed the static rules on every citation batch (issue #1687 measured ~6.4% prefix-cache hit rate on wiki synthesis). Reorder to [rules | output schema | candidate_slugs | chunks] so every batch after the first shares the long [rules | candidate_slugs] prefix. No instruction text was dropped; only block order and the inline above/below directional references changed. Verified the cacheable prefix grows from 281 to 2366 bytes and is byte-identical across batches. Addresses #1687 Co-Authored-By: Claude Opus 4.8 --- internal/agent/prompts_wiki.go | 31 +++++++++------ internal/agent/prompts_wiki_test.go | 62 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 3402a006..f200a8c7 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -207,30 +207,27 @@ Output ONLY valid JSON. Example: // to read a batch of chunks and, for each candidate entity/concept, list the // chunk IDs that substantively discuss it. This keeps per-slug "facts" in // their verbatim form (the chunk text) instead of asking the LLM to paraphrase. +// Block order matters for provider prefix caching: the static rules, +// output schema and the per-document-stable are placed +// BEFORE the per-batch block. Within one document only ChunksXML +// changes between batches, so every batch after the first shares the long +// [rules | candidate_slugs] prefix and avoids re-billing the static rules. const WikiChunkCitationPrompt = `You are a precise citation system. Your job is to scan a batch of document chunks and decide, for each candidate entity/concept below, which chunks substantively discuss it. - -{{.CandidateSlugs}} - - - -{{.ChunksXML}} - - **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}**. ### Primary task -For each candidate slug above, select the chunk IDs (from the block) that **substantively discuss** that entity/concept. "Substantively" means the chunk states at least one concrete fact, attribute, step, date, number, relationship, or other useful piece of information about the candidate — not a passing mention. +For each candidate slug (listed in below), select the chunk IDs (from the block below) that **substantively discuss** that entity/concept. "Substantively" means the chunk states at least one concrete fact, attribute, step, date, number, relationship, or other useful piece of information about the candidate — not a passing mention. -- Only cite chunks that appear in the block above. +- Only cite chunks that appear in the block below. - Use the "id" attribute of each element verbatim (e.g. "c003"). - If a candidate is not meaningfully discussed in ANY chunk in this batch, omit it from the output (do not include empty arrays). - A chunk CAN be cited by multiple candidates if it genuinely discusses multiple of them. - If a chunk is overly long or mixes unrelated topics, still cite it for every candidate it discusses. ### Secondary task: new slugs -If this batch reveals a significant entity/concept that is **NOT** in , you may add it under "new_slugs" so it gets incorporated. Only add genuinely new, substantively-discussed items. Do NOT rediscover items already listed above — reuse their slug if they are already candidates. +If this batch reveals a significant entity/concept that is **NOT** in , you may add it under "new_slugs" so it gets incorporated. Only add genuinely new, substantively-discussed items. Do NOT rediscover items already listed in — reuse their slug if they are already candidates. Each new slug must include: - "type": "entity" or "concept" @@ -261,7 +258,17 @@ Output format: ] } -If nothing in this batch is cite-worthy, return: {"citations": {}, "new_slugs": []}` +If nothing in this batch is cite-worthy, return: {"citations": {}, "new_slugs": []} + + +{{.CandidateSlugs}} + + + +{{.ChunksXML}} + + +Now apply the instructions above to the chunks and output ONLY the JSON.` // WikiPageModifyPrompt updates an existing wiki page with new additions and removes stale/deleted information in a single pass. const WikiPageModifyPrompt = `You are a wiki editor tasked with updating an existing wiki page. You must process a set of NEW information to add, AND/OR a set of deleted documents whose exclusive contributions must be REMOVED. diff --git a/internal/agent/prompts_wiki_test.go b/internal/agent/prompts_wiki_test.go index 38c84e7f..2a538aa3 100644 --- a/internal/agent/prompts_wiki_test.go +++ b/internal/agent/prompts_wiki_test.go @@ -3,6 +3,7 @@ package agent import ( "strings" "testing" + "text/template" ) func TestWikiGranularityGuidance_RoutesByKey(t *testing.T) { @@ -61,3 +62,64 @@ func TestWikiGranularityGuidance_BlocksAreDistinct(t *testing.T) { t.Error("exhaustive block should self-identify") } } + +func renderWikiChunkCitation(t *testing.T, candidateSlugs, chunksXML, lang string) string { + t.Helper() + tmpl, err := template.New("cite").Parse(WikiChunkCitationPrompt) + if err != nil { + t.Fatalf("parse template: %v", err) + } + var b strings.Builder + if err := tmpl.Execute(&b, map[string]string{ + "CandidateSlugs": candidateSlugs, + "ChunksXML": chunksXML, + "Language": lang, + }); err != nil { + t.Fatalf("execute template: %v", err) + } + return b.String() +} + +// TestWikiChunkCitationPrompt_StablePrefixAcrossBatches verifies the property +// that enables provider prefix caching (issue #1687): within one document the +// candidate slugs and the static rules are constant and only the per-batch +// block changes, so everything up to must be byte-identical +// across batches. If the static rules trailed they would be re-billed +// on every batch and this prefix would diverge. +func TestWikiChunkCitationPrompt_StablePrefixAcrossBatches(t *testing.T) { + const slugs = "entity/acme = Acme Corp\nconcept/rag = Retrieval-Augmented Generation" + + a := renderWikiChunkCitation(t, slugs, `first batch text`, "English") + b := renderWikiChunkCitation(t, slugs, `a completely different second batch`, "English") + + // Match the standalone tag line (the per-batch data block), not the + // inline " block" references inside the instructions prose. + const marker = "\n\n" + ia := strings.Index(a, marker) + ib := strings.Index(b, marker) + if ia < 0 || ib < 0 { + t.Fatalf("rendered prompt missing %q block", marker) + } + + if a[:ia] != b[:ib] { + t.Errorf("prompt prefix before differs across batches — provider prefix cache will miss.\nA-prefix:\n%s\n---\nB-prefix:\n%s", a[:ia], b[:ib]) + } + + // The static rules and per-document candidate-slug block must live inside + // that shared prefix, not after the varying chunks. + for _, must := range []string{"### Primary task", "### JSON Formatting Rules", "\n\n"} { + if idx := strings.Index(a, must); idx < 0 || idx > ia { + t.Errorf("%q must appear before to be part of the cached prefix (idx=%d, chunks=%d)", must, idx, ia) + } + } +} + +// TestWikiChunkCitationPrompt_PreservesPlaceholders guards against accidental +// loss of a template field during future reorders. +func TestWikiChunkCitationPrompt_PreservesPlaceholders(t *testing.T) { + for _, field := range []string{"{{.Language}}", "{{.CandidateSlugs}}", "{{.ChunksXML}}"} { + if !strings.Contains(WikiChunkCitationPrompt, field) { + t.Errorf("WikiChunkCitationPrompt lost template field %q", field) + } + } +}