diff --git a/internal/application/repository/retriever/opensearch/crud.go b/internal/application/repository/retriever/opensearch/crud.go
new file mode 100644
index 00000000..1a03efbe
--- /dev/null
+++ b/internal/application/repository/retriever/opensearch/crud.go
@@ -0,0 +1,405 @@
+package opensearch
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "strings"
+
+ osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// additionalParams contract (verified against
+// elasticsearch/structs.go:ToDBVectorEmbedding and
+// keywords_vector_hybrid_indexer.go:concurrentBatchSave on
+// upstream/main):
+//
+// additionalParams["embedding"] map[string][]float32 // keyed by IndexInfo.SourceID
+// additionalParams["chunk_enabled"] map[string]bool // keyed by IndexInfo.ChunkID
+//
+// Both keys may be absent. Missing "embedding" → keyword-only path
+// (vector field omitted from the OpenSearch document, routed to the
+// dim-less keywords index). Missing "chunk_enabled" → IndexInfo's own
+// IsEnabled field is used as-is.
+
+// Save indexes a single chunk. Idempotent: _id is set to chunk_id so
+// re-indexing the same chunk overwrites instead of creating a duplicate.
+func (r *Repository) Save(
+ ctx context.Context,
+ info *types.IndexInfo,
+ params map[string]any,
+) error {
+ emb := lookupEmbedding(params, info.SourceID)
+ enabled := lookupChunkEnabled(params, info.ChunkID, info.IsEnabled)
+ dim := len(emb)
+
+ var targetIndex string
+ if dim > 0 {
+ if err := r.ensureReady(ctx, dim); err != nil {
+ return err
+ }
+ targetIndex = r.indexAlias(dim)
+ } else {
+ // Keyword-only path: no embedding supplied. Route to the
+ // dim-less _keywords index.
+ if err := r.ensureKeywordsIndex(ctx); err != nil {
+ return err
+ }
+ targetIndex = r.keywordsIndex()
+ }
+
+ doc, err := json.Marshal(toDoc(info, emb, enabled))
+ if err != nil {
+ return fmt.Errorf("opensearch: marshal doc: %w", err)
+ }
+ req := osapi.IndexReq{
+ Index: targetIndex,
+ DocumentID: info.ChunkID, // _id = chunk_id (idempotent)
+ Body: bytes.NewReader(doc),
+ }
+ resp, err := r.client.Index(ctx, req)
+ if err != nil {
+ return wrapTransport(err)
+ }
+ if resp != nil {
+ drainAndClose(resp.Inspect().Response.Body)
+ }
+ return nil
+}
+
+// BatchSave NDJSON-bulk-indexes chunks. Inspects per-item errors and
+// returns a typed error if ANY item fails.
+//
+// Dim-aware pre-marshal size cap: estimateBulkBodyBytes predicts NDJSON
+// size before allocating; over-cap callers get ErrBatchTooLarge instead
+// of a 50MB allocation that's then rejected. The service layer is
+// responsible for dim-aware sub-batching above this.
+func (r *Repository) BatchSave(
+ ctx context.Context,
+ infos []*types.IndexInfo,
+ params map[string]any,
+) error {
+ if len(infos) == 0 {
+ return nil
+ }
+ emb, dim, err := extractBatchEmbeddings(params, infos)
+ if err != nil {
+ return err
+ }
+
+ // Pre-marshal size + count check.
+ if estimated := estimateBulkBodyBytes(len(infos), dim); estimated > 10*1024*1024 {
+ return fmt.Errorf(
+ "opensearch: estimated bulk body %dB exceeds 10MB cap (n=%d, dim=%d): %w",
+ estimated, len(infos), dim, ErrBatchTooLarge,
+ )
+ }
+ if len(infos) > 1000 {
+ return fmt.Errorf("opensearch: bulk n=%d exceeds 1000-doc cap: %w",
+ len(infos), ErrBatchTooLarge)
+ }
+
+ // Route by dim: dim==0 means the entire batch is keyword-only
+ // (concurrentBatchSaveNoEmbedding path), use the dim-less index.
+ var alias string
+ if dim == 0 {
+ if err := r.ensureKeywordsIndex(ctx); err != nil {
+ return err
+ }
+ alias = r.keywordsIndex()
+ } else {
+ if err := r.ensureReady(ctx, dim); err != nil {
+ return err
+ }
+ alias = r.indexAlias(dim)
+ }
+
+ body, err := buildBulkBody(alias, infos, emb, params)
+ if err != nil {
+ return err
+ }
+ req := osapi.BulkReq{Body: bytes.NewReader(body)}
+ resp, err := r.client.Bulk(ctx, req)
+ if err != nil {
+ return wrapTransport(err)
+ }
+ if resp != nil {
+ defer drainAndClose(resp.Inspect().Response.Body)
+ }
+ // Cap response body at 64MB (bulks of 1000 items are ~1-2MB success,
+ // can balloon with full per-item errors).
+ return inspectBulkResponse(io.LimitReader(resp.Inspect().Response.Body, 64<<20))
+}
+
+// estimateBulkBodyBytes returns a rough NDJSON size in bytes. Slight
+// overestimate so the cap fires before real allocation. Per-item:
+// - action header `{"index":{"_index":"...","_id":"..."}}\n` ≈ 100B
+// - doc body: dim*5 (float ASCII) + ~700B metadata + content ≈ 1024B
+func estimateBulkBodyBytes(n, dim int) int {
+ return n * (100 + dim*5 + 1024)
+}
+
+// buildBulkBody constructs the NDJSON request body. emb[i] may be nil/
+// empty — toDoc handles the keyword-only case by omitting the
+// "embedding" field. chunk_enabled lookup also happens per-doc.
+func buildBulkBody(alias string, infos []*types.IndexInfo, emb [][]float32, params map[string]any) ([]byte, error) {
+ var buf bytes.Buffer
+ for i, info := range infos {
+ action := map[string]any{
+ "index": map[string]any{
+ "_index": alias,
+ "_id": info.ChunkID,
+ },
+ }
+ actionJSON, err := json.Marshal(action)
+ if err != nil {
+ return nil, fmt.Errorf("opensearch: marshal bulk action %d: %w", i, err)
+ }
+ buf.Write(actionJSON)
+ buf.WriteByte('\n')
+
+ enabled := lookupChunkEnabled(params, info.ChunkID, info.IsEnabled)
+ docJSON, err := json.Marshal(toDoc(info, emb[i], enabled))
+ if err != nil {
+ return nil, fmt.Errorf("opensearch: marshal doc %d: %w", i, err)
+ }
+ buf.Write(docJSON)
+ buf.WriteByte('\n')
+ }
+ return buf.Bytes(), nil
+}
+
+// inspectBulkResponse parses OpenSearch bulk response, returns a typed
+// error if any item has a per-item .error. Surfaces up to 5 distinct
+// error messages + total count for operator diagnosis.
+//
+// Does NOT include det.Error.Reason in the wrapped public error
+// (Reason can contain document content fragments). Only ID + bounded
+// enum Type is included. Full per-item Reason goes to log.Debug.
+func inspectBulkResponse(body io.Reader) error {
+ var r struct {
+ Errors bool `json:"errors"`
+ Items []map[string]struct {
+ ID string `json:"_id"`
+ Status int `json:"status"`
+ Error *struct {
+ Type string `json:"type"`
+ Reason string `json:"reason"`
+ } `json:"error,omitempty"`
+ } `json:"items"`
+ }
+ if err := json.NewDecoder(body).Decode(&r); err != nil {
+ return fmt.Errorf("opensearch: parse bulk response: %w", ErrTransport)
+ }
+ if !r.Errors {
+ return nil
+ }
+ log := logger.GetLogger(context.Background())
+ var msgs []string
+ total := 0
+ for _, item := range r.Items {
+ for op, det := range item { // {"index": {...}}
+ if det.Error == nil {
+ continue
+ }
+ total++
+ // Full reason → debug log only (may contain doc content).
+ log.Debugf("[OpenSearch] bulk item err: op=%s id=%s type=%s reason=%s",
+ op, det.ID, det.Error.Type, det.Error.Reason)
+ if len(msgs) < 5 {
+ // Public message: ID (operator-meaningful) + Type (bounded enum).
+ msgs = append(msgs, fmt.Sprintf("[%s %s] %s", op, det.ID, det.Error.Type))
+ }
+ }
+ }
+ if total == 0 {
+ return nil // shouldn't happen if r.Errors=true, but defensive
+ }
+ return fmt.Errorf("opensearch: bulk partial failure (%d items failed, first 5: %s): %w",
+ total, strings.Join(msgs, "; "), ErrTransport)
+}
+
+// toDoc projects IndexInfo + embedding to the OpenSearch document
+// shape. source_type is `int` (not stringified) — mapping declared as
+// `integer` field for stable cross-enum-extension behavior. Omits the
+// "embedding" field entirely when emb is empty (keyword-only doc, no
+// k-NN graph entry).
+func toDoc(info *types.IndexInfo, emb []float32, enabled bool) map[string]any {
+ doc := map[string]any{
+ "chunk_id": info.ChunkID,
+ "knowledge_id": info.KnowledgeID,
+ "knowledge_base_id": info.KnowledgeBaseID,
+ "source_id": info.SourceID,
+ "source_type": int(info.SourceType),
+ "tag_id": info.TagID,
+ "content": info.Content,
+ "is_enabled": enabled,
+ "is_recommended": info.IsRecommended,
+ }
+ if len(emb) > 0 {
+ doc["embedding"] = emb
+ }
+ return doc
+}
+
+// lookupEmbedding returns the embedding vector for info.SourceID.
+// Empty slice (not nil) indicates "no embedding provided" — caller
+// MUST check len(emb) before calling ensureReady.
+//
+// Defense in depth: tolerates the wrong shape rather than crash. The
+// service layer is responsible for getting params right; we degrade
+// to keyword-only for this doc and log a warning instead of panicking
+// downstream.
+func lookupEmbedding(params map[string]any, sourceID string) []float32 {
+ if params == nil {
+ return nil
+ }
+ raw, ok := params["embedding"]
+ if !ok {
+ return nil
+ }
+ embMap, ok := raw.(map[string][]float32)
+ if !ok {
+ logger.GetLogger(context.Background()).Warnf(
+ "[OpenSearch] additionalParams[\"embedding\"] is %T, want map[string][]float32", raw)
+ return nil
+ }
+ return embMap[sourceID] // nil/empty if SourceID not in map
+}
+
+// lookupChunkEnabled overlays the chunk_enabled map on top of
+// info.IsEnabled. Returns the overlay value if present, else the
+// default (info.IsEnabled).
+func lookupChunkEnabled(params map[string]any, chunkID string, def bool) bool {
+ if params == nil {
+ return def
+ }
+ raw, ok := params["chunk_enabled"]
+ if !ok {
+ return def
+ }
+ enMap, ok := raw.(map[string]bool)
+ if !ok {
+ return def
+ }
+ if v, present := enMap[chunkID]; present {
+ return v
+ }
+ return def
+}
+
+// extractBatchEmbeddings projects the params["embedding"] map onto the
+// info list, returning per-info embeddings (some may be empty) and the
+// dimension inferred from the first non-empty entry. Returns dim=0 if
+// no info has an embedding — caller routes to the keywords-only path.
+//
+// Defense in depth: all non-empty embeddings MUST share the same
+// dimension. Mixed-dim → ErrDimensionMismatch.
+func extractBatchEmbeddings(params map[string]any, infos []*types.IndexInfo) ([][]float32, int, error) {
+ out := make([][]float32, len(infos))
+ dim := 0
+ for i, info := range infos {
+ emb := lookupEmbedding(params, info.SourceID)
+ out[i] = emb
+ if len(emb) > 0 {
+ if dim == 0 {
+ dim = len(emb)
+ } else if len(emb) != dim {
+ return nil, 0, fmt.Errorf(
+ "opensearch: embedding[%s] dim=%d != first non-empty dim=%d: %w",
+ info.SourceID, len(emb), dim, ErrDimensionMismatch,
+ )
+ }
+ }
+ }
+ return out, dim, nil
+}
+
+// DeleteByChunkIDList runs _delete_by_query with a terms filter on
+// chunk_id. Sync only — large batches (>1000) are rejected with
+// ErrBatchTooLarge; the async task path that lifts the cap arrives in
+// a later change.
+//
+// Refresh policy: opensearch-go v4's typed Params.Refresh is *bool, so
+// the wire-level "wait_for" value documented in the OpenSearch REST
+// API is not directly expressible. Refresh:&true forces immediate
+// segment flush (read-your-writes guaranteed, but expensive at scale).
+// A follow-up can revisit via a custom raw request if telemetry shows
+// the cost is material at production batch frequencies.
+func (r *Repository) DeleteByChunkIDList(
+ ctx context.Context, chunkIDs []string, dim int, _ string,
+) error {
+ return r.deleteByList(ctx, chunkIDs, dim, "chunk_id")
+}
+
+func (r *Repository) DeleteBySourceIDList(
+ ctx context.Context, sourceIDs []string, dim int, _ string,
+) error {
+ return r.deleteByList(ctx, sourceIDs, dim, "source_id")
+}
+
+func (r *Repository) DeleteByKnowledgeIDList(
+ ctx context.Context, knowledgeIDs []string, dim int, _ string,
+) error {
+ return r.deleteByList(ctx, knowledgeIDs, dim, "knowledge_id")
+}
+
+// deleteByList factors the common cap / empty / ensureReady / dispatch
+// logic out of the three DeleteBy* methods. dim==0 routes to the
+// dim-less keywords index.
+func (r *Repository) deleteByList(
+ ctx context.Context, ids []string, dim int, field string,
+) error {
+ if len(ids) == 0 {
+ return nil
+ }
+ if len(ids) > 1000 {
+ return fmt.Errorf("opensearch: %s-delete batch %d > 1000 cap: %w",
+ field, len(ids), ErrBatchTooLarge)
+ }
+ var index string
+ if dim == 0 {
+ if err := r.ensureKeywordsIndex(ctx); err != nil {
+ return err
+ }
+ index = r.keywordsIndex()
+ } else {
+ if err := r.ensureReady(ctx, dim); err != nil {
+ return err
+ }
+ index = r.indexAlias(dim)
+ }
+ return r.deleteByTerms(ctx, index, field, ids)
+}
+
+func (r *Repository) deleteByTerms(
+ ctx context.Context, index, field string, values []string,
+) error {
+ body, err := json.Marshal(map[string]any{
+ "query": map[string]any{
+ "terms": map[string]any{field: values},
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("opensearch: marshal delete body: %w", err)
+ }
+ refresh := true
+ req := osapi.DocumentDeleteByQueryReq{
+ Indices: []string{index},
+ Body: bytes.NewReader(body),
+ Params: osapi.DocumentDeleteByQueryParams{Refresh: &refresh},
+ }
+ resp, err := r.client.Document.DeleteByQuery(ctx, req)
+ if err != nil {
+ return wrapTransport(err)
+ }
+ if resp != nil {
+ drainAndClose(resp.Inspect().Response.Body)
+ }
+ return nil
+}
diff --git a/internal/application/repository/retriever/opensearch/query.go b/internal/application/repository/retriever/opensearch/query.go
new file mode 100644
index 00000000..ae306889
--- /dev/null
+++ b/internal/application/repository/retriever/opensearch/query.go
@@ -0,0 +1,154 @@
+package opensearch
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// retrieveFilters is the typed shape of OpenSearch query filter
+// clauses. It is the ONLY way callers (Retrieve) can constrain a
+// query — there is no JSON-fragment escape hatch. This prevents the
+// "map[string]interface{} injection" failure mode where a hostile
+// chunk_id like {"$or":[...]} would otherwise be inlined as JSON DSL.
+//
+// All slice fields are interpreted as IN-set membership (OS terms
+// query). Empty slice = no constraint on that dimension.
+//
+// IncludeDisabled controls the implicit is_enabled = true clause:
+// - false (default) — appends is_enabled term, hiding disabled chunks
+// - true — omits the clause; admin / audit callers can
+// observe disabled chunks. Service layer pins false for user-facing
+// APIs.
+type retrieveFilters struct {
+ KBIDs []string
+ KnowledgeIDs []string
+ TagIDs []string
+ ExcludeChunkIDs []string
+ ExcludeKnowledgeIDs []string
+ IncludeDisabled bool
+}
+
+// fromParams projects RetrieveParams to retrieveFilters, applying the
+// multi-KB input from the service-layer dispatcher as terms.
+// ExcludeKnowledgeIDs is included so the OpenSearch driver matches the
+// behaviour of the Qdrant / pgvector drivers.
+func fromParams(p types.RetrieveParams) *retrieveFilters {
+ return &retrieveFilters{
+ KBIDs: p.KnowledgeBaseIDs,
+ KnowledgeIDs: p.KnowledgeIDs,
+ TagIDs: p.TagIDs,
+ ExcludeChunkIDs: p.ExcludeChunkIDs,
+ ExcludeKnowledgeIDs: p.ExcludeKnowledgeIDs,
+ // IncludeDisabled stays false — set explicitly by admin callers
+ // only. Driver receives this from a typed field, not from
+ // AdditionalParams, so the contract is checked at compile time.
+ }
+}
+
+// toBoolMust emits the OpenSearch bool.must clause list for this
+// filter. Returns a freshly allocated slice each call so callers can
+// safely append additional clauses (e.g. the keyword match clause in
+// buildKeywordQuery) without aliasing.
+func (f *retrieveFilters) toBoolMust() []map[string]any {
+ var must []map[string]any
+ if len(f.KBIDs) > 0 {
+ must = append(must, map[string]any{
+ "terms": map[string]any{"knowledge_base_id": f.KBIDs},
+ })
+ }
+ if len(f.KnowledgeIDs) > 0 {
+ must = append(must, map[string]any{
+ "terms": map[string]any{"knowledge_id": f.KnowledgeIDs},
+ })
+ }
+ if len(f.TagIDs) > 0 {
+ must = append(must, map[string]any{
+ "terms": map[string]any{"tag_id": f.TagIDs},
+ })
+ }
+ if len(f.ExcludeChunkIDs) > 0 {
+ must = append(must, map[string]any{
+ "bool": map[string]any{
+ "must_not": map[string]any{
+ "terms": map[string]any{"chunk_id": f.ExcludeChunkIDs},
+ },
+ },
+ })
+ }
+ if len(f.ExcludeKnowledgeIDs) > 0 {
+ must = append(must, map[string]any{
+ "bool": map[string]any{
+ "must_not": map[string]any{
+ "terms": map[string]any{"knowledge_id": f.ExcludeKnowledgeIDs},
+ },
+ },
+ })
+ }
+ if !f.IncludeDisabled {
+ must = append(must, map[string]any{
+ "term": map[string]any{"is_enabled": true},
+ })
+ }
+ return must
+}
+
+// buildKNNQuery composes a pure k-NN ANN query with caller filters. No
+// hybrid pipeline — native hybrid is out of scope for this PR; fusion
+// stays in the service-layer RRF.
+//
+// Applies `min_score` when threshold > 0. The OpenSearch k-NN plugin's
+// SpaceType.COSINESIMIL.scoreTranslation produces (1 + cos) / 2 ∈ [0, 1]
+// at the client, so the caller's Threshold value passes through to
+// min_score directly without further scaling.
+//
+// Signature returns ([]byte, error) so callers can propagate
+// json.Marshal failures (reachable for NaN / +/-Inf in embedding).
+func buildKNNQuery(embedding []float32, topK int, threshold float64, f *retrieveFilters) ([]byte, error) {
+ knn := map[string]any{
+ "embedding": map[string]any{
+ "vector": embedding,
+ "k": topK,
+ "filter": map[string]any{
+ "bool": map[string]any{"must": f.toBoolMust()},
+ },
+ },
+ }
+ body := map[string]any{
+ "size": topK,
+ "query": map[string]any{"knn": knn},
+ }
+ if threshold > 0 {
+ body["min_score"] = threshold
+ }
+ out, err := json.Marshal(body)
+ if err != nil {
+ return nil, fmt.Errorf("marshal knn query: %w", err)
+ }
+ return out, nil
+}
+
+// buildKeywordQuery composes a BM25 match query against the content
+// field with caller filters. Score range is OpenSearch BM25 native (no
+// upper bound) — the normalizer keyword-passthrough rule treats keyword
+// retrievers as rank-based for RRF, so the score scale does not need
+// normalization here. min_score still applied if requested.
+func buildKeywordQuery(queryText string, topK int, threshold float64, f *retrieveFilters) ([]byte, error) {
+ must := f.toBoolMust()
+ must = append(must, map[string]any{
+ "match": map[string]any{"content": queryText},
+ })
+ body := map[string]any{
+ "size": topK,
+ "query": map[string]any{"bool": map[string]any{"must": must}},
+ }
+ if threshold > 0 {
+ body["min_score"] = threshold
+ }
+ out, err := json.Marshal(body)
+ if err != nil {
+ return nil, fmt.Errorf("marshal keyword query: %w", err)
+ }
+ return out, nil
+}
diff --git a/internal/application/repository/retriever/opensearch/repository_test.go b/internal/application/repository/retriever/opensearch/repository_test.go
index 418f77c6..76bf83c6 100644
--- a/internal/application/repository/retriever/opensearch/repository_test.go
+++ b/internal/application/repository/retriever/opensearch/repository_test.go
@@ -301,18 +301,6 @@ func TestBuildIndexMapping_LuceneEngine_768Dim(t *testing.T) {
t.Errorf("%s: want keyword, got %v", f, props[f])
}
}
-
- // Boolean flags from the IndexInfo / VectorEmbedding shape.
- for _, f := range []string{"is_enabled", "is_recommended"} {
- fp, ok := props[f].(map[string]any)
- if !ok {
- t.Errorf("%s: missing from mapping", f)
- continue
- }
- if fp["type"] != "boolean" {
- t.Errorf("%s: want boolean, got %v", f, fp["type"])
- }
- }
}
func TestBuildIndexMapping_FaissEngine_CustomParams(t *testing.T) {
@@ -353,23 +341,394 @@ func TestBuildKeywordsMapping_OmitsEmbeddingField(t *testing.T) {
if _, ok := props["content"]; !ok {
t.Error("keywords mapping must include content field for BM25")
}
- // Boolean flags from the IndexInfo / VectorEmbedding shape.
- for _, f := range []string{"is_enabled", "is_recommended"} {
- fp, ok := props[f].(map[string]any)
- if !ok {
- t.Errorf("%s: missing from keywords mapping", f)
- continue
- }
- if fp["type"] != "boolean" {
- t.Errorf("%s: want boolean, got %v", f, fp["type"])
- }
- }
}
// ============================================================================
// retrieveFilters — typed filter struct closes JSON injection surface
// ============================================================================
+func TestRetrieveFilters_RejectsJSONFragmentInjection(t *testing.T) {
+ t.Parallel()
+ // Hostile caller embeds JSON fragments in chunk IDs. The driver must
+ // treat them as opaque strings (JSON-injection regression guard).
+ f := &retrieveFilters{
+ ExcludeChunkIDs: []string{`{"$or":[{"_id":"x"}]}`, `*`, `"; DROP TABLE chunks; --`},
+ }
+ body, err := buildKNNQuery([]float32{0.1, 0.2}, 5, 0, f)
+ if err != nil {
+ t.Fatalf("buildKNNQuery: %v", err)
+ }
+ // The hostile strings must appear as JSON-escaped string literals
+ // inside the terms array, not as parsed DSL.
+ asStr := string(body)
+ if !strings.Contains(asStr, `"{\"$or\":[{\"_id\":\"x\"}]}"`) {
+ t.Errorf("hostile chunk_id must be JSON-escaped string literal in body: %s", asStr)
+ }
+}
+
+func TestRetrieveFilters_AppliesExcludeKnowledgeIDs(t *testing.T) {
+ t.Parallel()
+ f := &retrieveFilters{ExcludeKnowledgeIDs: []string{"k1", "k2"}}
+ body, err := buildKNNQuery([]float32{0.1}, 5, 0, f)
+ if err != nil {
+ t.Fatalf("buildKNNQuery: %v", err)
+ }
+ if !strings.Contains(string(body), `"knowledge_id"`) {
+ t.Errorf("must_not.terms.knowledge_id missing from query body: %s", body)
+ }
+}
+
+func TestRetrieveFilters_IncludeDisabled_SkipsIsEnabledClause(t *testing.T) {
+ t.Parallel()
+ f := &retrieveFilters{IncludeDisabled: true}
+ clauses := f.toBoolMust()
+ for _, c := range clauses {
+ if term, ok := c["term"].(map[string]any); ok {
+ if _, hasEnabled := term["is_enabled"]; hasEnabled {
+ t.Error("IncludeDisabled=true must skip is_enabled clause")
+ }
+ }
+ }
+}
+
+func TestRetrieveFilters_DefaultPinsIsEnabledTrue(t *testing.T) {
+ t.Parallel()
+ f := &retrieveFilters{}
+ clauses := f.toBoolMust()
+ found := false
+ for _, c := range clauses {
+ if term, ok := c["term"].(map[string]any); ok {
+ if v, hasEnabled := term["is_enabled"]; hasEnabled {
+ if v != true {
+ t.Errorf("is_enabled term: want true, got %v", v)
+ }
+ found = true
+ }
+ }
+ }
+ if !found {
+ t.Error("default filter must pin is_enabled=true")
+ }
+}
+
+// ============================================================================
+// buildKNNQuery / buildKeywordQuery — min_score from threshold
+// ============================================================================
+
+func TestBuildKNNQuery_AppliesMinScoreWhenThreshold(t *testing.T) {
+ t.Parallel()
+ body, err := buildKNNQuery([]float32{0.1}, 5, 0.75, &retrieveFilters{})
+ if err != nil {
+ t.Fatalf("buildKNNQuery: %v", err)
+ }
+ var parsed map[string]any
+ _ = json.Unmarshal(body, &parsed)
+ if ms, ok := parsed["min_score"]; !ok || ms.(float64) != 0.75 {
+ t.Errorf("min_score: want 0.75, got %v (present=%v)", ms, ok)
+ }
+}
+
+func TestBuildKNNQuery_OmitsMinScoreWhenThresholdZero(t *testing.T) {
+ t.Parallel()
+ body, err := buildKNNQuery([]float32{0.1}, 5, 0, &retrieveFilters{})
+ if err != nil {
+ t.Fatalf("buildKNNQuery: %v", err)
+ }
+ if strings.Contains(string(body), "min_score") {
+ t.Errorf("min_score must be omitted when threshold=0: %s", body)
+ }
+}
+
+func TestBuildKeywordQuery_AppendsContentMatch(t *testing.T) {
+ t.Parallel()
+ body, err := buildKeywordQuery("hello world", 5, 0, &retrieveFilters{})
+ if err != nil {
+ t.Fatalf("buildKeywordQuery: %v", err)
+ }
+ if !strings.Contains(string(body), `"content":"hello world"`) {
+ t.Errorf("content match missing: %s", body)
+ }
+}
+
+// ============================================================================
+// toDoc — keyword-only path omits embedding field; source_type is integer
+// ============================================================================
+
+func TestToDoc_OmitsEmbeddingField_WhenEmpty(t *testing.T) {
+ t.Parallel()
+ info := &types.IndexInfo{ChunkID: "c1", Content: "x"}
+ doc := toDoc(info, nil, true)
+ if _, ok := doc["embedding"]; ok {
+ t.Error("toDoc must omit embedding when emb is nil")
+ }
+ doc2 := toDoc(info, []float32{}, true)
+ if _, ok := doc2["embedding"]; ok {
+ t.Error("toDoc must omit embedding when emb is empty slice")
+ }
+}
+
+func TestToDoc_IncludesEmbedding_WhenPresent(t *testing.T) {
+ t.Parallel()
+ info := &types.IndexInfo{ChunkID: "c1"}
+ doc := toDoc(info, []float32{0.1, 0.2, 0.3}, true)
+ if emb, ok := doc["embedding"]; !ok {
+ t.Error("toDoc must include embedding when emb is non-empty")
+ } else if len(emb.([]float32)) != 3 {
+ t.Errorf("embedding length: want 3, got %d", len(emb.([]float32)))
+ }
+}
+
+func TestToDoc_SourceTypeIsInteger(t *testing.T) {
+ t.Parallel()
+ info := &types.IndexInfo{
+ ChunkID: "c1",
+ SourceType: types.PassageSourceType, // iota=1
+ }
+ doc := toDoc(info, []float32{0.1}, true)
+ if st, ok := doc["source_type"].(int); !ok {
+ t.Errorf("source_type: want int, got %T", doc["source_type"])
+ } else if st != 1 {
+ t.Errorf("source_type: want 1 (PassageSourceType), got %d", st)
+ }
+}
+
+func TestToDoc_PreservesIsRecommended(t *testing.T) {
+ t.Parallel()
+ for _, want := range []bool{true, false} {
+ info := &types.IndexInfo{ChunkID: "c1", IsRecommended: want}
+ doc := toDoc(info, []float32{0.1}, true)
+ got, ok := doc["is_recommended"].(bool)
+ if !ok {
+ t.Errorf("IsRecommended=%v: doc missing is_recommended field", want)
+ continue
+ }
+ if got != want {
+ t.Errorf("IsRecommended=%v: doc[is_recommended]=%v", want, got)
+ }
+ }
+}
+
+// ============================================================================
+// lookupEmbedding / lookupChunkEnabled — additionalParams keying contract
+// (embedding keyed by SourceID, chunk_enabled keyed by ChunkID)
+// ============================================================================
+
+func TestLookupEmbedding_KeyedBySourceID(t *testing.T) {
+ t.Parallel()
+ params := map[string]any{
+ "embedding": map[string][]float32{
+ "src-A": {1, 2, 3},
+ "src-B": {4, 5, 6},
+ },
+ }
+ if got := lookupEmbedding(params, "src-A"); len(got) != 3 || got[0] != 1 {
+ t.Errorf("lookup src-A: want [1 2 3], got %v", got)
+ }
+ if got := lookupEmbedding(params, "src-B"); got[2] != 6 {
+ t.Errorf("lookup src-B: want [4 5 6], got %v", got)
+ }
+ if got := lookupEmbedding(params, "nonexistent"); got != nil {
+ t.Errorf("missing source: want nil, got %v", got)
+ }
+}
+
+func TestLookupEmbedding_NilParams_ReturnsNil(t *testing.T) {
+ t.Parallel()
+ if got := lookupEmbedding(nil, "any"); got != nil {
+ t.Errorf("nil params: want nil, got %v", got)
+ }
+}
+
+func TestLookupEmbedding_WrongType_DegradesToNil(t *testing.T) {
+ t.Parallel()
+ // If the service layer sends the wrong shape, the driver must degrade
+ // gracefully (keyword-only path) rather than panic. This is the
+ // defense-in-depth from crud.go's documented contract.
+ params := map[string]any{"embedding": "not a map"}
+ if got := lookupEmbedding(params, "any"); got != nil {
+ t.Errorf("wrong type: want nil (degrade), got %v", got)
+ }
+}
+
+func TestLookupChunkEnabled_OverrideAndDefault(t *testing.T) {
+ t.Parallel()
+ params := map[string]any{
+ "chunk_enabled": map[string]bool{
+ "c1": true,
+ "c2": false,
+ },
+ }
+ if !lookupChunkEnabled(params, "c1", false) {
+ t.Error("c1 overlay should be true")
+ }
+ if lookupChunkEnabled(params, "c2", true) {
+ t.Error("c2 overlay should be false (overrides default)")
+ }
+ if !lookupChunkEnabled(params, "missing", true) {
+ t.Error("missing key must fall back to default=true")
+ }
+ if lookupChunkEnabled(nil, "any", false) {
+ t.Error("nil params must fall back to default=false")
+ }
+}
+
+// ============================================================================
+// extractBatchEmbeddings — mixed-dim rejection
+// ============================================================================
+
+func TestExtractBatchEmbeddings_MixedDims_Rejected(t *testing.T) {
+ t.Parallel()
+ infos := []*types.IndexInfo{
+ {SourceID: "a"}, {SourceID: "b"},
+ }
+ params := map[string]any{
+ "embedding": map[string][]float32{
+ "a": {1, 2, 3, 4}, // dim 4
+ "b": {5, 6, 7}, // dim 3 — mismatch
+ },
+ }
+ _, _, err := extractBatchEmbeddings(params, infos)
+ if !errors.Is(err, ErrDimensionMismatch) {
+ t.Errorf("mixed dims: want ErrDimensionMismatch, got %v", err)
+ }
+}
+
+func TestExtractBatchEmbeddings_AllEmpty_DimZero(t *testing.T) {
+ t.Parallel()
+ infos := []*types.IndexInfo{
+ {SourceID: "a"}, {SourceID: "b"},
+ }
+ embs, dim, err := extractBatchEmbeddings(nil, infos)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if dim != 0 {
+ t.Errorf("dim: want 0 (keyword-only), got %d", dim)
+ }
+ if len(embs) != 2 {
+ t.Errorf("embs length: want 2 (parallel to infos), got %d", len(embs))
+ }
+}
+
+// ============================================================================
+// estimateBulkBodyBytes
+// ============================================================================
+
+func TestEstimateBulkBodyBytes(t *testing.T) {
+ t.Parallel()
+ // 1000 docs at 768-dim should fit under 10MB; 1000 at 1536-dim should NOT.
+ if got := estimateBulkBodyBytes(1000, 768); got > 10*1024*1024 {
+ t.Errorf("1000@768 estimate %dB should be under 10MB cap", got)
+ }
+ if got := estimateBulkBodyBytes(2000, 1536); got <= 10*1024*1024 {
+ t.Errorf("2000@1536 estimate %dB should exceed 10MB cap (got under)", got)
+ }
+}
+
+// ============================================================================
+// effectiveTopK
+// ============================================================================
+
+func TestEffectiveTopK(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ cases := []struct {
+ in int
+ want int
+ }{
+ {0, 10}, // default
+ {-5, 10}, // negative falls back to default
+ {5, 5}, // normal
+ {10000, 10000},
+ {50000, 10000}, // cap
+ }
+ for _, tc := range cases {
+ got := effectiveTopK(ctx, types.RetrieveParams{TopK: tc.in})
+ if got != tc.want {
+ t.Errorf("effectiveTopK(%d): want %d, got %d", tc.in, tc.want, got)
+ }
+ }
+}
+
+// ============================================================================
+// resolveDim — dim resolution priority + cross-dim fallback
+// ============================================================================
+
+func TestResolveDim_AdditionalParamsTakesPrecedence(t *testing.T) {
+ t.Parallel()
+ p := types.RetrieveParams{
+ Embedding: []float32{1, 2, 3, 4}, // would suggest dim=4
+ AdditionalParams: map[string]any{"dim": 768},
+ }
+ dim, multi := resolveDim(p)
+ if dim != 768 || multi {
+ t.Errorf("resolveDim: want (768, false), got (%d, %v)", dim, multi)
+ }
+}
+
+func TestResolveDim_EmbeddingFallback(t *testing.T) {
+ t.Parallel()
+ p := types.RetrieveParams{Embedding: []float32{1, 2, 3}}
+ dim, multi := resolveDim(p)
+ if dim != 3 || multi {
+ t.Errorf("resolveDim: want (3, false), got (%d, %v)", dim, multi)
+ }
+}
+
+func TestResolveDim_NoneTriggersMultiIndex(t *testing.T) {
+ t.Parallel()
+ p := types.RetrieveParams{}
+ dim, multi := resolveDim(p)
+ if dim != 0 || !multi {
+ t.Errorf("resolveDim: want (0, true), got (%d, %v)", dim, multi)
+ }
+}
+
+// ============================================================================
+// inspectBulkResponse — Reason leak guard + total count
+// ============================================================================
+
+func TestInspectBulkResponse_AllSucceeded_NoError(t *testing.T) {
+ t.Parallel()
+ body := `{"errors": false, "items": [{"index": {"_id": "c1", "status": 201}}]}`
+ if err := inspectBulkResponse(strings.NewReader(body)); err != nil {
+ t.Errorf("all-succeeded bulk: want nil, got %v", err)
+ }
+}
+
+func TestInspectBulkResponse_TotalCount(t *testing.T) {
+ t.Parallel()
+ // 7 items, 3 failed. Public error must surface count=3 + up to 5
+ // distinct messages, but never expose the per-item Reason field.
+ body := `{
+ "errors": true,
+ "items": [
+ {"index": {"_id": "c1", "status": 201}},
+ {"index": {"_id": "c2", "status": 400, "error": {"type": "mapper_parsing_exception", "reason": "secret-content-snippet"}}},
+ {"index": {"_id": "c3", "status": 400, "error": {"type": "mapper_parsing_exception", "reason": "another-secret"}}},
+ {"index": {"_id": "c4", "status": 201}},
+ {"index": {"_id": "c5", "status": 400, "error": {"type": "version_conflict_engine_exception", "reason": "another-secret"}}},
+ {"index": {"_id": "c6", "status": 201}},
+ {"index": {"_id": "c7", "status": 201}}
+ ]
+ }`
+ err := inspectBulkResponse(strings.NewReader(body))
+ if err == nil {
+ t.Fatal("partial failure must return non-nil error")
+ }
+ msg := err.Error()
+ if !strings.Contains(msg, "3 items failed") {
+ t.Errorf("error must report total count of failures: %s", msg)
+ }
+ // Leak guard: per-item Reason must NOT leak to public error.
+ if strings.Contains(msg, "secret-content-snippet") || strings.Contains(msg, "another-secret") {
+ t.Errorf("error must NOT leak cluster-side Reason: %s", msg)
+ }
+ // Type (bounded enum) is OK to include.
+ if !strings.Contains(msg, "mapper_parsing_exception") {
+ t.Errorf("error should surface error.type for diagnosis: %s", msg)
+ }
+}
// ============================================================================
// probeVersion — supported-version acceptance/rejection matrix
@@ -813,71 +1172,72 @@ func TestStub_SwapToVersion_ReturnsFeatureNotEnabled(t *testing.T) {
}
// ============================================================================
-// Behavioural-method stubs — a follow-up commit replaces each of these
-// (Save / BatchSave / Retrieve / DeleteBy*) with the real implementation,
-// at which point the corresponding stub-test below disappears. Each stub
-// returns ErrFeatureNotEnabled so any accidental invocation at this stage
-// surfaces loudly.
+// Delete path cap — ErrBatchTooLarge guard
// ============================================================================
-func TestStub_Save_ReturnsFeatureNotEnabled(t *testing.T) {
+func TestDeleteByChunkIDList_OverCap_ReturnsErrBatchTooLarge(t *testing.T) {
t.Parallel()
- r := &Repository{}
- err := r.Save(context.Background(), &types.IndexInfo{ChunkID: "c1"}, nil)
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("Save: want ErrFeatureNotEnabled, got %v", err)
+ r := &Repository{
+ once: make(map[int]*sync.Once),
+ initErr: make(map[int]error),
+ }
+ ids := make([]string, 1001)
+ for i := range ids {
+ ids[i] = fmt.Sprintf("c%d", i)
+ }
+ err := r.DeleteByChunkIDList(context.Background(), ids, 768, "")
+ if !errors.Is(err, ErrBatchTooLarge) {
+ t.Errorf("1001 ids: want ErrBatchTooLarge, got %v", err)
}
}
-func TestStub_BatchSave_ReturnsFeatureNotEnabled(t *testing.T) {
+func TestDeleteByChunkIDList_Empty_NoOp(t *testing.T) {
t.Parallel()
- r := &Repository{}
- err := r.BatchSave(context.Background(),
- []*types.IndexInfo{{ChunkID: "c1"}}, nil)
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("BatchSave: want ErrFeatureNotEnabled, got %v", err)
+ r := &Repository{
+ once: make(map[int]*sync.Once),
+ initErr: make(map[int]error),
+ }
+ if err := r.DeleteByChunkIDList(context.Background(), nil, 768, ""); err != nil {
+ t.Errorf("empty list: want nil, got %v", err)
}
}
-func TestStub_Retrieve_ReturnsFeatureNotEnabled(t *testing.T) {
+// ============================================================================
+// BatchSave pre-marshal cap — ErrBatchTooLarge guard
+// ============================================================================
+
+func TestBatchSave_OverDocCountCap_ReturnsErrBatchTooLarge(t *testing.T) {
t.Parallel()
- r := &Repository{}
- res, err := r.Retrieve(context.Background(), types.RetrieveParams{TopK: 10})
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("Retrieve: want ErrFeatureNotEnabled, got %v", err)
+ r := &Repository{
+ once: make(map[int]*sync.Once),
+ initErr: make(map[int]error),
}
- if res != nil {
- t.Errorf("Retrieve: want nil result, got %v", res)
+ infos := make([]*types.IndexInfo, 1001)
+ embMap := make(map[string][]float32, 1001)
+ for i := range infos {
+ sid := fmt.Sprintf("s%d", i)
+ infos[i] = &types.IndexInfo{SourceID: sid, ChunkID: fmt.Sprintf("c%d", i)}
+ embMap[sid] = []float32{0.1, 0.2}
+ }
+ err := r.BatchSave(context.Background(), infos,
+ map[string]any{"embedding": embMap})
+ if !errors.Is(err, ErrBatchTooLarge) {
+ t.Errorf("1001 infos: want ErrBatchTooLarge, got %v", err)
}
}
-func TestStub_DeleteByChunkIDList_ReturnsFeatureNotEnabled(t *testing.T) {
- t.Parallel()
- r := &Repository{}
- err := r.DeleteByChunkIDList(context.Background(),
- []string{"c1"}, 768, "")
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("DeleteByChunkIDList: want ErrFeatureNotEnabled, got %v", err)
- }
-}
+// ============================================================================
+// Empty BatchSave is a no-op
+// ============================================================================
-func TestStub_DeleteBySourceIDList_ReturnsFeatureNotEnabled(t *testing.T) {
+func TestBatchSave_EmptyList_NoOp(t *testing.T) {
t.Parallel()
- r := &Repository{}
- err := r.DeleteBySourceIDList(context.Background(),
- []string{"s1"}, 768, "")
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("DeleteBySourceIDList: want ErrFeatureNotEnabled, got %v", err)
+ r := &Repository{
+ once: make(map[int]*sync.Once),
+ initErr: make(map[int]error),
}
-}
-
-func TestStub_DeleteByKnowledgeIDList_ReturnsFeatureNotEnabled(t *testing.T) {
- t.Parallel()
- r := &Repository{}
- err := r.DeleteByKnowledgeIDList(context.Background(),
- []string{"k1"}, 768, "")
- if !errors.Is(err, ErrFeatureNotEnabled) {
- t.Errorf("DeleteByKnowledgeIDList: want ErrFeatureNotEnabled, got %v", err)
+ if err := r.BatchSave(context.Background(), nil, nil); err != nil {
+ t.Errorf("empty infos: want nil, got %v", err)
}
}
diff --git a/internal/application/repository/retriever/opensearch/retrieve.go b/internal/application/repository/retriever/opensearch/retrieve.go
new file mode 100644
index 00000000..61101d78
--- /dev/null
+++ b/internal/application/repository/retriever/opensearch/retrieve.go
@@ -0,0 +1,205 @@
+package opensearch
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+
+ osapi "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Retrieve dispatches to vectorRetrieve or keywordsRetrieve based on
+// params.RetrieverType. Returns exactly one *RetrieveResult — the
+// service-layer multi-store dispatcher depends on this one-bundle-per-
+// driver protocol. Score in result is RAW [0, 1] passthrough (the
+// OpenSearch k-NN plugin's SpaceType.COSINESIMIL.scoreTranslation has
+// already mapped to (1 + cos) / 2 before the score reaches us).
+//
+// Dim resolution order:
+// 1. params.AdditionalParams["dim"] if present and positive
+// 2. len(params.Embedding) if positive
+// 3. cross-dim multi-index search for keyword-only paths via
+// "_*" wildcard pattern
+//
+// Vector retrieve always requires a positive dim (it's intrinsic to k-NN).
+func (r *Repository) Retrieve(
+ ctx context.Context,
+ params types.RetrieveParams,
+) ([]*types.RetrieveResult, error) {
+ dim, multiIndex := resolveDim(params)
+
+ switch params.RetrieverType {
+ case types.VectorRetrieverType:
+ if dim == 0 {
+ return nil, fmt.Errorf(
+ "opensearch: vector retrieve requires embedding or AdditionalParams[\"dim\"]: %w",
+ ErrDimensionMismatch)
+ }
+ if err := r.ensureReady(ctx, dim); err != nil {
+ return nil, err
+ }
+ return r.vectorRetrieve(ctx, params, dim)
+ case types.KeywordsRetrieverType:
+ if multiIndex {
+ // BM25-only path without dim — search across "_*"
+ // wildcard pattern. Slower than a single-index search but
+ // correct for admin / keyword-only callers.
+ return r.keywordsRetrieve(ctx, params, r.baseIndex+"_*")
+ }
+ if err := r.ensureReady(ctx, dim); err != nil {
+ return nil, err
+ }
+ return r.keywordsRetrieve(ctx, params, r.indexAlias(dim))
+ default:
+ return nil, fmt.Errorf("opensearch: unsupported retriever type %q", params.RetrieverType)
+ }
+}
+
+// resolveDim returns (dim, multiIndex). multiIndex=true means dim is
+// unknown and the caller should use a wildcard alias pattern (keyword
+// path only — vector retrieve cannot be dim-less).
+func resolveDim(p types.RetrieveParams) (int, bool) {
+ if p.AdditionalParams != nil {
+ if v, ok := p.AdditionalParams["dim"].(int); ok && v > 0 {
+ return v, false
+ }
+ }
+ if len(p.Embedding) > 0 {
+ return len(p.Embedding), false
+ }
+ return 0, true // unknown — multi-index fallback for keyword
+}
+
+func (r *Repository) vectorRetrieve(
+ ctx context.Context, p types.RetrieveParams, dim int,
+) ([]*types.RetrieveResult, error) {
+ body, err := buildKNNQuery(p.Embedding, effectiveTopK(ctx, p), p.Threshold, fromParams(p))
+ if err != nil {
+ return nil, err
+ }
+ hits, err := r.search(ctx, r.indexAlias(dim), body)
+ if err != nil {
+ return nil, err
+ }
+ return wrapResults(ctx, hits, types.VectorRetrieverType, types.MatchTypeEmbedding), nil
+}
+
+func (r *Repository) keywordsRetrieve(
+ ctx context.Context, p types.RetrieveParams, indexPattern string,
+) ([]*types.RetrieveResult, error) {
+ body, err := buildKeywordQuery(p.Query, effectiveTopK(ctx, p), p.Threshold, fromParams(p))
+ if err != nil {
+ return nil, err
+ }
+ hits, err := r.search(ctx, indexPattern, body)
+ if err != nil {
+ return nil, err
+ }
+ return wrapResults(ctx, hits, types.KeywordsRetrieverType, types.MatchTypeKeywords), nil
+}
+
+// effectiveTopK returns the caller's TopK or 10 if unset. The 10 is a
+// defensive default — every real caller sets TopK, so a TopK=10 in
+// production logs indicates a caller bug. Caps at 10000 to stay below
+// OpenSearch's index.max_result_window default.
+func effectiveTopK(ctx context.Context, p types.RetrieveParams) int {
+ if p.TopK <= 0 {
+ logger.GetLogger(ctx).Warnf("[OpenSearch] Retrieve called with TopK<=0; defaulting to 10 (caller bug?)")
+ return 10
+ }
+ if p.TopK > 10000 {
+ return 10000
+ }
+ return p.TopK
+}
+
+func (r *Repository) search(ctx context.Context, indexPattern string, body []byte) ([]hit, error) {
+ req := osapi.SearchReq{Indices: []string{indexPattern}, Body: bytes.NewReader(body)}
+ resp, err := r.client.Search(ctx, &req)
+ if err != nil {
+ if isNotFound(err) {
+ return nil, fmt.Errorf("opensearch: index %s missing: %w", indexPattern, ErrIndexNotFound)
+ }
+ return nil, wrapTransport(err)
+ }
+ defer func() {
+ if resp != nil {
+ drainAndClose(resp.Inspect().Response.Body)
+ }
+ }()
+ // Cap response body at 16MB (search results bounded by max_result_window * doc_size).
+ return parseSearchHits(io.LimitReader(resp.Inspect().Response.Body, 16<<20))
+}
+
+// hit is the structural shape we pull from each OpenSearch search hit.
+// Field-by-field decode (vs map[string]any) keeps the JSON shape
+// pinned at compile time.
+type hit struct {
+ ID string `json:"_id"` // equals chunk_id per the indexing invariant
+ Score float64 `json:"_score"`
+ Source struct {
+ Content string `json:"content"`
+ ChunkID string `json:"chunk_id"`
+ KnowledgeID string `json:"knowledge_id"`
+ KnowledgeBaseID string `json:"knowledge_base_id"`
+ SourceID string `json:"source_id"`
+ SourceType int `json:"source_type"` // integer, not stringified
+ TagID string `json:"tag_id"`
+ IsEnabled bool `json:"is_enabled"`
+ } `json:"_source"`
+}
+
+func parseSearchHits(body io.Reader) ([]hit, error) {
+ var resp struct {
+ Hits struct {
+ Hits []hit `json:"hits"`
+ } `json:"hits"`
+ }
+ if err := json.NewDecoder(body).Decode(&resp); err != nil {
+ return nil, fmt.Errorf("opensearch: parse search response: %w", ErrTransport)
+ }
+ return resp.Hits.Hits, nil
+}
+
+// wrapResults converts OpenSearch hits to the upstream IndexWithScore
+// shape. Always returns exactly one *RetrieveResult — the service-layer
+// multi-store dispatcher depends on this "one engine result per driver"
+// contract.
+//
+// Defensive: warns if hit._id != _source.chunk_id (we set _id=chunk_id
+// on every write, so a mismatch means external tooling has been writing
+// into the index) but does not fail the query — the result is still
+// usable for ranking, just suspicious.
+func wrapResults(ctx context.Context, hits []hit, rt types.RetrieverType, mt types.MatchType) []*types.RetrieveResult {
+ log := logger.GetLogger(ctx)
+ out := make([]*types.IndexWithScore, 0, len(hits))
+ for _, h := range hits {
+ if h.ID != h.Source.ChunkID {
+ log.Warnf("[OpenSearch] hit._id=%q != _source.chunk_id=%q (D12 invariant violation)",
+ h.ID, h.Source.ChunkID)
+ }
+ out = append(out, &types.IndexWithScore{
+ ID: h.ID,
+ ChunkID: h.Source.ChunkID,
+ KnowledgeID: h.Source.KnowledgeID,
+ KnowledgeBaseID: h.Source.KnowledgeBaseID,
+ SourceID: h.Source.SourceID,
+ SourceType: types.SourceType(h.Source.SourceType),
+ TagID: h.Source.TagID,
+ Content: h.Source.Content,
+ Score: h.Score,
+ MatchType: mt,
+ IsEnabled: h.Source.IsEnabled,
+ })
+ }
+ return []*types.RetrieveResult{{
+ Results: out,
+ RetrieverEngineType: types.OpenSearchRetrieverEngineType,
+ RetrieverType: rt,
+ }}
+}
diff --git a/internal/application/repository/retriever/opensearch/stubs.go b/internal/application/repository/retriever/opensearch/stubs.go
index c60a68f7..668ddb23 100644
--- a/internal/application/repository/retriever/opensearch/stubs.go
+++ b/internal/application/repository/retriever/opensearch/stubs.go
@@ -6,70 +6,16 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
-// This file ships every behavioural method of the OpenSearch driver as a
-// stub returning ErrFeatureNotEnabled (or, for EstimateStorageSize, a
-// conservative lower-bound). The package is a hollow, interface-compliant
-// shell at this commit — every real code path is gated dead code (no
-// registry / factory / env path mentions this driver), and any accidental
-// invocation surfaces loudly.
-//
-// Follow-up commits land the real implementations: the read/write methods
-// (Save / BatchSave / DeleteBy* / Retrieve) migrate to dedicated files
-// (crud.go / retrieve.go + supporting query.go); the async / batch paths
-// (CopyIndices, BatchUpdate*) and the real EstimateStorageSize arrive
-// alongside the activation switch.
-
-// Save is replaced by a real bulk-driven single-doc path in a later commit.
-func (r *Repository) Save(
- _ context.Context, _ *types.IndexInfo, _ map[string]any,
-) error {
- return ErrFeatureNotEnabled
-}
-
-// BatchSave is replaced by a real dim-aware pre-marshal + per-item-error
-// inspection path in a later commit.
-func (r *Repository) BatchSave(
- _ context.Context, _ []*types.IndexInfo, _ map[string]any,
-) error {
- return ErrFeatureNotEnabled
-}
-
-// DeleteByChunkIDList is replaced by a capped sync _delete_by_query path
-// in a later commit.
-func (r *Repository) DeleteByChunkIDList(
- _ context.Context, _ []string, _ int, _ string,
-) error {
- return ErrFeatureNotEnabled
-}
-
-// DeleteBySourceIDList is replaced by a capped sync _delete_by_query path
-// in a later commit.
-func (r *Repository) DeleteBySourceIDList(
- _ context.Context, _ []string, _ int, _ string,
-) error {
- return ErrFeatureNotEnabled
-}
-
-// DeleteByKnowledgeIDList is replaced by a capped sync _delete_by_query
-// path in a later commit.
-func (r *Repository) DeleteByKnowledgeIDList(
- _ context.Context, _ []string, _ int, _ string,
-) error {
- return ErrFeatureNotEnabled
-}
-
-// Retrieve is replaced by a dim-resolve + per-dim alias k-NN / BM25 path
-// in a later commit.
-func (r *Repository) Retrieve(
- _ context.Context, _ types.RetrieveParams,
-) ([]*types.RetrieveResult, error) {
- return nil, ErrFeatureNotEnabled
-}
+// This file holds the remaining stubs for methods whose real
+// implementation has not landed yet — the async / batch paths and the
+// rolling-reindex swap. Each stub returns ErrFeatureNotEnabled (or, for
+// EstimateStorageSize, a conservative lower-bound) so any accidental
+// invocation surfaces loudly. The driver as a whole is still gated dead
+// code (no registry / factory / env path mentions it); these stubs
+// disappear when their behaviours arrive in follow-up commits.
// CopyIndices: the async _reindex path with task polling for >10K-doc
-// batches arrives in a later change. Until then this is unreachable in
-// production (no registry registration path activates the OpenSearch
-// driver), but the stub fails closed if reached.
+// batches arrives in a later change.
func (r *Repository) CopyIndices(
_ context.Context,
_ string, // sourceKnowledgeBaseID
@@ -98,15 +44,16 @@ func (r *Repository) BatchUpdateChunkTagID(
return ErrFeatureNotEnabled
}
-// EstimateStorageSize: the real implementation that reads cluster `_stats`
-// for the per-dim alias arrives in a later change. For now we return a
-// conservative lower-bound estimate using the HNSW memory formula so the
-// upstream KB-delete guard fails-closed (treats non-empty KBs as "may free
-// non-trivial storage, force confirmation") rather than failing open.
+// EstimateStorageSize: the real implementation that reads cluster
+// `_stats` for the per-dim alias arrives in a later change. For now we
+// return a conservative lower-bound estimate using the HNSW memory
+// formula so the upstream KB-delete guard fails-closed (treats non-empty
+// KBs as "may free non-trivial storage, force confirmation") rather than
+// failing open.
//
// Formula: N * (1024 content bytes + 4*dimGuess float + 128 HNSW M=16 overhead)
-// dimGuess = 768 (common embedding size; the real implementation reads the
-// actual dim from the cluster).
+// dimGuess = 768 (common embedding size; the real implementation reads
+// the actual dim from the cluster).
func (r *Repository) EstimateStorageSize(
_ context.Context,
indexInfoList []*types.IndexInfo,