package librarysnapshot import ( "encoding/json" "errors" "io" "sort" ) // IndexDocument 是归档索引部件里的一条原始文档(含向量)。 // 生产同步直接搬运它,不重新解析、不重新 Embedding。 type IndexDocument struct { ID string `json:"id"` Source json.RawMessage `json:"source"` } // ReadRowIDs 读出行数组里每行的 id 并排序,用于"身份集合"比对。 func ReadRowIDs(r io.Reader) ([]string, error) { decoder := json.NewDecoder(r) token, err := decoder.Token() if err != nil || token != json.Delim('[') { return nil, errors.New("snapshot component must be a JSON row array") } ids := make([]string, 0, 64) for decoder.More() { var row struct { ID string `json:"id"` } if err := decoder.Decode(&row); err != nil { return nil, errors.New("invalid archived row") } if row.ID == "" { return nil, errors.New("archived row without id") } ids = append(ids, row.ID) } sort.Strings(ids) return ids, nil } // ReadIndexDocuments 读取索引部件。归档部件有两种历史形状:直接是文档数组, // 或 [{id, source}] 包装;两种都必须支持,否则旧归档无法同步。 func ReadIndexDocuments(r io.Reader) ([]IndexDocument, error) { decoder := json.NewDecoder(r) token, err := decoder.Token() if err != nil || token != json.Delim('[') { return nil, errors.New("index component must be a JSON array") } documents := make([]IndexDocument, 0, 64) for decoder.More() { var row map[string]json.RawMessage if err := decoder.Decode(&row); err != nil { return nil, errors.New("invalid archived index row") } // 历史归档有两种键风格:导出结构体的 "ID"/"Source",以及小写 "id"/"source"。 wrapped, hasSource := row["source"] if !hasSource { wrapped, hasSource = row["Source"] } if hasSource { idRaw, ok := row["id"] if !ok { idRaw = row["ID"] } var id string _ = json.Unmarshal(idRaw, &id) documents = append(documents, IndexDocument{ID: id, Source: wrapped}) continue } id := documentIdentity(row) if id == "" { return nil, errors.New("archived index row has no stable id") } raw, err := json.Marshal(row) if err != nil { return nil, err } documents = append(documents, IndexDocument{ID: id, Source: raw}) } if len(documents) == 0 { return documents, nil } for _, document := range documents { if document.ID == "" || len(document.Source) == 0 { return nil, errors.New("archived index document incomplete") } } sort.Slice(documents, func(i, j int) bool { return documents[i].ID < documents[j].ID }) return documents, nil } func documentIdentity(row map[string]json.RawMessage) string { for _, key := range []string{"chunk_id", "source_id", "id"} { if raw, ok := row[key]; ok { var value string if json.Unmarshal(raw, &value) == nil && value != "" { return value } } } return "" }