From 3e3ffa4b0b4f7c247983d808e1769edbfa92c948 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 24 Mar 2026 11:50:54 +0800 Subject: [PATCH] feat(i18n): add JSON file type support across multiple locales - Introduced support for JSON file types in English, Korean, Russian, and Chinese locale files. - Updated the file type acceptance criteria in the document upload component to include JSON. - Enhanced the knowledge base parser settings to recognize JSON as a valid file type. - Implemented JSON conversion functionality in the document parser, allowing for JSON content to be processed and converted to markdown. - Added comprehensive tests for JSON to markdown conversion to ensure functionality and reliability. --- frontend/src/i18n/locales/en-US.ts | 1 + frontend/src/i18n/locales/ko-KR.ts | 1 + frontend/src/i18n/locales/ru-RU.ts | 1 + frontend/src/i18n/locales/zh-CN.ts | 1 + .../src/views/knowledge/KnowledgeBase.vue | 2 +- .../knowledge/settings/KBParserSettings.vue | 2 + internal/application/service/knowledge.go | 2 +- .../docparser/builtin_converter.go | 9 +- .../docparser/engine_registry.go | 2 +- .../docparser/json_converter.go | 254 +++++++ .../docparser/json_converter_test.go | 705 ++++++++++++++++++ 11 files changed, 976 insertions(+), 4 deletions(-) create mode 100644 internal/infrastructure/docparser/json_converter.go create mode 100644 internal/infrastructure/docparser/json_converter_test.go diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index d06b872d..51fd9afc 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -3016,6 +3016,7 @@ export default { fileTypeExcel: 'Excel Spreadsheets', fileTypeCsv: 'CSV Files', fileTypeText: 'Plain Text', + fileTypeJson: 'JSON Files', fileTypeImage: 'Images', engines: { builtin: { diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 563fdbb5..d1ba39ca 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -3079,6 +3079,7 @@ export default { fileTypeExcel: 'Excel 스프레드시트', fileTypeCsv: 'CSV 파일', fileTypeText: '일반 텍스트', + fileTypeJson: 'JSON 파일', fileTypeImage: '이미지', engines: { builtin: { diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index b2663f2b..4be4f54e 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2689,6 +2689,7 @@ export default { fileTypeExcel: 'Таблицы Excel', fileTypeCsv: 'Файлы CSV', fileTypeText: 'Текстовые файлы', + fileTypeJson: 'Файлы JSON', fileTypeImage: 'Изображения', engines: { builtin: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index fef7cdf9..c9904f7d 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -3014,6 +3014,7 @@ export default { fileTypeExcel: "Excel 表格", fileTypeCsv: "CSV 文件", fileTypeText: "纯文本", + fileTypeJson: "JSON 文件", fileTypeImage: "图片", engines: { builtin: { diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 67789f63..3e021c56 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -1494,7 +1494,7 @@ async function createNewSession(value: string): Promise { ref="uploadInputRef" type="file" class="document-upload-input" - :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'" + :accept="acceptFileTypes || '.pdf,.docx,.doc,.txt,.md,.json,.jpg,.jpeg,.png,.csv,.xlsx,.xls,.pptx,.ppt'" multiple @change="handleDocumentUpload" /> diff --git a/frontend/src/views/knowledge/settings/KBParserSettings.vue b/frontend/src/views/knowledge/settings/KBParserSettings.vue index 0ae33a4e..c737d13f 100644 --- a/frontend/src/views/knowledge/settings/KBParserSettings.vue +++ b/frontend/src/views/knowledge/settings/KBParserSettings.vue @@ -157,6 +157,7 @@ const fileTypeGroups = computed(() => { const csvExts = ['csv'].filter(e => ft.has(e)) const mdExts = ['md', 'markdown'].filter(e => ft.has(e)) const txtExts = ['txt'].filter(e => ft.has(e)) + const jsonExts = ['json'].filter(e => ft.has(e)) const imageExts = ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'webp'].filter(e => ft.has(e)) if (pdfExts.length) groups.push({ key: 'pdf', label: t('kbSettings.parser.fileTypePdf'), icon: 'file-pdf', extensions: pdfExts }) @@ -166,6 +167,7 @@ const fileTypeGroups = computed(() => { if (csvExts.length) groups.push({ key: 'csv', label: t('kbSettings.parser.fileTypeCsv'), icon: 'file-excel', extensions: csvExts }) if (mdExts.length) groups.push({ key: 'markdown', label: 'Markdown', icon: 'file-code', extensions: mdExts }) if (txtExts.length) groups.push({ key: 'text', label: t('kbSettings.parser.fileTypeText'), icon: 'file', extensions: txtExts }) + if (jsonExts.length) groups.push({ key: 'json', label: t('kbSettings.parser.fileTypeJson'), icon: 'file-code', extensions: jsonExts }) if (imageExts.length) groups.push({ key: 'image', label: t('kbSettings.parser.fileTypeImage'), icon: 'image', extensions: imageExts }) return groups diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 0ae76722..a2c18541 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2814,7 +2814,7 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str // isValidFileType checks if a file type is supported func isValidFileType(filename string) bool { switch strings.ToLower(getFileType(filename)) { - case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt": + case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif", "csv", "xlsx", "xls", "pptx", "ppt", "json": return true default: return false diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go index b4469505..70480ee5 100644 --- a/internal/infrastructure/docparser/builtin_converter.go +++ b/internal/infrastructure/docparser/builtin_converter.go @@ -15,7 +15,8 @@ import ( var simpleFormats = map[string]bool{ "md": true, "markdown": true, "txt": true, "text": true, - "csv": true, + "csv": true, + "json": true, } var imageFormats = map[string]bool{ @@ -56,6 +57,12 @@ func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*t return nil, fmt.Errorf("csv conversion failed: %w", err) } return &types.ReadResult{MarkdownContent: md}, nil + case ft == "json": + md, err := jsonToMarkdown(req.FileContent) + if err != nil { + return nil, fmt.Errorf("json conversion failed: %w", err) + } + return &types.ReadResult{MarkdownContent: md}, nil case imageFormats[ft]: return imageToResult(req.FileName, req.FileContent), nil default: diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 43b58fc6..4a5940ee 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -67,7 +67,7 @@ func (e *simpleEngine) Description() string { return "Simple format & image parsing (no external service required)" } func (e *simpleEngine) FileTypes(_ bool) []string { - return []string{"md", "markdown", "txt", "csv", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"} + return []string{"md", "markdown", "txt", "csv", "json", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"} } func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) { return true, "" diff --git a/internal/infrastructure/docparser/json_converter.go b/internal/infrastructure/docparser/json_converter.go new file mode 100644 index 00000000..d600b62d --- /dev/null +++ b/internal/infrastructure/docparser/json_converter.go @@ -0,0 +1,254 @@ +package docparser + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" +) + +// defaultJSONChunkSize is the target chunk size in bytes for JSON semantic +// splitting. Approximates 512 tokens (1 token ≈ 3-4 bytes for mixed content). +const defaultJSONChunkSize = 1536 + +// minJSONChunkSize is the minimum chunk size. A new chunk is only started +// when the current chunk has reached at least this size. +var minJSONChunkSize = defaultJSONChunkSize - 200 + +// jsonToMarkdown converts raw JSON bytes into markdown text +// +// Key properties: +// - Every output chunk is a **valid JSON object** (not a fragment). +// - Nested paths from root to leaf are **fully preserved** in each chunk. +// - Arrays are converted to index-keyed dicts so the algorithm is uniform. +// - Small objects that fit within maxChunkSize are kept intact (not split). +// - The output is a series of fenced ```json code blocks separated by \n\n, +// which the downstream text chunker can split at block boundaries. +func jsonToMarkdown(data []byte) (string, error) { + data = trimBOM(data) + if len(data) == 0 { + return "", fmt.Errorf("empty JSON content") + } + if !json.Valid(data) { + return "", fmt.Errorf("invalid JSON content") + } + + var parsed interface{} + if err := json.Unmarshal(data, &parsed); err != nil { + return "", fmt.Errorf("failed to parse JSON: %w", err) + } + + // Normalize: convert top-level arrays to index-keyed dicts + normalized := listToDictPreprocess(parsed) + + // If the whole thing fits in one chunk, just format it + wholeSize := jsonSize(normalized) + if wholeSize <= defaultJSONChunkSize { + formatted := formatValue(normalized) + return wrapCodeBlock(formatted), nil + } + + // Recursive split + chunks := recursiveJSONSplit(normalized, nil, nil) + + // Convert each chunk dict to a fenced code block + blocks := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + if len(chunk) == 0 { + continue + } + blocks = append(blocks, wrapCodeBlock(formatValue(chunk))) + } + + if len(blocks) == 0 { + return wrapCodeBlock(formatValue(normalized)), nil + } + return strings.Join(blocks, "\n\n"), nil +} + +// --------------------------------------------------------------------------- +// RecursiveJsonSplitter core algorithm +// --------------------------------------------------------------------------- + +// recursiveJSONSplit splits a JSON dict into a list of JSON dicts, +// each fitting within defaultJSONChunkSize. It preserves the full nested +// path from root to each leaf by using setNestedDict. +// +// This is a Go port of LangChain's RecursiveJsonSplitter._json_split. +func recursiveJSONSplit( + data interface{}, + currentPath []string, + chunks []map[string]interface{}, +) []map[string]interface{} { + if chunks == nil { + chunks = []map[string]interface{}{{}} + } + + dict, ok := data.(map[string]interface{}) + if !ok { + // Scalar or already-processed value: place it at the current path + if len(currentPath) > 0 && len(chunks) > 0 { + setNestedDict(chunks[len(chunks)-1], currentPath, data) + } + return chunks + } + + // Sort keys for deterministic output + keys := sortedKeys(dict) + + for _, key := range keys { + value := dict[key] + newPath := append(append([]string{}, currentPath...), key) + + // Measure sizes + chunkSize := jsonSize(chunks[len(chunks)-1]) + itemSize := jsonSize(map[string]interface{}{key: value}) + remaining := defaultJSONChunkSize - chunkSize + + if itemSize <= remaining { + // Item fits in the current chunk — add it preserving the path + setNestedDict(chunks[len(chunks)-1], newPath, value) + } else { + // Item doesn't fit + if chunkSize >= minJSONChunkSize { + // Current chunk is big enough, start a new one + chunks = append(chunks, map[string]interface{}{}) + } + + // Check if the value itself is a dict/list that can be recursed into + normalized := listToDictPreprocess(value) + if subDict, isDict := normalized.(map[string]interface{}); isDict && canSplitDict(subDict) { + // Recurse into the sub-object + chunks = recursiveJSONSplit(subDict, newPath, chunks) + } else { + // Cannot split further (scalar or single-key dict) — place as-is + setNestedDict(chunks[len(chunks)-1], newPath, value) + } + } + } + + return chunks +} + +// setNestedDict sets a value in a nested dict structure, creating +// intermediate dicts as needed. This preserves the full JSON path. +// +// Example: setNestedDict(d, ["config","db","host"], "localhost") +// produces: {"config": {"db": {"host": "localhost"}}} +func setNestedDict(d map[string]interface{}, path []string, value interface{}) { + if len(path) == 0 { + return + } + current := d + for _, key := range path[:len(path)-1] { + next, ok := current[key] + if !ok { + next = map[string]interface{}{} + current[key] = next + } + if nextDict, ok := next.(map[string]interface{}); ok { + current = nextDict + } else { + // Path conflict (existing value is not a dict) — overwrite + newDict := map[string]interface{}{} + current[key] = newDict + current = newDict + } + } + current[path[len(path)-1]] = value +} + +// listToDictPreprocess recursively converts JSON arrays to index-keyed +// dicts so the splitter can treat everything uniformly. +// +// Example: ["a","b","c"] → {"0":"a", "1":"b", "2":"c"} +func listToDictPreprocess(data interface{}) interface{} { + switch v := data.(type) { + case map[string]interface{}: + result := make(map[string]interface{}, len(v)) + for k, val := range v { + result[k] = listToDictPreprocess(val) + } + return result + case []interface{}: + result := make(map[string]interface{}, len(v)) + for i, item := range v { + result[fmt.Sprintf("%d", i)] = listToDictPreprocess(item) + } + return result + default: + return data + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// jsonSize returns the serialized JSON byte length of a value. +func jsonSize(v interface{}) int { + b, err := json.Marshal(v) + if err != nil { + return 0 + } + return len(b) +} + +// formatValue formats a JSON value with indentation. +func formatValue(v interface{}) string { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + b, _ = json.Marshal(v) + } + return string(b) +} + +// indentJSON formats raw JSON bytes with indentation. +func indentJSON(data []byte) (string, error) { + var buf bytes.Buffer + if err := json.Indent(&buf, data, "", " "); err != nil { + return string(data), err + } + return buf.String(), nil +} + +// wrapCodeBlock wraps content in a fenced JSON code block. +func wrapCodeBlock(content string) string { + return "```json\n" + content + "\n```" +} + +// trimBOM removes a UTF-8 BOM prefix if present. +func trimBOM(data []byte) []byte { + if len(data) >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF { + return data[3:] + } + return data +} + +// canSplitDict returns true if a dict can be meaningfully split. +// A dict with multiple keys can be split by distributing keys across chunks. +// A dict with a single key can still be split if its value is a splittable dict. +func canSplitDict(d map[string]interface{}) bool { + if len(d) > 1 { + return true + } + if len(d) == 1 { + for _, v := range d { + if sub, ok := v.(map[string]interface{}); ok && len(sub) > 1 { + return true + } + } + } + return false +} + +// sortedKeys returns the keys of a map in sorted order. +func sortedKeys(m map[string]interface{}) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/infrastructure/docparser/json_converter_test.go b/internal/infrastructure/docparser/json_converter_test.go new file mode 100644 index 00000000..4527dca3 --- /dev/null +++ b/internal/infrastructure/docparser/json_converter_test.go @@ -0,0 +1,705 @@ +package docparser + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// ═══════════════════════════════════════════════════════════════════════════ +// Helper: visual test reporter +// ═══════════════════════════════════════════════════════════════════════════ + +func visualReport(t *testing.T, label string, input string, result string, err error, checks []checkResult) { + t.Helper() + + const width = 72 + bar := strings.Repeat("─", width) + thickBar := strings.Repeat("━", width) + + t.Logf("\n%s", thickBar) + t.Logf("📋 TEST: %s", label) + t.Logf("%s", bar) + + // ── Input ── + inputPreview := input + if len(inputPreview) > 200 { + inputPreview = inputPreview[:200] + fmt.Sprintf("... (%d bytes total)", len(input)) + } + t.Logf("📥 INPUT (%d bytes):", len(input)) + for _, line := range strings.Split(inputPreview, "\n") { + t.Logf(" %s", line) + } + t.Logf("%s", bar) + + // ── Output / Error ── + if err != nil { + t.Logf("❌ ERROR: %v", err) + } else { + blockCount := strings.Count(result, "```json") + totalLen := len(result) + t.Logf("📤 OUTPUT: %d code block(s), %d bytes total", blockCount, totalLen) + t.Logf("%s", bar) + + blocks := splitBlocks(result) + for i, blk := range blocks { + preview := blk + lines := strings.Split(preview, "\n") + if len(lines) > 10 { + preview = strings.Join(lines[:4], "\n") + + fmt.Sprintf("\n ... (%d lines omitted) ...\n", len(lines)-8) + + strings.Join(lines[len(lines)-4:], "\n") + } + t.Logf(" 📦 Block #%d (%d bytes):", i+1, len(blk)) + for _, line := range strings.Split(preview, "\n") { + t.Logf(" │ %s", line) + } + } + } + t.Logf("%s", bar) + + // ── Checks ── + allPass := true + for _, c := range checks { + icon := "✅" + if !c.pass { + icon = "💥" + allPass = false + } + t.Logf("%s %s", icon, c.desc) + if !c.pass { + t.Errorf("FAIL: %s", c.desc) + } + } + if allPass { + t.Logf("🎉 ALL CHECKS PASSED") + } + t.Logf("%s\n", thickBar) +} + +// splitBlocks extracts the content between ```json and ``` fences. +func splitBlocks(result string) []string { + var blocks []string + rest := result + for { + start := strings.Index(rest, "```json\n") + if start < 0 { + break + } + rest = rest[start+len("```json\n"):] + end := strings.Index(rest, "\n```") + if end < 0 { + blocks = append(blocks, rest) + break + } + blocks = append(blocks, rest[:end]) + rest = rest[end+len("\n```"):] + } + return blocks +} + +type checkResult struct { + desc string + pass bool +} + +func check(desc string, pass bool) checkResult { return checkResult{desc, pass} } + +// blockIsValidJSON checks that each code block inside the result is valid JSON. +func allBlocksValidJSON(result string) bool { + for _, blk := range splitBlocks(result) { + if !json.Valid([]byte(blk)) { + return false + } + } + return true +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 1: Small inputs — kept intact, NOT split +// ═══════════════════════════════════════════════════════════════════════════ + +func TestJsonToMarkdown_SmallObject(t *testing.T) { + input := `{"name": "test", "version": "1.0"}` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Small Object → single block, kept intact", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains key 'name'", strings.Contains(result, `"name"`)), + check("contains key 'version'", strings.Contains(result, `"version"`)), + check(fmt.Sprintf("exactly 1 block (got %d)", blocks), blocks == 1), + check("both keys in SAME block (not split apart)", + strings.Contains(result, `"name"`) && strings.Contains(result, `"version"`) && blocks == 1), + }) +} + +func TestJsonToMarkdown_SmallArray(t *testing.T) { + input := `[{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Small Array → single block, kept intact", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("exactly 1 block (got %d)", blocks), blocks == 1), + }) +} + +func TestJsonToMarkdown_EmptyObject(t *testing.T) { + input := `{}` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Empty Object {}", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains '{}'", strings.Contains(result, "{}")), + }) +} + +func TestJsonToMarkdown_EmptyArray(t *testing.T) { + input := `[]` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Empty Array []", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains '{}'", strings.Contains(result, "{}")), // [] → {} after list-to-dict + }) +} + +func TestJsonToMarkdown_PrimitiveString(t *testing.T) { + input := `"hello world"` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Primitive String", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains 'hello world'", strings.Contains(result, "hello world")), + }) +} + +func TestJsonToMarkdown_PrimitiveNumber(t *testing.T) { + input := `42` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Primitive Number", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains '42'", strings.Contains(result, "42")), + }) +} + +func TestJsonToMarkdown_PrimitiveBoolean(t *testing.T) { + input := `true` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Primitive Boolean", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains 'true'", strings.Contains(result, "true")), + }) +} + +func TestJsonToMarkdown_PrimitiveNull(t *testing.T) { + input := `null` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Primitive Null", input, result, err, []checkResult{ + check("no error", err == nil), + check("contains 'null'", strings.Contains(result, "null")), + }) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 2: Large data — smart recursive splitting +// ═══════════════════════════════════════════════════════════════════════════ + +func TestJsonToMarkdown_LargeObject(t *testing.T) { + obj := make(map[string]interface{}) + for i := 0; i < 50; i++ { + obj[fmt.Sprintf("key_%02d", i)] = strings.Repeat("value", 50) + } + data, _ := json.Marshal(obj) + + result, err := jsonToMarkdown(data) + blocks := strings.Count(result, "```json") + visualReport(t, "Large Object (50 keys) → multiple blocks", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("multiple blocks (got %d)", blocks), blocks >= 2), + }) +} + +func TestJsonToMarkdown_LargeArray(t *testing.T) { + arr := make([]interface{}, 100) + for i := range arr { + arr[i] = map[string]interface{}{ + "id": i, + "name": strings.Repeat("name", 20), + "description": strings.Repeat("desc", 30), + } + } + data, _ := json.Marshal(arr) + + result, err := jsonToMarkdown(data) + blocks := strings.Count(result, "```json") + visualReport(t, "Large Array (100 elements) → multiple blocks", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("multiple blocks (got %d)", blocks), blocks >= 2), + }) +} + +func TestJsonToMarkdown_RecursiveSplitLargeKey(t *testing.T) { + inner := make(map[string]interface{}) + for i := 0; i < 30; i++ { + inner[fmt.Sprintf("sub_%02d", i)] = strings.Repeat("val", 50) + } + obj := map[string]interface{}{"bigkey": inner} + data, _ := json.Marshal(obj) + + result, err := jsonToMarkdown(data) + blocks := strings.Count(result, "```json") + visualReport(t, "Recursive Split: 1 big key with 30 sub-keys", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("multiple blocks (got %d)", blocks), blocks >= 2), + check("path preserved: chunks contain 'bigkey'", strings.Contains(result, "bigkey")), + }) +} + +func TestJsonToMarkdown_PathPreservation(t *testing.T) { + // Core test: verify that nested paths are preserved in each chunk + inner := make(map[string]interface{}) + for i := 0; i < 40; i++ { + inner[fmt.Sprintf("field_%02d", i)] = strings.Repeat("data", 60) + } + obj := map[string]interface{}{ + "config": map[string]interface{}{ + "database": inner, + }, + } + data, _ := json.Marshal(obj) + + result, err := jsonToMarkdown(data) + blocks := splitBlocks(result) + // Every block should contain the full path "config" → "database" + allHavePath := true + for _, blk := range blocks { + if !strings.Contains(blk, `"config"`) || !strings.Contains(blk, `"database"`) { + allHavePath = false + break + } + } + visualReport(t, "Path Preservation: config.database.* across chunks", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("multiple blocks (got %d)", len(blocks)), len(blocks) >= 2), + check("ALL blocks preserve path: config → database", allHavePath), + }) +} + +func TestJsonToMarkdown_MixedArrayElements(t *testing.T) { + arr := []interface{}{ + map[string]interface{}{"id": 1}, + map[string]interface{}{"id": 2}, + map[string]interface{}{"id": 3, "data": strings.Repeat("x", 2000)}, + map[string]interface{}{"id": 4}, + } + data, _ := json.Marshal(arr) + + result, err := jsonToMarkdown(data) + blocks := strings.Count(result, "```json") + visualReport(t, "Mixed Array: 3 small + 1 large element", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("at least 2 blocks (got %d)", blocks), blocks >= 2), + }) +} + +func TestJsonToMarkdown_DeepNested(t *testing.T) { + input := `{"l1": {"l2": {"l3": {"data": "` + strings.Repeat("x", 2000) + `"}}}}` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Deep Nested (3 levels, large leaf)", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("block count ≥ 1 (got %d)", blocks), blocks >= 1), + }) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 3: Error Handling +// ═══════════════════════════════════════════════════════════════════════════ + +func TestJsonToMarkdown_InvalidJSON(t *testing.T) { + input := `{invalid json}` + _, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Invalid JSON → error", input, "", err, []checkResult{ + check("returns error", err != nil), + check("error mentions 'invalid JSON'", + err != nil && strings.Contains(err.Error(), "invalid JSON")), + }) +} + +func TestJsonToMarkdown_EmptyInput(t *testing.T) { + input := "" + _, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Empty Input → error", input, "", err, []checkResult{ + check("returns error", err != nil), + check("error mentions 'empty'", + err != nil && strings.Contains(err.Error(), "empty")), + }) +} + +func TestJsonToMarkdown_WhitespaceOnly(t *testing.T) { + input := " \n\t " + _, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Whitespace-Only Input → error", input, "", err, []checkResult{ + check("returns error", err != nil), + }) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 4: Edge Cases & Encoding +// ═══════════════════════════════════════════════════════════════════════════ + +func TestJsonToMarkdown_BOM(t *testing.T) { + raw := append([]byte{0xEF, 0xBB, 0xBF}, []byte(`{"key": "value"}`)...) + result, err := jsonToMarkdown(raw) + visualReport(t, "UTF-8 BOM prefix → stripped", string(raw), result, err, []checkResult{ + check("no error", err == nil), + check("contains key after BOM removal", strings.Contains(result, `"key"`)), + }) +} + +func TestJsonToMarkdown_UnicodeContent(t *testing.T) { + input := `{"名称": "WeKnora 知识库", "描述": "支持中文 JSON 🎉", "emoji": "🚀"}` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Unicode / Chinese / Emoji", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains Chinese", strings.Contains(result, "知识库")), + check("contains emoji", strings.Contains(result, "🚀")), + }) +} + +func TestJsonToMarkdown_SpecialCharsInStrings(t *testing.T) { + input := `{"html": "
hello
", "path": "C:\\Users\\test", "newline": "line1\nline2"}` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Special characters (HTML, backslash, newline)", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + }) +} + +func TestJsonToMarkdown_NullValues(t *testing.T) { + input := `{"name": "test", "value": null, "list": [null, 1, null]}` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Object with null values", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains 'null'", strings.Contains(result, "null")), + }) +} + +func TestJsonToMarkdown_MixedValueTypes(t *testing.T) { + input := `{"str": "hello", "num": 42, "float": 3.14, "bool": true, "null_val": null, "arr": [1,2], "obj": {"a":1}}` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Mixed value types in one object", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains string", strings.Contains(result, "hello")), + check("contains number", strings.Contains(result, "42")), + check("contains float", strings.Contains(result, "3.14")), + }) +} + +func TestJsonToMarkdown_NestedArrayOfArrays(t *testing.T) { + input := `[[1, 2, 3], [4, 5, 6], [7, 8, 9]]` + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Nested Array of Arrays (matrix)", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + }) +} + +func TestJsonToMarkdown_LargePrimitiveString(t *testing.T) { + bigStr := strings.Repeat("abcdefghij", 300) + input := fmt.Sprintf(`"%s"`, bigStr) + result, err := jsonToMarkdown([]byte(input)) + visualReport(t, "Large Primitive String (3000 chars) → fallback", input, result, err, []checkResult{ + check("no error", err == nil), + check("content preserved", strings.Contains(result, "abcdefghij")), + }) +} + +func TestJsonToMarkdown_SingleKeyObject(t *testing.T) { + input := `{"only_key": "only_value"}` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Single-Key Object", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("exactly 1 block (got %d)", blocks), blocks == 1), + }) +} + +func TestJsonToMarkdown_SingleElementArray(t *testing.T) { + input := `[{"id": 1, "name": "solo"}]` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Single-Element Array", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("exactly 1 block (got %d)", blocks), blocks == 1), + }) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 5: Realistic Scenarios +// ═══════════════════════════════════════════════════════════════════════════ + +func TestJsonToMarkdown_RealisticConfig(t *testing.T) { + input := `{ + "app": {"name": "WeKnora", "version": "2.0.0", "debug": false}, + "database": {"host": "localhost", "port": 5432, "name": "weknora_db", "pool_size": 10}, + "redis": {"host": "localhost", "port": 6379}, + "features": {"json_upload": true, "multimodel": true, "graph_extraction": false} +}` + result, err := jsonToMarkdown([]byte(input)) + blocks := strings.Count(result, "```json") + visualReport(t, "Realistic: App Config (fits in 1 block)", input, result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains 'WeKnora'", strings.Contains(result, "WeKnora")), + check("contains 'database'", strings.Contains(result, "database")), + check(fmt.Sprintf("single block (fits, got %d)", blocks), blocks == 1), + }) +} + +func TestJsonToMarkdown_RealisticAPIResponse(t *testing.T) { + users := make([]interface{}, 40) + for i := range users { + users[i] = map[string]interface{}{ + "id": i + 1, + "username": fmt.Sprintf("user_%03d", i+1), + "email": fmt.Sprintf("user%03d@example.com", i+1), + "role": "editor", + "active": i%3 != 0, + } + } + apiResp := map[string]interface{}{ + "status": "ok", + "total": 40, + "page": 1, + "data": users, + } + data, _ := json.Marshal(apiResp) + + result, err := jsonToMarkdown(data) + blocks := strings.Count(result, "```json") + visualReport(t, "Realistic: API Response (40 user records)", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check("contains user data", strings.Contains(result, "user_001")), + check(fmt.Sprintf("multiple blocks (got %d)", blocks), blocks >= 2), + }) +} + +func TestJsonToMarkdown_RealisticLargeNestedConfig(t *testing.T) { + // A config where one section is huge and must be recursively split, + // while other sections are small + rules := make(map[string]interface{}) + for i := 0; i < 30; i++ { + rules[fmt.Sprintf("rule_%02d", i)] = map[string]interface{}{ + "pattern": strings.Repeat("pattern", 10), + "action": "allow", + "priority": i, + } + } + config := map[string]interface{}{ + "version": "3.0", + "metadata": map[string]interface{}{"author": "admin", "updated": "2026-03-24"}, + "firewall": map[string]interface{}{ + "enabled": true, + "rules": rules, + }, + } + data, _ := json.Marshal(config) + + result, err := jsonToMarkdown(data) + blocks := splitBlocks(result) + // Each block should still have the path "firewall" → "rules" where applicable + visualReport(t, "Realistic: Nested config with 30 firewall rules", string(data), result, err, []checkResult{ + check("no error", err == nil), + check("all blocks are valid JSON", allBlocksValidJSON(result)), + check(fmt.Sprintf("multiple blocks needed (got %d)", len(blocks)), len(blocks) >= 2), + check("contains 'firewall'", strings.Contains(result, "firewall")), + check("contains 'version'", strings.Contains(result, "version")), + }) +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 6: Integration — SimpleFormatReader +// ═══════════════════════════════════════════════════════════════════════════ + +func TestSimpleFormatReader_JSON(t *testing.T) { + reader := &SimpleFormatReader{} + input := `{"test": "integration", "count": 42}` + req := &types.ReadRequest{ + FileName: "config.json", + FileType: "json", + FileContent: []byte(input), + } + + result, err := reader.Read(context.Background(), req) + + var resultStr string + if result != nil { + resultStr = result.MarkdownContent + } + visualReport(t, "SimpleFormatReader.Read() with .json file", input, resultStr, err, []checkResult{ + check("no error", err == nil), + check("result not nil", result != nil), + check("markdown contains 'test'", + result != nil && strings.Contains(result.MarkdownContent, "test")), + check("markdown has code block", + result != nil && strings.Contains(result.MarkdownContent, "```json")), + check("output is valid JSON inside block", + result != nil && allBlocksValidJSON(result.MarkdownContent)), + }) +} + +func TestSimpleFormatReader_JSON_Invalid(t *testing.T) { + reader := &SimpleFormatReader{} + req := &types.ReadRequest{ + FileName: "bad.json", + FileType: "json", + FileContent: []byte(`{not valid json}`), + } + + _, err := reader.Read(context.Background(), req) + visualReport(t, "SimpleFormatReader with invalid JSON → error", `{not valid json}`, "", err, []checkResult{ + check("returns error", err != nil), + check("error mentions 'json'", + err != nil && strings.Contains(strings.ToLower(err.Error()), "json")), + }) +} + +func TestIsSimpleFormat_JSON(t *testing.T) { + cases := []struct { + input string + want bool + }{ + {"json", true}, {"JSON", true}, {".json", true}, {"Json", true}, + {"txt", true}, {"csv", true}, + {"pdf", false}, {"docx", false}, + } + + t.Logf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + t.Logf("📋 TEST: IsSimpleFormat() recognizes JSON") + t.Logf("─────────────────────────────────────────────────────────") + allPass := true + for _, tc := range cases { + got := IsSimpleFormat(tc.input) + icon := "✅" + if got != tc.want { + icon = "💥" + allPass = false + t.Errorf("IsSimpleFormat(%q) = %v, want %v", tc.input, got, tc.want) + } + t.Logf("%s IsSimpleFormat(%q) = %v (want %v)", icon, tc.input, got, tc.want) + } + if allPass { + t.Logf("🎉 ALL CHECKS PASSED") + } + t.Logf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n") +} + +// ═══════════════════════════════════════════════════════════════════════════ +// Group 7: list-to-dict preprocessing +// ═══════════════════════════════════════════════════════════════════════════ + +func TestListToDictPreprocess(t *testing.T) { + // Simulate what json.Unmarshal produces: numbers become float64 + input := []interface{}{"a", "b", map[string]interface{}{"nested": []interface{}{float64(1), float64(2)}}} + result := listToDictPreprocess(input) + + dict, ok := result.(map[string]interface{}) + if !ok { + t.Fatalf("expected map, got %T", result) + } + + t.Logf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + t.Logf("📋 TEST: listToDictPreprocess") + t.Logf("─────────────────────────────────────────────────────────") + t.Logf("📥 INPUT: [\"a\", \"b\", {\"nested\": [1, 2]}]") + t.Logf("📤 OUTPUT: %s", formatValue(dict)) + t.Logf("─────────────────────────────────────────────────────────") + + checks := []checkResult{ + check("top level is dict", ok), + check(`key "0" = "a"`, dict["0"] == "a"), + check(`key "1" = "b"`, dict["1"] == "b"), + } + // Check nested array also converted + if nested, ok2 := dict["2"].(map[string]interface{}); ok2 { + if innerNested, ok3 := nested["nested"].(map[string]interface{}); ok3 { + checks = append(checks, check(`nested array converted to dict`, innerNested["0"] == float64(1))) + } else { + checks = append(checks, check(`nested array converted to dict`, false)) + } + } + allPass := true + for _, c := range checks { + icon := "✅" + if !c.pass { + icon = "💥" + allPass = false + t.Errorf("FAIL: %s", c.desc) + } + t.Logf("%s %s", icon, c.desc) + } + if allPass { + t.Logf("🎉 ALL CHECKS PASSED") + } + t.Logf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n") +} + +func TestSetNestedDict(t *testing.T) { + d := map[string]interface{}{} + setNestedDict(d, []string{"config", "database", "host"}, "localhost") + + t.Logf("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + t.Logf("📋 TEST: setNestedDict path preservation") + t.Logf("─────────────────────────────────────────────────────────") + t.Logf("📥 path: [\"config\", \"database\", \"host\"]") + t.Logf("📥 value: \"localhost\"") + t.Logf("📤 result: %s", formatValue(d)) + t.Logf("─────────────────────────────────────────────────────────") + + // Verify the structure + config, ok1 := d["config"].(map[string]interface{}) + db, ok2 := map[string]interface{}{}, false + host := "" + if ok1 { + db, ok2 = config["database"].(map[string]interface{}) + } + if ok2 { + host, _ = db["host"].(string) + } + + checks := []checkResult{ + check(`d["config"] is dict`, ok1), + check(`d["config"]["database"] is dict`, ok2), + check(`d["config"]["database"]["host"] = "localhost"`, host == "localhost"), + } + allPass := true + for _, c := range checks { + icon := "✅" + if !c.pass { + icon = "💥" + allPass = false + t.Errorf("FAIL: %s", c.desc) + } + t.Logf("%s %s", icon, c.desc) + } + if allPass { + t.Logf("🎉 ALL CHECKS PASSED") + } + t.Logf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n") +}