feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing. Optional odl-hybrid sidecar stays local-build only and is excluded from default dev-start and full profiles.
This commit is contained in:
@@ -3256,6 +3256,10 @@ func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileTyp
|
||||
return docparser.NewMinerUReader(overrides)
|
||||
case "mineru_cloud":
|
||||
return docparser.NewMinerUCloudReader(overrides)
|
||||
case "paddleocr_vl":
|
||||
return docparser.NewPaddleOCRVLReader(overrides)
|
||||
case "paddleocr_vl_cloud":
|
||||
return docparser.NewPaddleOCRVLCloudReader(overrides)
|
||||
case "builtin":
|
||||
// 明确指定使用 builtin 引擎(docreader),不使用 simple format 兜底
|
||||
return s.documentReader
|
||||
|
||||
@@ -87,6 +87,10 @@ type settingSpec struct {
|
||||
// Description is shown in the UI under the key. Stored on the row
|
||||
// at first write (mirrors Category).
|
||||
Description string
|
||||
// RequiresRestart marks keys whose value is bound at process startup
|
||||
// (e.g. asynq worker pool size). The UI shows a restart badge; the
|
||||
// service persists the flag on first write.
|
||||
RequiresRestart bool
|
||||
}
|
||||
|
||||
// registry pins the set of legal keys. Expanding it is a deliberate,
|
||||
@@ -161,6 +165,20 @@ var registry = map[string]settingSpec{
|
||||
"仅在创建时读取,修改后只对之后新建的租户生效,不会回写已存在的租户。" +
|
||||
"0 或负数表示使用内置默认值 10GB。",
|
||||
},
|
||||
// asynq.concurrency is the asynq worker pool size (parallel in-flight
|
||||
// tasks). Read once when the asynq server starts — changing it in the
|
||||
// UI requires a process restart to take effect. Mirrors
|
||||
// WEKNORA_ASYNQ_CONCURRENCY (default 16).
|
||||
"asynq.concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
|
||||
Default: int64(16),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "异步任务 worker 并发数(asynq 线程池大小)。" +
|
||||
"文档解析、嵌入等任务多为 I/O 等待,适当提高可缩短批量上传排队时间。" +
|
||||
"修改后需重启服务进程方可生效。",
|
||||
},
|
||||
}
|
||||
|
||||
// systemSettingService wires the repository, audit log, and (P2)
|
||||
@@ -655,7 +673,7 @@ func (s *systemSettingService) virtualSetting(key string, spec settingSpec) *typ
|
||||
Category: category,
|
||||
Description: spec.Description,
|
||||
IsSecret: false,
|
||||
RequiresRestart: false,
|
||||
RequiresRestart: spec.RequiresRestart,
|
||||
LastModifiedBy: "",
|
||||
Enum: spec.Enum,
|
||||
}
|
||||
@@ -817,6 +835,7 @@ func (s *systemSettingService) Update(ctx context.Context, key string, rawValue
|
||||
category = "general"
|
||||
}
|
||||
description = spec.Description
|
||||
requiresRestart = spec.RequiresRestart
|
||||
}
|
||||
|
||||
row := &types.SystemSetting{
|
||||
@@ -1142,6 +1161,14 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
|
||||
// 400 body verbatim).
|
||||
func validateRegistryEntry(key string, rawValue any) error {
|
||||
switch key {
|
||||
case "asynq.concurrency":
|
||||
n, err := coerceToPositiveInt64(rawValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n <= 0 {
|
||||
return errors.New("concurrency must be a positive integer")
|
||||
}
|
||||
case "ssrf.whitelist":
|
||||
// Coerce into the same shape encodeForType produced. We don't
|
||||
// look at the encoded JSON because that's already canonicalised
|
||||
@@ -1155,6 +1182,23 @@ func validateRegistryEntry(key string, rawValue any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// coerceToPositiveInt64 accepts int / int64 / float64 from JSON decoding.
|
||||
func coerceToPositiveInt64(rawValue any) (int64, error) {
|
||||
switch v := rawValue.(type) {
|
||||
case int:
|
||||
return int64(v), nil
|
||||
case int64:
|
||||
return v, nil
|
||||
case float64:
|
||||
if v != float64(int64(v)) {
|
||||
return 0, errors.New("expected integer value")
|
||||
}
|
||||
return int64(v), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("expected integer, got %T", rawValue)
|
||||
}
|
||||
}
|
||||
|
||||
// coerceToStringSlice mirrors the input shapes accepted by
|
||||
// encodeForType for "string_list": []any of strings, []string, or a
|
||||
// comma-separated string. Returns the trimmed, empty-stripped result.
|
||||
|
||||
Reference in New Issue
Block a user