Merge pull request #949 from peterchan2020/feat_oss

Feat: Add Alibaba Cloud Storage Configuration
This commit is contained in:
lyingbug
2026-04-14 11:08:56 +08:00
committed by GitHub
28 changed files with 1075 additions and 31 deletions
+1 -1
View File
@@ -181,7 +181,7 @@ Fully modular pipeline from document parsing, vectorization, and retrieval to LL
| LLMs | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
| Embeddings | Ollama / BGE / GTE / OpenAI-compatible APIs |
| Vector DBs | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS |
| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS |
| IM Channels | WeCom / Feishu / Slack / Telegram / DingTalk / Mattermost |
| Web Search | DuckDuckGo / Bing / Google / Tavily |
+1 -1
View File
@@ -180,7 +180,7 @@ WeKnora 提供**快速问答**与**智能推理**两种问答模式。快速问
|------|------|
| 模型厂商 | OpenAI / DeepSeek / Qwen(阿里云)/ 智谱 / 混元 / 豆包(火山引擎)/ Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
| 向量数据库 | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 |
| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 / 阿里云 OSS |
| IM 集成 | 企业微信 / 飞书 / Slack / Telegram / 钉钉 / Mattermost |
| 网络搜索 | DuckDuckGo / Bing / Google / Tavily |
+1 -1
View File
@@ -182,7 +182,7 @@ Feishuなどの外部プラットフォームからのナレッジ自動同期
| 大規模モデル | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
| Embedding | Ollama / BGE / GTE / OpenAI 互換 API |
| ベクトル DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS |
| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS / Alibaba Cloud OSS |
| IM 統合 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost |
| Web 検索 | DuckDuckGo / Bing / Google / Tavily |
+1 -1
View File
@@ -191,7 +191,7 @@ Feishu 등 외부 플랫폼에서 지식 자동 동기화를 지원하며(추가
| LLM | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
| Embedding | Ollama / BGE / GTE / OpenAI 호환 API |
| 벡터 DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS |
| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS |
| IM 통합 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost |
| 웹 검색 | DuckDuckGo / Bing / Google / Tavily |
+24
View File
@@ -126,6 +126,16 @@ DocReader 支持多种存储后端:
- `COS_PATH_PREFIX`: 文件路径前缀
- `COS_ENABLE_OLD_DOMAIN`: 是否使用旧域名(默认:true)
#### 阿里云 OSS 存储
- `STORAGE_TYPE`: 设置为 `oss`
- `OSS_ACCESS_KEY_ID`: OSS 访问密钥 ID
- `OSS_ACCESS_KEY_SECRET`: OSS 访问密钥
- `OSS_ENDPOINT`: OSS 端点(如 `oss-cn-hangzhou.aliyuncs.com`)
- `OSS_BUCKET_NAME`: OSS 存储桶名称
- `OSS_REGION`: OSS 区域(如 `cn-hangzhou`)
- `OSS_PATH_PREFIX`: 文件路径前缀
### 代理配置
如果需要通过代理访问外部服务:
@@ -178,6 +188,20 @@ docreader:
- MAX_FILE_SIZE_MB=50
```
### 使用阿里云 OSS
```yaml
docreader:
environment:
- STORAGE_TYPE=oss
- OSS_ACCESS_KEY_ID=your_access_key_id
- OSS_ACCESS_KEY_SECRET=your_access_key_secret
- OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
- OSS_BUCKET_NAME=your-bucket
- OSS_REGION=cn-hangzhou
- MAX_FILE_SIZE_MB=50
```
## 常见问题
### 1. DocReader 服务无法启动?
+101 -1
View File
@@ -10,6 +10,11 @@ from typing import Dict, Optional
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
try:
import oss2
except ImportError:
oss2 = None # type: ignore[assignment]
from docreader.utils import endecode
logger = logging.getLogger(__name__)
@@ -232,6 +237,99 @@ class MinioStorage(Storage):
return ""
class OssStorage(Storage):
"""Alibaba Cloud OSS storage implementation"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.bucket, self.endpoint, self.prefix = self._init_oss_bucket()
def _init_oss_bucket(self):
if oss2 is None:
logger.error("oss2 package not installed. Install with: pip install oss2")
return None, None, None
try:
sc = self.storage_config
access_key = _cfg(sc, "access_key", "OSS_ACCESS_KEY")
secret_key = _cfg(sc, "secret_key", "OSS_SECRET_KEY")
endpoint_raw = _cfg(sc, "endpoint", "OSS_ENDPOINT")
bucket_name = _cfg(sc, "bucket_name", "OSS_BUCKET_NAME")
prefix_raw = _cfg(sc, "path_prefix", "OSS_PATH_PREFIX")
if not all([access_key, secret_key, endpoint_raw, bucket_name]):
logger.error(
"Incomplete OSS configuration: "
"access_key=%s, endpoint=%s, bucket=%s",
bool(access_key), endpoint_raw, bucket_name,
)
return None, None, None
# oss2 requires endpoint to include the scheme (http:// or https://)
if not endpoint_raw.startswith(("http://", "https://")):
endpoint_raw = f"https://{endpoint_raw}"
auth = oss2.Auth(access_key, secret_key)
bucket = oss2.Bucket(auth, endpoint_raw, bucket_name)
# Verify bucket exists; do NOT auto-create to avoid unintended public-read buckets
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
logger.error(
"OSS bucket %s does not exist. Please create it manually in the console.",
bucket_name,
)
return None, None, None
prefix = prefix_raw.strip().strip("/") if prefix_raw else ""
return bucket, endpoint_raw, prefix
except Exception as e:
logger.error("Failed to initialize OSS client: %s", e)
return None, None, None
def _get_download_url(self, object_key: str) -> str:
# Use virtual-hosted style URL: https://{bucket}.{endpoint_without_scheme}/{key}
endpoint_no_scheme = self.endpoint.removeprefix("https://").removeprefix("http://")
return f"https://{self.bucket.bucket_name}.{endpoint_no_scheme}/{object_key}"
def upload_file(self, file_path: str) -> str:
try:
if not self.bucket:
return ""
file_ext = os.path.splitext(file_path)[1]
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
self.bucket.put_object_from_file(object_key, file_path)
file_url = self._get_download_url(object_key)
logger.info("OSS upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error("OSS upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
try:
if not self.bucket:
return ""
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
self.bucket.put_object(object_key, content)
file_url = self._get_download_url(object_key)
logger.info("OSS upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error("OSS upload_bytes failed: %s", e)
traceback.print_exc()
return ""
class LocalStorage(Storage):
"""Local file system storage implementation.
@@ -297,7 +395,7 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
"""Create a storage instance based on storage_config dict.
The ``provider`` key in storage_config determines the backend:
minio, cos, local, base64.
minio, cos, oss, local, base64.
Falls back to STORAGE_TYPE env var, then ``local``.
"""
storage_type = ""
@@ -315,6 +413,8 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
return MinioStorage(storage_config)
elif storage_type == "cos":
return CosStorage(storage_config)
elif storage_type == "oss":
return OssStorage(storage_config)
elif storage_type == "local":
return LocalStorage(storage_config)
elif storage_type == "base64":
+1
View File
@@ -21,6 +21,7 @@ dependencies = [
"mistletoe>=1.5.0",
"ollama>=0.6.0",
"openai>=2.7.1",
"oss2>=2.18.0",
"paddleocr>=2.10.0,<3.0.0",
"paddlepaddle>=3.0.0,<4.0.0",
"pdfplumber>=0.11.7",
+48 -6
View File
@@ -1,5 +1,5 @@
version = 1
revision = 2
revision = 3
requires-python = ">=3.10.18"
resolution-markers = [
"python_full_version >= '3.14' and sys_platform == 'darwin'",
@@ -59,6 +59,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8e/64/013409c451a44b61310fb757af4527f3de57fc98a00f40448de28b864290/albumentations-2.0.8-py3-none-any.whl", hash = "sha256:c4c4259aaf04a7386ad85c7fdcb73c6c7146ca3057446b745cc035805acb1017", size = 369423, upload-time = "2025-05-27T21:23:15.609Z" },
]
[[package]]
name = "aliyun-python-sdk-core"
version = "2.16.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cryptography" },
{ name = "jmespath" },
]
sdist = { url = "https://files.pythonhosted.org/packages/3e/09/da9f58eb38b4fdb97ba6523274fbf445ef6a06be64b433693da8307b4bec/aliyun-python-sdk-core-2.16.0.tar.gz", hash = "sha256:651caad597eb39d4fad6cf85133dffe92837d53bdf62db9d8f37dab6508bb8f9", size = 449555, upload-time = "2024-10-09T06:01:01.762Z" }
[[package]]
name = "aliyun-python-sdk-kms"
version = "2.16.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aliyun-python-sdk-core" },
]
sdist = { url = "https://files.pythonhosted.org/packages/a8/2c/9877d0e6b18ecf246df671ac65a5d1d9fecbf85bdcb5d43efbde0d4662eb/aliyun-python-sdk-kms-2.16.5.tar.gz", hash = "sha256:f328a8a19d83ecbb965ffce0ec1e9930755216d104638cd95ecd362753b813b3", size = 12018, upload-time = "2024-08-30T09:01:20.104Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/11/5c/0132193d7da2c735669a1ed103b142fd63c9455984d48c5a88a1a516efaa/aliyun_python_sdk_kms-2.16.5-py2.py3-none-any.whl", hash = "sha256:24b6cdc4fd161d2942619479c8d050c63ea9cd22b044fe33b60bbb60153786f0", size = 99495, upload-time = "2024-08-30T09:01:18.462Z" },
]
[[package]]
name = "annotated-types"
version = "0.7.0"
@@ -690,6 +712,7 @@ dependencies = [
{ name = "mistletoe" },
{ name = "ollama" },
{ name = "openai" },
{ name = "oss2" },
{ name = "paddleocr" },
{ name = "paddlepaddle" },
{ name = "pdfplumber" },
@@ -724,6 +747,7 @@ requires-dist = [
{ name = "mistletoe", specifier = ">=1.5.0" },
{ name = "ollama", specifier = ">=0.6.0" },
{ name = "openai", specifier = ">=2.7.1" },
{ name = "oss2", specifier = ">=2.18.0" },
{ name = "paddleocr", specifier = ">=2.10.0,<3.0.0" },
{ name = "paddlepaddle", specifier = ">=3.0.0,<4.0.0" },
{ name = "pdfplumber", specifier = ">=0.11.7" },
@@ -1322,6 +1346,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d9/71/71408b02c6133153336d29fa3ba53000f1e1a3f78bb2fc2d1a1865d2e743/jiter-0.11.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18c77aaa9117510d5bdc6a946baf21b1f0cfa58ef04d31c8d016f206f2118960", size = 343697, upload-time = "2025-10-17T11:31:13.773Z" },
]
[[package]]
name = "jmespath"
version = "0.10.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/3c/56/3f325b1eef9791759784aa5046a8f6a1aff8f7c898a2e34506771d3b99d8/jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", size = 21607, upload-time = "2020-05-12T22:03:47.267Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f", size = 24489, upload-time = "2020-05-12T22:03:45.643Z" },
]
[[package]]
name = "joblib"
version = "1.5.2"
@@ -1984,21 +2017,16 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" },
{ url = "https://files.pythonhosted.org/packages/db/db/81bf3d7cecfbfed9092b6b4052e857a769d62ed90561b410014e0aae18db/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:b28740f4ecef1738ea8f807461dd541b8287d5650b5be33bca7b474e3cbd1f36", size = 19153079, upload-time = "2025-10-27T23:05:57.686Z" },
{ url = "https://files.pythonhosted.org/packages/2e/4d/a382452b17cf70a2313153c520ea4c96ab670c996cb3a95cc5d5ac7bfdac/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f7d1fe034090a1e371b7f3ca9d3ccae2fabae8c1d8844fb7371d1ea38e8e8d2", size = 15219883, upload-time = "2025-10-22T03:46:21.66Z" },
{ url = "https://files.pythonhosted.org/packages/fb/56/179bf90679984c85b417664c26aae4f427cba7514bd2d65c43b181b7b08b/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ca88747e708e5c67337b0f65eed4b7d0dd70d22ac332038c9fc4635760018f7", size = 17370357, upload-time = "2025-10-22T03:46:57.968Z" },
{ url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113, upload-time = "2025-10-22T03:47:33.526Z" },
{ url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857, upload-time = "2025-10-22T03:46:37.578Z" },
{ url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095, upload-time = "2025-10-22T03:46:24.769Z" },
{ url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080, upload-time = "2025-10-22T03:47:00.265Z" },
{ url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929, upload-time = "2025-10-22T03:47:36.24Z" },
{ url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" },
{ url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915, upload-time = "2025-10-22T03:46:27.773Z" },
{ url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649, upload-time = "2025-10-22T03:47:02.782Z" },
{ url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337, upload-time = "2025-10-22T03:46:35.168Z" },
{ url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" },
{ url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898, upload-time = "2025-10-22T03:46:30.039Z" },
{ url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518, upload-time = "2025-10-22T03:47:05.407Z" },
{ url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610, upload-time = "2025-10-22T03:46:32.239Z" },
{ url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" },
]
[[package]]
@@ -2099,6 +2127,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/19/404708a7e54ad2798907210462fd950c3442ea51acc8790f3da48d2bee8b/opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147", size = 65486, upload-time = "2020-07-19T22:40:30.301Z" },
]
[[package]]
name = "oss2"
version = "2.19.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aliyun-python-sdk-core" },
{ name = "aliyun-python-sdk-kms" },
{ name = "crcmod" },
{ name = "pycryptodome" },
{ name = "requests" },
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/df/b5/f2cb1950dda46ac2284d6c950489fdacd0e743c2d79a347924d3cc44b86f/oss2-2.19.1.tar.gz", hash = "sha256:a8ab9ee7eb99e88a7e1382edc6ea641d219d585a7e074e3776e9dec9473e59c1", size = 298845, upload-time = "2024-10-25T11:37:46.638Z" }
[[package]]
name = "packaging"
version = "25.0"
+1 -1
View File
@@ -458,7 +458,7 @@ curl --location 'http://localhost:8080/api/v1/agents/placeholders' \
|------|------|--------|------|
| `image_upload_enabled` | bool | false | 是否允许上传图片 |
| `vlm_model_id` | string | - | 图片分析所用的 VLM 模型 ID |
| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`,为空使用全局默认 |
| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`/`oss`,为空使用全局默认 |
### FAQ 策略设置
+5
View File
@@ -155,6 +155,11 @@ curl --location 'http://localhost:8080/api/v1/system/storage-engine-status' \
"name": "s3",
"available": false,
"description": "AWS S3 对象存储"
},
{
"name": "oss",
"available": false,
"description": "阿里云 OSS 对象存储"
}
],
"minio_env_available": true
+30 -5
View File
@@ -429,7 +429,7 @@ definitions:
type: integer
image_storage_provider:
description: |-
Storage provider for image uploads: "local", "minio", "cos", "tos"
Storage provider for image uploads: "local", "minio", "cos", "tos", "s3", "oss"
Empty means use the global/tenant default provider.
type: string
image_upload_enabled:
@@ -2067,12 +2067,14 @@ definitions:
cos:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.COSEngineConfig'
default_provider:
description: '"local", "minio", "cos", "tos", "s3"'
description: '"local", "minio", "cos", "tos", "s3", "oss"'
type: string
local:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.LocalEngineConfig'
minio:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig'
oss:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig'
s3:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig'
tos:
@@ -2081,7 +2083,7 @@ definitions:
github_com_Tencent_WeKnora_internal_types.StorageProviderConfig:
properties:
provider:
description: '"local", "minio", "cos", "tos"'
description: '"local", "minio", "cos", "tos", "s3", "oss"'
type: string
type: object
github_com_Tencent_WeKnora_internal_types.SubmitJoinRequestRequest:
@@ -2169,6 +2171,27 @@ definitions:
secret_key:
type: string
type: object
github_com_Tencent_WeKnora_internal_types.OSSEngineConfig:
properties:
access_key:
type: string
bucket_name:
type: string
endpoint:
type: string
path_prefix:
type: string
region:
type: string
secret_key:
type: string
temp_bucket_name:
type: string
temp_region:
type: string
use_temp_bucket:
type: boolean
type: object
github_com_Tencent_WeKnora_internal_types.Tenant:
properties:
agent_config:
@@ -2671,8 +2694,10 @@ definitions:
minio:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig'
provider:
description: '"minio", "cos", "tos", or "s3"'
description: '"minio", "cos", "tos", "s3", or "oss"'
type: string
oss:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig'
s3:
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig'
tos:
@@ -2696,7 +2721,7 @@ definitions:
description: short description for UI
type: string
name:
description: '"local", "minio", "cos", "tos"'
description: '"local", "minio", "cos", "tos", "s3", "oss"'
type: string
type: object
internal_handler.TextRelationExtractionRequest:
+14 -2
View File
@@ -183,7 +183,7 @@ export function reconnectDocReader(addr: string): Promise<ParserEnginesResponse
// ---- 存储引擎配置(租户级,供文档/图片存储与 docreader 使用) ----
export interface StorageEngineConfig {
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3"
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3" | "oss"
local?: { path_prefix: string }
minio?: { mode: string; endpoint: string; access_key_id: string; secret_access_key: string; bucket_name: string; use_ssl: boolean; path_prefix: string }
cos?: {
@@ -210,6 +210,17 @@ export interface StorageEngineConfig {
bucket_name: string
path_prefix: string
}
oss?: {
endpoint: string
region: string
access_key: string
secret_key: string
bucket_name: string
path_prefix: string
use_temp_bucket: boolean
temp_bucket_name: string
temp_region: string
}
}
export interface StorageEngineStatusItem {
@@ -236,11 +247,12 @@ export function getStorageEngineStatus(): Promise<{ data: GetStorageEngineStatus
}
export interface StorageCheckRequest {
provider: string // "minio" | "cos" | "tos" | "s3"
provider: string // "minio" | "cos" | "tos" | "s3" | "oss"
minio?: StorageEngineConfig['minio']
cos?: StorageEngineConfig['cos']
tos?: StorageEngineConfig['tos']
s3?: StorageEngineConfig['s3']
oss?: StorageEngineConfig['oss']
}
export interface StorageCheckResponse {
+8
View File
@@ -659,6 +659,7 @@ export default {
engineLocal: 'Local',
engineCos: 'Tencent Cloud COS',
engineTos: 'Volcengine TOS',
engineOss: 'Alibaba Cloud OSS',
engineS3: 'AWS S3',
localTitle: 'Local Storage',
localDesc: 'Store files on the server local filesystem, suitable for single-node deployment only.',
@@ -692,6 +693,10 @@ export default {
s3Desc: 'AWS S3 and S3-compatible object storage services, suitable for public cloud deployment.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Alibaba Cloud Object Storage Service (OSS), suitable for public cloud deployment.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
console: 'Console',
docs: 'Docs',
testConnection: 'Test Connection',
@@ -703,6 +708,7 @@ export default {
requestFailed: 'Request failed',
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
},
},
webSearchSettings: {
@@ -3117,6 +3123,8 @@ export default {
engineTosDesc: 'Volcengine object storage, for public cloud deployment',
engineS3: 'AWS S3',
engineS3Desc: 'AWS S3 and compatible storage, for public cloud deployment',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Alibaba Cloud Object Storage Service, for public cloud deployment',
},
parser: {
title: 'Parser Engine',
+8
View File
@@ -516,6 +516,7 @@ export default {
engineLocal: 'Local(로컬)',
engineCos: 'Tencent Cloud COS',
engineTos: 'Volcengine TOS',
engineOss: 'Alibaba Cloud OSS',
engineS3: 'AWS S3',
localTitle: 'Local(로컬 스토리지)',
localDesc: '서버 로컬 파일시스템에 파일을 저장합니다. 단일 노드 배포에만 적합합니다.',
@@ -549,6 +550,10 @@ export default {
s3Desc: 'AWS S3 및 호환 오브젝트 스토리지 서비스, 퍼블릭 클라우드 배포에 적합합니다.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Alibaba Cloud 오브젝트 스토리지 서비스(OSS), 퍼블릭 클라우드 배포에 적합합니다.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
console: '콘솔',
docs: '문서',
testConnection: '연결 테스트',
@@ -560,6 +565,7 @@ export default {
requestFailed: '요청 실패',
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
},
},
webSearchSettings: {
@@ -3172,6 +3178,8 @@ export default {
engineTosDesc: 'Volcengine 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
engineS3: 'AWS S3',
engineS3Desc: 'AWS S3 및 호환 스토리지, 퍼블릭 클라우드 배포에 적합',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Alibaba Cloud 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
},
parser: {
title: '파서 엔진',
+9 -1
View File
@@ -640,6 +640,7 @@ export default {
engineLocal: 'Локальное',
engineCos: 'Tencent Cloud COS',
engineTos: 'Volcengine TOS',
engineOss: 'Alibaba Cloud OSS',
engineS3: 'AWS S3',
localTitle: 'Локальное хранилище',
localDesc: 'Хранение файлов в локальной файловой системе сервера. Подходит только для однонодового развёртывания.',
@@ -673,6 +674,10 @@ export default {
s3Desc: 'AWS S3 и совместимые сервисы объектного хранилища для публичного облака.',
s3AccessKeyPlaceholder: 'AWS Access Key',
s3SecretKeyPlaceholder: 'AWS Secret Key',
ossTitle: 'Alibaba Cloud OSS',
ossDesc: 'Объектное хранилище Alibaba Cloud (OSS), подходит для публичного облака.',
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
console: 'Консоль',
docs: 'Документация',
testConnection: 'Тест подключения',
@@ -683,7 +688,8 @@ export default {
unknownError: 'Неизвестная ошибка',
requestFailed: 'Ошибка запроса',
cos: 'Tencent Cloud COS',
tos: 'Volcengine TOS'
tos: 'Volcengine TOS',
oss: 'Alibaba Cloud OSS',
}
},
webSearchSettings: {
@@ -2787,6 +2793,8 @@ export default {
engineTosDesc: 'Объектное хранилище Volcengine, для публичного облака',
engineS3: 'AWS S3',
engineS3Desc: 'AWS S3 и совместимые хранилища, для публичного облака',
engineOss: 'Alibaba Cloud OSS',
engineOssDesc: 'Объектное хранилище Alibaba Cloud, для публичного облака',
},
parser: {
title: 'Парсер',
+8
View File
@@ -513,6 +513,7 @@ export default {
engineLocal: "Local(本地)",
engineCos: "腾讯云 COS",
engineTos: "火山引擎 TOS",
engineOss: "阿里云 OSS",
engineS3: "AWS S3",
localTitle: "Local(本地存储)",
localDesc: "使用服务器本地文件系统存储文件,仅适合单机部署。",
@@ -546,6 +547,10 @@ export default {
s3Desc: "AWS S3 及兼容的对象存储服务,适合公有云部署。",
s3AccessKeyPlaceholder: "AWS Access Key",
s3SecretKeyPlaceholder: "AWS Secret Key",
ossTitle: "阿里云 OSS",
ossDesc: "阿里云对象存储服务(OSS),适合公有云部署。",
ossAccessKeyPlaceholder: "阿里云 Access Key",
ossSecretKeyPlaceholder: "阿里云 Secret Key",
console: "控制台",
docs: "文档",
testConnection: "测试连接",
@@ -557,6 +562,7 @@ export default {
requestFailed: "请求失败",
cos: "腾讯云 COS",
tos: "火山引擎 TOS",
oss: "阿里云 OSS",
},
},
webSearchSettings: {
@@ -3115,6 +3121,8 @@ export default {
engineTosDesc: "火山引擎对象存储,适合公有云部署",
engineS3: "AWS S3",
engineS3Desc: "AWS S3 及兼容存储,适合公有云部署",
engineOss: "阿里云 OSS",
engineOssDesc: "阿里云对象存储,适合公有云部署",
},
parser: {
title: "解析引擎",
+5 -5
View File
@@ -41,7 +41,7 @@ const DOMPurifyConfig = {
'display', 'pointer-events', 'cursor', 'data-emit', 'direction'
],
// 允许的协议
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3|oss):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
// 禁止的标签和属性
FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'],
FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'],
@@ -119,7 +119,7 @@ function protectProviderImageSrcInHTML(html: string): string {
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"');
return html.replace(
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3):(?:\/\/|&#x2f;&#x2f;|&#47;&#47;)([^"']+)\2([^>]*)>/gi,
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss):(?:\/\/|&#x2f;&#x2f;|&#47;&#47;)([^"']+)\2([^>]*)>/gi,
(_m, before, quote, provider, restPathRaw, after) => {
const restPath = decodeProviderURL(restPathRaw);
const protectedSrc = `${provider}://${restPath}`;
@@ -173,7 +173,7 @@ export function isValidURL(url: string): boolean {
}
// 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL
if (/^(local|minio|cos|tos|s3):\/\/\S+$/i.test(trimmed)) {
if (/^(local|minio|cos|tos|s3|oss):\/\/\S+$/i.test(trimmed)) {
return true;
}
@@ -299,7 +299,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
}
const images = root.querySelectorAll<HTMLImageElement>(
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"]',
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"]',
);
if (!images.length) {
return;
@@ -319,7 +319,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
}
img.dataset.authHydrated = '1';
const isProviderScheme = /^(local|minio|cos|tos|s3):\/\//.test(sourceURL);
const isProviderScheme = /^(local|minio|cos|tos|s3|oss):\/\//.test(sourceURL);
const requestURL = isProviderScheme
? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}`
: sourceURL;
@@ -1266,6 +1266,7 @@ const imageStorageOptions = computed(() => {
{ value: 'cos', label: t('settings.storage.engineCos'), disabled: statusMap.cos === false },
{ value: 'tos', label: t('settings.storage.engineTos'), disabled: statusMap.tos === false },
{ value: 's3', label: 'Amazon S3', disabled: statusMap.s3 === false },
{ value: 'oss', label: t('settings.storage.engineOss'), disabled: statusMap.oss === false },
];
});
@@ -114,6 +114,13 @@ const engineOptions = computed(() => {
available: statusMap.s3,
disabled: statusMap.s3 === false,
},
{
value: 'oss',
label: t('kbSettings.storage.engineOss'),
desc: t('kbSettings.storage.engineOssDesc'),
available: statusMap.oss,
disabled: statusMap.oss === false,
},
]
})
+1
View File
@@ -201,6 +201,7 @@ const navItems = computed(() => [
{ key: 'cos', label: t('settings.storage.cos') },
{ key: 'tos', label: t('settings.storage.tos') },
{ key: 's3', label: 'AWS S3' },
{ key: 'oss', label: t('settings.storage.oss') },
]
},
{ key: 'mcp', icon: 'tools', label: t('settings.mcpService') },
@@ -34,6 +34,7 @@
<t-option value="cos" :label="$t('settings.storage.engineCos')" />
<t-option value="tos" :label="$t('settings.storage.engineTos')" />
<t-option value="s3" label="AWS S3" />
<t-option value="oss" :label="$t('settings.storage.engineOss')" />
</t-select>
</div>
</div>
@@ -422,6 +423,80 @@
</div>
</div>
<!-- OSS -->
<div class="engine-section" data-model-type="oss">
<div class="engine-header">
<div class="engine-header-info">
<div class="engine-title-row">
<h3>{{ $t('settings.storage.ossTitle') }}</h3>
<t-tag theme="success" variant="light" size="small">{{ $t('settings.storage.configurable') }}</t-tag>
</div>
<p>
{{ $t('settings.storage.ossDesc') }}
<a class="engine-link" href="https://oss.console.aliyun.com/" target="_blank" rel="noopener">{{ $t('settings.storage.console') }} ↗</a>
<a class="engine-link" href="https://help.aliyun.com/zh/oss/" target="_blank" rel="noopener">{{ $t('settings.storage.docs') }} ↗</a>
</p>
</div>
</div>
<div class="engine-form">
<div class="form-field">
<label>Endpoint</label>
<t-input
v-model="config.oss.endpoint"
placeholder="e.g. https://oss-cn-hangzhou.aliyuncs.com"
clearable
/>
</div>
<div class="form-field">
<label>Region</label>
<t-input
v-model="config.oss.region"
placeholder="e.g. cn-hangzhou"
clearable
/>
</div>
<div class="form-field">
<label>Access Key</label>
<t-input
v-model="config.oss.access_key"
:placeholder="$t('settings.storage.ossAccessKeyPlaceholder')"
clearable
/>
</div>
<div class="form-field">
<label>Secret Key</label>
<t-input
v-model="config.oss.secret_key"
type="password"
:placeholder="$t('settings.storage.ossSecretKeyPlaceholder')"
clearable
/>
</div>
<div class="form-field">
<label>{{ $t('settings.storage.bucketName') }}</label>
<t-input
v-model="config.oss.bucket_name"
:placeholder="$t('settings.storage.bucketPlaceholder')"
clearable
/>
</div>
<div class="form-field">
<label>{{ $t('settings.storage.pathPrefix') }}</label>
<t-input
v-model="config.oss.path_prefix"
:placeholder="$t('settings.storage.prefixPlaceholder')"
clearable
/>
</div>
</div>
<div class="test-bar">
<t-button size="small" variant="outline" :loading="checkingOss" @click="onCheckOss">{{ $t('settings.storage.testConnection') }}</t-button>
<span v-if="ossCheckResult" :class="['test-msg', ossCheckResult.ok ? 'success' : 'error']">
{{ ossCheckResult.message }}
</span>
</div>
</div>
<!-- Save -->
<div class="save-bar">
<t-button theme="primary" :loading="saving" @click="onSave">{{ $t('settings.storage.saveConfig') }}</t-button>
@@ -476,6 +551,17 @@ const defaultConfig = (): StorageEngineConfig => ({
bucket_name: '',
path_prefix: '',
},
oss: {
endpoint: '',
region: '',
access_key: '',
secret_key: '',
bucket_name: '',
path_prefix: '',
use_temp_bucket: false,
temp_bucket_name: '',
temp_region: '',
},
})
const loading = ref(true)
@@ -501,6 +587,8 @@ const checkingTos = ref(false)
const tosCheckResult = ref<{ ok: boolean; message: string } | null>(null)
const checkingS3 = ref(false)
const s3CheckResult = ref<{ ok: boolean; message: string } | null>(null)
const checkingOss = ref(false)
const ossCheckResult = ref<{ ok: boolean; message: string } | null>(null)
const minioAvailable = computed(() => {
if (config.value.minio?.mode === 'remote') {
@@ -558,6 +646,19 @@ async function loadConfig() {
path_prefix: d.s3.path_prefix || '',
}
: defaultConfig().s3!,
oss: d.oss
? {
endpoint: d.oss.endpoint || '',
region: d.oss.region || '',
access_key: d.oss.access_key || '',
secret_key: d.oss.secret_key || '',
bucket_name: d.oss.bucket_name || '',
path_prefix: d.oss.path_prefix || '',
use_temp_bucket: d.oss.use_temp_bucket ?? false,
temp_bucket_name: d.oss.temp_bucket_name || '',
temp_region: d.oss.temp_region || '',
}
: defaultConfig().oss!,
}
}
} catch {
@@ -649,6 +750,18 @@ function buildPayload(): StorageEngineConfig {
bucket_name: (config.value.s3?.bucket_name || '').trim(),
path_prefix: (config.value.s3?.path_prefix || '').trim(),
},
oss: {
endpoint: (config.value.oss?.endpoint || '').trim(),
region: (config.value.oss?.region || '').trim(),
access_key: (config.value.oss?.access_key || '').trim(),
secret_key: (config.value.oss?.secret_key || '').trim(),
bucket_name: (config.value.oss?.bucket_name || '').trim(),
path_prefix: (config.value.oss?.path_prefix || '').trim(),
// Temp bucket fields: not exposed in UI; server manages these independently
use_temp_bucket: config.value.oss?.use_temp_bucket ?? false,
temp_bucket_name: (config.value.oss?.temp_bucket_name || '').trim(),
temp_region: (config.value.oss?.temp_region || '').trim(),
},
}
}
@@ -728,6 +841,20 @@ async function onCheckS3() {
}
}
async function onCheckOss() {
checkingOss.value = true
ossCheckResult.value = null
try {
const payload = buildPayload()
const res = await checkStorageEngine({ provider: 'oss', oss: payload.oss })
ossCheckResult.value = res?.data ?? { ok: false, message: t('settings.storage.unknownError') }
} catch (e: unknown) {
ossCheckResult.value = { ok: false, message: e instanceof Error ? e.message : t('settings.storage.requestFailed') }
} finally {
checkingOss.value = false
}
}
onMounted(loadAll)
</script>
@@ -872,7 +999,7 @@ onMounted(loadAll)
label {
font-size: 13px;
font-weight: 500;
color: var(--td-text-color-secondary)555;
color: var(--td-text-color-secondary);
}
&--inline {
@@ -101,6 +101,30 @@ func NewFileServiceFromStorageConfig(
svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix)
return svc, p, err
case "oss":
if sec == nil || sec.OSS == nil || sec.OSS.Endpoint == "" || sec.OSS.Region == "" || sec.OSS.AccessKey == "" || sec.OSS.SecretKey == "" || sec.OSS.BucketName == "" {
return nil, p, fmt.Errorf("incomplete oss config")
}
pathPrefix := strings.TrimSpace(sec.OSS.PathPrefix)
if pathPrefix == "" {
pathPrefix = "weknora/"
}
var svc interfaces.FileService
var err error
if sec.OSS.UseTempBucket && sec.OSS.TempBucketName != "" {
svc, err = NewOssFileServiceWithTempBucket(
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
sec.OSS.BucketName, pathPrefix,
sec.OSS.TempBucketName, sec.OSS.TempRegion,
)
} else {
svc, err = NewOssFileService(
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
sec.OSS.BucketName, pathPrefix,
)
}
return svc, p, err
default:
return nil, p, fmt.Errorf("unsupported provider %q", p)
}
+331
View File
@@ -0,0 +1,331 @@
package file
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
)
// ossFileService implements the FileService interface for Aliyun OSS
// using the S3-compatible protocol with virtual-hosted style addressing.
type ossFileService struct {
client *s3.Client
tempClient *s3.Client
pathPrefix string
bucketName string
tempBucketName string
}
const ossScheme = "oss://"
// newOSSClient creates a bare s3.Client configured for OSS S3-compatible mode.
// OSS uses virtual-hosted style addressing and does not support aws-chunked encoding.
func newOSSClient(endpoint, region, accessKey, secretKey string) (*s3.Client, error) {
cfg, err := config.LoadDefaultConfig(context.Background(),
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
// Disable automatic aws-chunked encoding — OSS does not support it.
config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
)
if err != nil {
return nil, fmt.Errorf("failed to load AWS config: %w", err)
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = false
})
return client, nil
}
// ossBucketExists checks if the bucket exists using the provided client.
func ossBucketExists(ctx context.Context, client *s3.Client, bucketName string) (bool, error) {
_, err := client.HeadBucket(ctx, &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
})
if err != nil {
var notFound *types.NotFound
if errors.As(err, &notFound) {
return false, nil
}
return false, err
}
return true, nil
}
// ossCreateBucket creates a new bucket using the provided client.
func ossCreateBucket(ctx context.Context, client *s3.Client, bucketName string) error {
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
return err
}
// ossEnsureBucket checks if the bucket exists and creates it if missing.
func ossEnsureBucket(ctx context.Context, client *s3.Client, bucketName string) error {
exists, err := ossBucketExists(ctx, client, bucketName)
if err != nil {
return fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
if err := ossCreateBucket(ctx, client, bucketName); err != nil {
return fmt.Errorf("failed to create bucket: %w", err)
}
}
return nil
}
// NewOssFileService creates an Aliyun OSS file service.
// It verifies that the bucket exists and creates it if missing.
func NewOssFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
return NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
}
// NewOssFileServiceWithTempBucket creates an Aliyun OSS file service with optional temp bucket.
func NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
if err != nil {
return nil, err
}
if err := ossEnsureBucket(context.Background(), client, bucketName); err != nil {
return nil, err
}
var tempClient *s3.Client
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region
}
tempClient, err = newOSSClient(endpoint, tempRegion, accessKey, secretKey)
if err != nil {
return nil, fmt.Errorf("failed to initialize OSS temp client: %w", err)
}
if err := ossEnsureBucket(context.Background(), tempClient, tempBucketName); err != nil {
return nil, err
}
}
// Normalize pathPrefix: ensure it ends with '/' if not empty
if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") {
pathPrefix += "/"
}
return &ossFileService{
client: client,
tempClient: tempClient,
pathPrefix: pathPrefix,
bucketName: bucketName,
tempBucketName: tempBucketName,
}, nil
}
// CheckOssConnectivity tests OSS connectivity using the provided credentials.
func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
if err != nil {
return err
}
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
exists, err := ossBucketExists(checkCtx, client, bucketName)
if err != nil {
return fmt.Errorf("failed to check bucket: %w", err)
}
if !exists {
return fmt.Errorf("bucket %q does not exist", bucketName)
}
return nil
}
// parseOssFilePath extracts bucket and object key from: oss://{bucket}/{objectKey}
func parseOssFilePath(filePath string) (bucketName string, objectKey string, err error) {
if !strings.HasPrefix(filePath, ossScheme) {
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
}
rest := strings.TrimPrefix(filePath, ossScheme)
parts := strings.SplitN(rest, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
}
return parts[0], parts[1], nil
}
// CheckConnectivity verifies OSS is reachable and the main bucket exists.
func (s *ossFileService) CheckConnectivity(ctx context.Context) error {
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
exists, err := ossBucketExists(checkCtx, s.client, s.bucketName)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("bucket %q does not exist", s.bucketName)
}
return nil
}
// SaveFile saves a file to OSS.
func (s *ossFileService) SaveFile(ctx context.Context,
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
) (string, error) {
ext := filepath.Ext(file.Filename)
objectName := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
src, err := file.Open()
if err != nil {
return "", fmt.Errorf("failed to open file: %w", err)
}
defer src.Close()
contentType := file.Header.Get("Content-Type")
if contentType == "" {
contentType = getContentTypeByExt(ext)
}
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
Body: src,
ContentType: aws.String(contentType),
})
if err != nil {
return "", fmt.Errorf("failed to upload file to OSS: %w", err)
}
return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil
}
// SaveBytes saves bytes data to OSS.
// If temp is true and temp bucket is configured, saves to temp bucket.
// Otherwise saves to main bucket.
func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
safeName, err := utils.SafeFileName(fileName)
if err != nil {
return "", fmt.Errorf("invalid file name: %w", err)
}
ext := filepath.Ext(safeName)
// If requesting temp bucket and it is configured, use it
if temp && s.tempClient != nil {
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
_, err := s.tempClient.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.tempBucketName),
Key: aws.String(objectName),
Body: bytes.NewReader(data),
ContentType: aws.String("text/csv; charset=utf-8"),
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to OSS temp bucket: %w", err)
}
return fmt.Sprintf("oss://%s/%s", s.tempBucketName, objectName), nil
}
// Save to main bucket
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
Body: bytes.NewReader(data),
ContentType: aws.String("text/csv; charset=utf-8"),
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to OSS: %w", err)
}
return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil
}
// GetFile retrieves a file from OSS by its path.
func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
bucketName, objectName, err := parseOssFilePath(filePath)
if err != nil {
return nil, err
}
if err := utils.SafeObjectKey(objectName); err != nil {
return nil, fmt.Errorf("invalid file path: %w", err)
}
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectName),
})
if err != nil {
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
}
return resp.Body, nil
}
// DeleteFile removes a file from OSS.
func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error {
bucketName, objectName, err := parseOssFilePath(filePath)
if err != nil {
return err
}
if err := utils.SafeObjectKey(objectName); err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectName),
})
if err != nil {
return fmt.Errorf("failed to delete file from OSS: %w", err)
}
return nil
}
// GetFileURL returns a presigned download URL for the file.
func (s *ossFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
bucketName, objectName, err := parseOssFilePath(filePath)
if err != nil {
return "", err
}
if err := utils.SafeObjectKey(objectName); err != nil {
return "", fmt.Errorf("invalid file path: %w", err)
}
// Determine which client to use
var client *s3.Client
if bucketName == s.tempBucketName && s.tempClient != nil {
client = s.tempClient
} else {
client = s.client
}
presignClient := s3.NewPresignClient(client)
presignedReq, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectName),
}, s3.WithPresignExpires(24*time.Hour))
if err != nil {
return "", fmt.Errorf("failed to generate OSS presigned URL: %w", err)
}
return presignedReq.URL, nil
}
@@ -0,0 +1,208 @@
package file
import (
"context"
"strings"
"testing"
"time"
)
func TestParseOssFilePath(t *testing.T) {
tests := []struct {
name string
input string
wantBucket string
wantKey string
wantErr bool
errContains string
}{
{
name: "valid path with nested key",
input: "oss://my-bucket/123/exports/abc123.csv",
wantBucket: "my-bucket",
wantKey: "123/exports/abc123.csv",
},
{
name: "valid path with simple key",
input: "oss://test-bucket/key",
wantBucket: "test-bucket",
wantKey: "key",
},
{
name: "valid path with deep nesting",
input: "oss://bucket/prefix/tenant/exports/uuid.png",
wantBucket: "bucket",
wantKey: "prefix/tenant/exports/uuid.png",
},
{
name: "invalid scheme",
input: "s3://bucket/key",
wantErr: true,
errContains: "invalid OSS file path",
},
{
name: "empty path",
input: "",
wantErr: true,
errContains: "invalid OSS file path",
},
{
name: "bucket only no key",
input: "oss://bucket/",
wantErr: true,
errContains: "invalid OSS file path",
},
{
name: "scheme only",
input: "oss://",
wantErr: true,
errContains: "invalid OSS file path",
},
{
name: "no slash after bucket",
input: "oss://bucket",
wantErr: true,
errContains: "invalid OSS file path",
},
{
name: "empty bucket name",
input: "oss:///some-key",
wantErr: true,
errContains: "invalid OSS file path",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
bucket, key, err := parseOssFilePath(tt.input)
if tt.wantErr {
if err == nil {
t.Errorf("parseOssFilePath(%q) expected error, got bucket=%q key=%q", tt.input, bucket, key)
}
if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) {
t.Errorf("parseOssFilePath(%q) error = %v, want containing %q", tt.input, err, tt.errContains)
}
return
}
if err != nil {
t.Errorf("parseOssFilePath(%q) unexpected error: %v", tt.input, err)
return
}
if bucket != tt.wantBucket {
t.Errorf("parseOssFilePath(%q) bucket = %q, want %q", tt.input, bucket, tt.wantBucket)
}
if key != tt.wantKey {
t.Errorf("parseOssFilePath(%q) key = %q, want %q", tt.input, key, tt.wantKey)
}
})
}
}
func TestNewOSSClient(t *testing.T) {
tests := []struct {
name string
endpoint string
region string
accessKey string
secretKey string
wantErr bool
}{
{
name: "valid parameters create client",
endpoint: "https://oss-cn-hangzhou.aliyuncs.com",
region: "cn-hangzhou",
accessKey: "test-access-key",
secretKey: "test-secret-key",
wantErr: false,
},
{
name: "custom endpoint",
endpoint: "https://custom-oss-endpoint.com",
region: "cn-shanghai",
accessKey: "ak",
secretKey: "sk",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client, err := newOSSClient(tt.endpoint, tt.region, tt.accessKey, tt.secretKey)
if tt.wantErr {
if err == nil {
t.Error("expected error but got nil")
}
return
}
if err != nil {
t.Errorf("newOSSClient() unexpected error: %v", err)
return
}
if client == nil {
t.Error("expected non-nil client")
}
})
}
}
func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Should fail with an invalid/unreachable endpoint
err := CheckOssConnectivity(ctx,
"https://invalid-oss-endpoint-that-does-not-exist.local",
"cn-hangzhou",
"invalid-access-key",
"invalid-secret-key",
"nonexistent-bucket",
)
if err == nil {
t.Error("CheckOssConnectivity with invalid endpoint should return an error")
}
}
func TestOssBucketExists_NonExistent(t *testing.T) {
client, err := newOSSClient(
"https://oss-cn-hangzhou.aliyuncs.com",
"cn-hangzhou",
"test-invalid-key",
"test-invalid-secret",
)
if err != nil {
t.Fatalf("newOSSClient() error: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Bucket that definitely doesn't exist - should return false, not error
// (auth may fail first, but we verify the function doesn't crash)
exists, err := ossBucketExists(ctx, client, "this-bucket-definitely-does-not-exist-12345")
// We don't assert on the result because it depends on network/auth,
// but the function should not panic or hang.
_ = exists
_ = err
}
func TestOssCreateBucket(t *testing.T) {
client, err := newOSSClient(
"https://oss-cn-hangzhou.aliyuncs.com",
"cn-hangzhou",
"test-invalid-key",
"test-invalid-secret",
)
if err != nil {
t.Fatalf("newOSSClient() error: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
// Should fail with invalid credentials
err = ossCreateBucket(ctx, client, "test-bucket")
if err == nil {
t.Error("ossCreateBucket with invalid credentials should return an error")
}
}
+22
View File
@@ -647,6 +647,28 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("S3_REGION"),
pathPrefix,
)
case "oss":
if os.Getenv("OSS_ENDPOINT") == "" ||
os.Getenv("OSS_REGION") == "" ||
os.Getenv("OSS_ACCESS_KEY") == "" ||
os.Getenv("OSS_SECRET_KEY") == "" ||
os.Getenv("OSS_BUCKET_NAME") == "" {
return nil, fmt.Errorf("missing OSS configuration")
}
pathPrefix := os.Getenv("OSS_PATH_PREFIX")
if pathPrefix == "" {
pathPrefix = "weknora/"
}
return file.NewOssFileServiceWithTempBucket(
os.Getenv("OSS_ENDPOINT"),
os.Getenv("OSS_REGION"),
os.Getenv("OSS_ACCESS_KEY"),
os.Getenv("OSS_SECRET_KEY"),
os.Getenv("OSS_BUCKET_NAME"),
pathPrefix,
os.Getenv("OSS_TEMP_BUCKET_NAME"),
os.Getenv("OSS_TEMP_REGION"),
)
case "local":
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
if baseDir == "" {
+69 -1
View File
@@ -396,6 +396,17 @@ func (h *SystemHandler) isTOSConfigured(c *gin.Context) bool {
return h.isTOSEnvAvailable()
}
// isOSSConfigured checks whether OSS connection info is available from tenant config.
func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool {
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.OSS != nil {
ossConf := tenant.StorageEngineConfig.OSS
return ossConf.Endpoint != "" && ossConf.Region != "" && ossConf.AccessKey != "" && ossConf.SecretKey != "" && ossConf.BucketName != ""
}
}
return false
}
// isTOSEnvAvailable checks whether TOS env vars are set.
func (h *SystemHandler) isTOSEnvAvailable() bool {
return os.Getenv("TOS_ENDPOINT") != "" &&
@@ -442,11 +453,13 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
minioEnvAvailable := h.isMinioEnvAvailable()
cosConfigured := h.isCOSConfigured(c)
tosConfigured := h.isTOSConfigured(c)
ossConfigured := h.isOSSConfigured(c)
engines := []StorageEngineStatusItem{
{Name: "local", Available: true, Description: "本地文件系统存储,仅适合单机部署"},
{Name: "minio", Available: minioConfigured || minioEnvAvailable, Description: "S3 兼容的自托管对象存储,适合内网和私有云部署"},
{Name: "cos", Available: cosConfigured, Description: "腾讯云对象存储服务,适合公有云部署,支持 CDN 加速"},
{Name: "tos", Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"},
{Name: "oss", Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"},
}
c.JSON(200, gin.H{
"code": 0,
@@ -640,6 +653,9 @@ func hasGetObjectAction(action interface{}) bool {
// cosFieldPattern validates COS region and bucket name format to prevent URL injection.
var cosFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
// ossFieldPattern validates OSS region and bucket name format to prevent URL injection.
var ossFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
// sanitizeStorageCheckError converts a raw storage connectivity error into a safe
// user-facing message that does not leak internal network details (hostnames, IPs, ports).
func sanitizeStorageCheckError(err error) string {
@@ -727,11 +743,12 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) {
// StorageCheckRequest is the body for POST /system/storage-engine-check.
type StorageCheckRequest struct {
Provider string `json:"provider"` // "minio", "cos", "tos", or "s3"
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss"
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
COS *types.COSEngineConfig `json:"cos,omitempty"`
TOS *types.TOSEngineConfig `json:"tos,omitempty"`
S3 *types.S3EngineConfig `json:"s3,omitempty"`
OSS *types.OSSEngineConfig `json:"oss,omitempty"`
}
// StorageCheckResponse is the response for a single-engine connectivity check.
@@ -768,6 +785,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
h.checkTOS(c, ctx, req.TOS)
case "s3":
h.checkS3(c, ctx, req.S3)
case "oss":
h.checkOSS(c, ctx, req.OSS)
default:
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
}
@@ -960,6 +979,55 @@ func (h *SystemHandler) checkS3(c *gin.Context, ctx context.Context, cfg *types.
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types.OSSEngineConfig) {
if cfg == nil {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 OSS 配置"}})
return
}
endpoint, accessKey, secretKey := cfg.Endpoint, cfg.AccessKey, cfg.SecretKey
if endpoint == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Access Key、Secret Key、Bucket Name 不能为空"}})
return
}
// Strip URL scheme before SSRF check — OSS endpoint may include http:// or https://
ssrfEndpoint := strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
if blocked, reason := isBlockedStorageEndpoint(ssrfEndpoint); blocked {
logger.Warnf(ctx, "Storage check: OSS endpoint blocked by SSRF protection", "endpoint", endpoint)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
return
}
if !ossFieldPattern.MatchString(cfg.Region) {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Region 格式不正确,仅允许字母、数字、点、连字符"}})
return
}
if !ossFieldPattern.MatchString(cfg.BucketName) {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Bucket 名称格式不正确,仅允许字母、数字、点、连字符"}})
return
}
err := file.CheckOssConnectivity(ctx, endpoint, cfg.Region, accessKey, secretKey, cfg.BucketName)
if err != nil {
errMsg := err.Error()
if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") {
logger.Errorf(ctx, "Storage check: OSS auth failed", "endpoint", endpoint, "bucket", cfg.BucketName)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}})
return
}
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
logger.Errorf(ctx, "Storage check: OSS bucket not found", "bucket", cfg.BucketName)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在", cfg.BucketName)}})
return
}
logger.Errorf(ctx, "Storage check: OSS connectivity failed", "endpoint", endpoint, "bucket", cfg.BucketName, "error", err)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("OSS 连通性检测失败: %s", sanitizeStorageCheckError(err))}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader {
if addr == "" {
return h.documentReader
+2 -2
View File
@@ -152,7 +152,7 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string {
// StorageProviderConfig stores the KB-level storage provider selection.
// Credentials are managed at the tenant level (StorageEngineConfig).
type StorageProviderConfig struct {
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos"
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss"
}
func (c StorageProviderConfig) Value() (driver.Value, error) {
@@ -266,7 +266,7 @@ func InferStorageFromFilePath(filePath string) string {
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
// Returns "" if the path does not use a known provider scheme.
func ParseProviderScheme(filePath string) string {
for _, provider := range []string{"local", "minio", "cos", "tos", "s3"} {
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss"} {
if strings.HasPrefix(filePath, provider+"://") {
return provider
}
+16 -2
View File
@@ -328,15 +328,16 @@ func (c *ParserEngineConfig) Scan(value interface{}) error {
return json.Unmarshal(b, c)
}
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, and S3.
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, and OSS.
// Knowledge bases select which provider to use; parameters are read from here.
type StorageEngineConfig struct {
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3"
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss"
Local *LocalEngineConfig `json:"local,omitempty"`
MinIO *MinIOEngineConfig `json:"minio,omitempty"`
COS *COSEngineConfig `json:"cos,omitempty"`
TOS *TOSEngineConfig `json:"tos,omitempty"`
S3 *S3EngineConfig `json:"s3,omitempty"`
OSS *OSSEngineConfig `json:"oss,omitempty"`
}
// LocalEngineConfig is for local file system storage (single-machine deployment only).
@@ -386,6 +387,19 @@ type S3EngineConfig struct {
PathPrefix string `json:"path_prefix"`
}
// OSSEngineConfig is for Alibaba Cloud OSS (对象存储服务).
type OSSEngineConfig struct {
Endpoint string `json:"endpoint"`
Region string `json:"region"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
BucketName string `json:"bucket_name"`
PathPrefix string `json:"path_prefix"`
UseTempBucket bool `json:"use_temp_bucket"`
TempBucketName string `json:"temp_bucket_name"`
TempRegion string `json:"temp_region"`
}
// Value implements the driver.Valuer interface for StorageEngineConfig
func (c *StorageEngineConfig) Value() (driver.Value, error) {
if c == nil {