Merge pull request #949 from peterchan2020/feat_oss
Feat: Add Alibaba Cloud Storage Configuration
This commit is contained in:
@@ -181,7 +181,7 @@ Fully modular pipeline from document parsing, vectorization, and retrieval to LL
|
||||
| LLMs | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
|
||||
| Embeddings | Ollama / BGE / GTE / OpenAI-compatible APIs |
|
||||
| Vector DBs | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
|
||||
| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS |
|
||||
| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS |
|
||||
| IM Channels | WeCom / Feishu / Slack / Telegram / DingTalk / Mattermost |
|
||||
| Web Search | DuckDuckGo / Bing / Google / Tavily |
|
||||
|
||||
|
||||
+1
-1
@@ -180,7 +180,7 @@ WeKnora 提供**快速问答**与**智能推理**两种问答模式。快速问
|
||||
|------|------|
|
||||
| 模型厂商 | OpenAI / DeepSeek / Qwen(阿里云)/ 智谱 / 混元 / 豆包(火山引擎)/ Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
|
||||
| 向量数据库 | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
|
||||
| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 |
|
||||
| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 / 阿里云 OSS |
|
||||
| IM 集成 | 企业微信 / 飞书 / Slack / Telegram / 钉钉 / Mattermost |
|
||||
| 网络搜索 | DuckDuckGo / Bing / Google / Tavily |
|
||||
|
||||
|
||||
+1
-1
@@ -182,7 +182,7 @@ Feishuなどの外部プラットフォームからのナレッジ自動同期
|
||||
| 大規模モデル | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
|
||||
| Embedding | Ollama / BGE / GTE / OpenAI 互換 API |
|
||||
| ベクトル DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
|
||||
| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS |
|
||||
| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS / Alibaba Cloud OSS |
|
||||
| IM 統合 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost |
|
||||
| Web 検索 | DuckDuckGo / Bing / Google / Tavily |
|
||||
|
||||
|
||||
+1
-1
@@ -191,7 +191,7 @@ Feishu 등 외부 플랫폼에서 지식 자동 동기화를 지원하며(추가
|
||||
| LLM | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama |
|
||||
| Embedding | Ollama / BGE / GTE / OpenAI 호환 API |
|
||||
| 벡터 DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant |
|
||||
| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS |
|
||||
| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS |
|
||||
| IM 통합 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost |
|
||||
| 웹 검색 | DuckDuckGo / Bing / Google / Tavily |
|
||||
|
||||
|
||||
@@ -126,6 +126,16 @@ DocReader 支持多种存储后端:
|
||||
- `COS_PATH_PREFIX`: 文件路径前缀
|
||||
- `COS_ENABLE_OLD_DOMAIN`: 是否使用旧域名(默认:true)
|
||||
|
||||
#### 阿里云 OSS 存储
|
||||
|
||||
- `STORAGE_TYPE`: 设置为 `oss`
|
||||
- `OSS_ACCESS_KEY_ID`: OSS 访问密钥 ID
|
||||
- `OSS_ACCESS_KEY_SECRET`: OSS 访问密钥
|
||||
- `OSS_ENDPOINT`: OSS 端点(如 `oss-cn-hangzhou.aliyuncs.com`)
|
||||
- `OSS_BUCKET_NAME`: OSS 存储桶名称
|
||||
- `OSS_REGION`: OSS 区域(如 `cn-hangzhou`)
|
||||
- `OSS_PATH_PREFIX`: 文件路径前缀
|
||||
|
||||
### 代理配置
|
||||
|
||||
如果需要通过代理访问外部服务:
|
||||
@@ -178,6 +188,20 @@ docreader:
|
||||
- MAX_FILE_SIZE_MB=50
|
||||
```
|
||||
|
||||
### 使用阿里云 OSS
|
||||
|
||||
```yaml
|
||||
docreader:
|
||||
environment:
|
||||
- STORAGE_TYPE=oss
|
||||
- OSS_ACCESS_KEY_ID=your_access_key_id
|
||||
- OSS_ACCESS_KEY_SECRET=your_access_key_secret
|
||||
- OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com
|
||||
- OSS_BUCKET_NAME=your-bucket
|
||||
- OSS_REGION=cn-hangzhou
|
||||
- MAX_FILE_SIZE_MB=50
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
### 1. DocReader 服务无法启动?
|
||||
|
||||
+101
-1
@@ -10,6 +10,11 @@ from typing import Dict, Optional
|
||||
from minio import Minio
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
try:
|
||||
import oss2
|
||||
except ImportError:
|
||||
oss2 = None # type: ignore[assignment]
|
||||
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -232,6 +237,99 @@ class MinioStorage(Storage):
|
||||
return ""
|
||||
|
||||
|
||||
class OssStorage(Storage):
|
||||
"""Alibaba Cloud OSS storage implementation"""
|
||||
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.bucket, self.endpoint, self.prefix = self._init_oss_bucket()
|
||||
|
||||
def _init_oss_bucket(self):
|
||||
if oss2 is None:
|
||||
logger.error("oss2 package not installed. Install with: pip install oss2")
|
||||
return None, None, None
|
||||
try:
|
||||
sc = self.storage_config
|
||||
access_key = _cfg(sc, "access_key", "OSS_ACCESS_KEY")
|
||||
secret_key = _cfg(sc, "secret_key", "OSS_SECRET_KEY")
|
||||
endpoint_raw = _cfg(sc, "endpoint", "OSS_ENDPOINT")
|
||||
bucket_name = _cfg(sc, "bucket_name", "OSS_BUCKET_NAME")
|
||||
prefix_raw = _cfg(sc, "path_prefix", "OSS_PATH_PREFIX")
|
||||
|
||||
if not all([access_key, secret_key, endpoint_raw, bucket_name]):
|
||||
logger.error(
|
||||
"Incomplete OSS configuration: "
|
||||
"access_key=%s, endpoint=%s, bucket=%s",
|
||||
bool(access_key), endpoint_raw, bucket_name,
|
||||
)
|
||||
return None, None, None
|
||||
|
||||
# oss2 requires endpoint to include the scheme (http:// or https://)
|
||||
if not endpoint_raw.startswith(("http://", "https://")):
|
||||
endpoint_raw = f"https://{endpoint_raw}"
|
||||
|
||||
auth = oss2.Auth(access_key, secret_key)
|
||||
bucket = oss2.Bucket(auth, endpoint_raw, bucket_name)
|
||||
|
||||
# Verify bucket exists; do NOT auto-create to avoid unintended public-read buckets
|
||||
try:
|
||||
bucket.get_bucket_info()
|
||||
except oss2.exceptions.NoSuchBucket:
|
||||
logger.error(
|
||||
"OSS bucket %s does not exist. Please create it manually in the console.",
|
||||
bucket_name,
|
||||
)
|
||||
return None, None, None
|
||||
|
||||
prefix = prefix_raw.strip().strip("/") if prefix_raw else ""
|
||||
return bucket, endpoint_raw, prefix
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize OSS client: %s", e)
|
||||
return None, None, None
|
||||
|
||||
def _get_download_url(self, object_key: str) -> str:
|
||||
# Use virtual-hosted style URL: https://{bucket}.{endpoint_without_scheme}/{key}
|
||||
endpoint_no_scheme = self.endpoint.removeprefix("https://").removeprefix("http://")
|
||||
return f"https://{self.bucket.bucket_name}.{endpoint_no_scheme}/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
try:
|
||||
if not self.bucket:
|
||||
return ""
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
self.bucket.put_object_from_file(object_key, file_path)
|
||||
file_url = self._get_download_url(object_key)
|
||||
logger.info("OSS upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error("OSS upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
try:
|
||||
if not self.bucket:
|
||||
return ""
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
self.bucket.put_object(object_key, content)
|
||||
file_url = self._get_download_url(object_key)
|
||||
logger.info("OSS upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error("OSS upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
class LocalStorage(Storage):
|
||||
"""Local file system storage implementation.
|
||||
|
||||
@@ -297,7 +395,7 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
|
||||
"""Create a storage instance based on storage_config dict.
|
||||
|
||||
The ``provider`` key in storage_config determines the backend:
|
||||
minio, cos, local, base64.
|
||||
minio, cos, oss, local, base64.
|
||||
Falls back to STORAGE_TYPE env var, then ``local``.
|
||||
"""
|
||||
storage_type = ""
|
||||
@@ -315,6 +413,8 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
|
||||
return MinioStorage(storage_config)
|
||||
elif storage_type == "cos":
|
||||
return CosStorage(storage_config)
|
||||
elif storage_type == "oss":
|
||||
return OssStorage(storage_config)
|
||||
elif storage_type == "local":
|
||||
return LocalStorage(storage_config)
|
||||
elif storage_type == "base64":
|
||||
|
||||
@@ -21,6 +21,7 @@ dependencies = [
|
||||
"mistletoe>=1.5.0",
|
||||
"ollama>=0.6.0",
|
||||
"openai>=2.7.1",
|
||||
"oss2>=2.18.0",
|
||||
"paddleocr>=2.10.0,<3.0.0",
|
||||
"paddlepaddle>=3.0.0,<4.0.0",
|
||||
"pdfplumber>=0.11.7",
|
||||
|
||||
Generated
+48
-6
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 2
|
||||
revision = 3
|
||||
requires-python = ">=3.10.18"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14' and sys_platform == 'darwin'",
|
||||
@@ -59,6 +59,28 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/64/013409c451a44b61310fb757af4527f3de57fc98a00f40448de28b864290/albumentations-2.0.8-py3-none-any.whl", hash = "sha256:c4c4259aaf04a7386ad85c7fdcb73c6c7146ca3057446b745cc035805acb1017", size = 369423, upload-time = "2025-05-27T21:23:15.609Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aliyun-python-sdk-core"
|
||||
version = "2.16.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "jmespath" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3e/09/da9f58eb38b4fdb97ba6523274fbf445ef6a06be64b433693da8307b4bec/aliyun-python-sdk-core-2.16.0.tar.gz", hash = "sha256:651caad597eb39d4fad6cf85133dffe92837d53bdf62db9d8f37dab6508bb8f9", size = 449555, upload-time = "2024-10-09T06:01:01.762Z" }
|
||||
|
||||
[[package]]
|
||||
name = "aliyun-python-sdk-kms"
|
||||
version = "2.16.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aliyun-python-sdk-core" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/2c/9877d0e6b18ecf246df671ac65a5d1d9fecbf85bdcb5d43efbde0d4662eb/aliyun-python-sdk-kms-2.16.5.tar.gz", hash = "sha256:f328a8a19d83ecbb965ffce0ec1e9930755216d104638cd95ecd362753b813b3", size = 12018, upload-time = "2024-08-30T09:01:20.104Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/11/5c/0132193d7da2c735669a1ed103b142fd63c9455984d48c5a88a1a516efaa/aliyun_python_sdk_kms-2.16.5-py2.py3-none-any.whl", hash = "sha256:24b6cdc4fd161d2942619479c8d050c63ea9cd22b044fe33b60bbb60153786f0", size = 99495, upload-time = "2024-08-30T09:01:18.462Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "annotated-types"
|
||||
version = "0.7.0"
|
||||
@@ -690,6 +712,7 @@ dependencies = [
|
||||
{ name = "mistletoe" },
|
||||
{ name = "ollama" },
|
||||
{ name = "openai" },
|
||||
{ name = "oss2" },
|
||||
{ name = "paddleocr" },
|
||||
{ name = "paddlepaddle" },
|
||||
{ name = "pdfplumber" },
|
||||
@@ -724,6 +747,7 @@ requires-dist = [
|
||||
{ name = "mistletoe", specifier = ">=1.5.0" },
|
||||
{ name = "ollama", specifier = ">=0.6.0" },
|
||||
{ name = "openai", specifier = ">=2.7.1" },
|
||||
{ name = "oss2", specifier = ">=2.18.0" },
|
||||
{ name = "paddleocr", specifier = ">=2.10.0,<3.0.0" },
|
||||
{ name = "paddlepaddle", specifier = ">=3.0.0,<4.0.0" },
|
||||
{ name = "pdfplumber", specifier = ">=0.11.7" },
|
||||
@@ -1322,6 +1346,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/71/71408b02c6133153336d29fa3ba53000f1e1a3f78bb2fc2d1a1865d2e743/jiter-0.11.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18c77aaa9117510d5bdc6a946baf21b1f0cfa58ef04d31c8d016f206f2118960", size = 343697, upload-time = "2025-10-17T11:31:13.773Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jmespath"
|
||||
version = "0.10.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3c/56/3f325b1eef9791759784aa5046a8f6a1aff8f7c898a2e34506771d3b99d8/jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", size = 21607, upload-time = "2020-05-12T22:03:47.267Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f", size = 24489, upload-time = "2020-05-12T22:03:45.643Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "joblib"
|
||||
version = "1.5.2"
|
||||
@@ -1984,21 +2017,16 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/db/81bf3d7cecfbfed9092b6b4052e857a769d62ed90561b410014e0aae18db/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:b28740f4ecef1738ea8f807461dd541b8287d5650b5be33bca7b474e3cbd1f36", size = 19153079, upload-time = "2025-10-27T23:05:57.686Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/4d/a382452b17cf70a2313153c520ea4c96ab670c996cb3a95cc5d5ac7bfdac/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f7d1fe034090a1e371b7f3ca9d3ccae2fabae8c1d8844fb7371d1ea38e8e8d2", size = 15219883, upload-time = "2025-10-22T03:46:21.66Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/56/179bf90679984c85b417664c26aae4f427cba7514bd2d65c43b181b7b08b/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ca88747e708e5c67337b0f65eed4b7d0dd70d22ac332038c9fc4635760018f7", size = 17370357, upload-time = "2025-10-22T03:46:57.968Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113, upload-time = "2025-10-22T03:47:33.526Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857, upload-time = "2025-10-22T03:46:37.578Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095, upload-time = "2025-10-22T03:46:24.769Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080, upload-time = "2025-10-22T03:47:00.265Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929, upload-time = "2025-10-22T03:47:36.24Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915, upload-time = "2025-10-22T03:46:27.773Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649, upload-time = "2025-10-22T03:47:02.782Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337, upload-time = "2025-10-22T03:46:35.168Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898, upload-time = "2025-10-22T03:46:30.039Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518, upload-time = "2025-10-22T03:47:05.407Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610, upload-time = "2025-10-22T03:46:32.239Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2099,6 +2127,20 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/19/404708a7e54ad2798907210462fd950c3442ea51acc8790f3da48d2bee8b/opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147", size = 65486, upload-time = "2020-07-19T22:40:30.301Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oss2"
|
||||
version = "2.19.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aliyun-python-sdk-core" },
|
||||
{ name = "aliyun-python-sdk-kms" },
|
||||
{ name = "crcmod" },
|
||||
{ name = "pycryptodome" },
|
||||
{ name = "requests" },
|
||||
{ name = "six" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/df/b5/f2cb1950dda46ac2284d6c950489fdacd0e743c2d79a347924d3cc44b86f/oss2-2.19.1.tar.gz", hash = "sha256:a8ab9ee7eb99e88a7e1382edc6ea641d219d585a7e074e3776e9dec9473e59c1", size = 298845, upload-time = "2024-10-25T11:37:46.638Z" }
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "25.0"
|
||||
|
||||
+1
-1
@@ -458,7 +458,7 @@ curl --location 'http://localhost:8080/api/v1/agents/placeholders' \
|
||||
|------|------|--------|------|
|
||||
| `image_upload_enabled` | bool | false | 是否允许上传图片 |
|
||||
| `vlm_model_id` | string | - | 图片分析所用的 VLM 模型 ID |
|
||||
| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`,为空使用全局默认 |
|
||||
| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`/`oss`,为空使用全局默认 |
|
||||
|
||||
### FAQ 策略设置
|
||||
|
||||
|
||||
@@ -155,6 +155,11 @@ curl --location 'http://localhost:8080/api/v1/system/storage-engine-status' \
|
||||
"name": "s3",
|
||||
"available": false,
|
||||
"description": "AWS S3 对象存储"
|
||||
},
|
||||
{
|
||||
"name": "oss",
|
||||
"available": false,
|
||||
"description": "阿里云 OSS 对象存储"
|
||||
}
|
||||
],
|
||||
"minio_env_available": true
|
||||
|
||||
+30
-5
@@ -429,7 +429,7 @@ definitions:
|
||||
type: integer
|
||||
image_storage_provider:
|
||||
description: |-
|
||||
Storage provider for image uploads: "local", "minio", "cos", "tos"
|
||||
Storage provider for image uploads: "local", "minio", "cos", "tos", "s3", "oss"
|
||||
Empty means use the global/tenant default provider.
|
||||
type: string
|
||||
image_upload_enabled:
|
||||
@@ -2067,12 +2067,14 @@ definitions:
|
||||
cos:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.COSEngineConfig'
|
||||
default_provider:
|
||||
description: '"local", "minio", "cos", "tos", "s3"'
|
||||
description: '"local", "minio", "cos", "tos", "s3", "oss"'
|
||||
type: string
|
||||
local:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.LocalEngineConfig'
|
||||
minio:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig'
|
||||
oss:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig'
|
||||
s3:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig'
|
||||
tos:
|
||||
@@ -2081,7 +2083,7 @@ definitions:
|
||||
github_com_Tencent_WeKnora_internal_types.StorageProviderConfig:
|
||||
properties:
|
||||
provider:
|
||||
description: '"local", "minio", "cos", "tos"'
|
||||
description: '"local", "minio", "cos", "tos", "s3", "oss"'
|
||||
type: string
|
||||
type: object
|
||||
github_com_Tencent_WeKnora_internal_types.SubmitJoinRequestRequest:
|
||||
@@ -2169,6 +2171,27 @@ definitions:
|
||||
secret_key:
|
||||
type: string
|
||||
type: object
|
||||
github_com_Tencent_WeKnora_internal_types.OSSEngineConfig:
|
||||
properties:
|
||||
access_key:
|
||||
type: string
|
||||
bucket_name:
|
||||
type: string
|
||||
endpoint:
|
||||
type: string
|
||||
path_prefix:
|
||||
type: string
|
||||
region:
|
||||
type: string
|
||||
secret_key:
|
||||
type: string
|
||||
temp_bucket_name:
|
||||
type: string
|
||||
temp_region:
|
||||
type: string
|
||||
use_temp_bucket:
|
||||
type: boolean
|
||||
type: object
|
||||
github_com_Tencent_WeKnora_internal_types.Tenant:
|
||||
properties:
|
||||
agent_config:
|
||||
@@ -2671,8 +2694,10 @@ definitions:
|
||||
minio:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig'
|
||||
provider:
|
||||
description: '"minio", "cos", "tos", or "s3"'
|
||||
description: '"minio", "cos", "tos", "s3", or "oss"'
|
||||
type: string
|
||||
oss:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig'
|
||||
s3:
|
||||
$ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig'
|
||||
tos:
|
||||
@@ -2696,7 +2721,7 @@ definitions:
|
||||
description: short description for UI
|
||||
type: string
|
||||
name:
|
||||
description: '"local", "minio", "cos", "tos"'
|
||||
description: '"local", "minio", "cos", "tos", "s3", "oss"'
|
||||
type: string
|
||||
type: object
|
||||
internal_handler.TextRelationExtractionRequest:
|
||||
|
||||
@@ -183,7 +183,7 @@ export function reconnectDocReader(addr: string): Promise<ParserEnginesResponse
|
||||
// ---- 存储引擎配置(租户级,供文档/图片存储与 docreader 使用) ----
|
||||
|
||||
export interface StorageEngineConfig {
|
||||
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3"
|
||||
default_provider: string // "local" | "minio" | "cos" | "tos" | "s3" | "oss"
|
||||
local?: { path_prefix: string }
|
||||
minio?: { mode: string; endpoint: string; access_key_id: string; secret_access_key: string; bucket_name: string; use_ssl: boolean; path_prefix: string }
|
||||
cos?: {
|
||||
@@ -210,6 +210,17 @@ export interface StorageEngineConfig {
|
||||
bucket_name: string
|
||||
path_prefix: string
|
||||
}
|
||||
oss?: {
|
||||
endpoint: string
|
||||
region: string
|
||||
access_key: string
|
||||
secret_key: string
|
||||
bucket_name: string
|
||||
path_prefix: string
|
||||
use_temp_bucket: boolean
|
||||
temp_bucket_name: string
|
||||
temp_region: string
|
||||
}
|
||||
}
|
||||
|
||||
export interface StorageEngineStatusItem {
|
||||
@@ -236,11 +247,12 @@ export function getStorageEngineStatus(): Promise<{ data: GetStorageEngineStatus
|
||||
}
|
||||
|
||||
export interface StorageCheckRequest {
|
||||
provider: string // "minio" | "cos" | "tos" | "s3"
|
||||
provider: string // "minio" | "cos" | "tos" | "s3" | "oss"
|
||||
minio?: StorageEngineConfig['minio']
|
||||
cos?: StorageEngineConfig['cos']
|
||||
tos?: StorageEngineConfig['tos']
|
||||
s3?: StorageEngineConfig['s3']
|
||||
oss?: StorageEngineConfig['oss']
|
||||
}
|
||||
|
||||
export interface StorageCheckResponse {
|
||||
|
||||
@@ -659,6 +659,7 @@ export default {
|
||||
engineLocal: 'Local',
|
||||
engineCos: 'Tencent Cloud COS',
|
||||
engineTos: 'Volcengine TOS',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineS3: 'AWS S3',
|
||||
localTitle: 'Local Storage',
|
||||
localDesc: 'Store files on the server local filesystem, suitable for single-node deployment only.',
|
||||
@@ -692,6 +693,10 @@ export default {
|
||||
s3Desc: 'AWS S3 and S3-compatible object storage services, suitable for public cloud deployment.',
|
||||
s3AccessKeyPlaceholder: 'AWS Access Key',
|
||||
s3SecretKeyPlaceholder: 'AWS Secret Key',
|
||||
ossTitle: 'Alibaba Cloud OSS',
|
||||
ossDesc: 'Alibaba Cloud Object Storage Service (OSS), suitable for public cloud deployment.',
|
||||
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
|
||||
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
|
||||
console: 'Console',
|
||||
docs: 'Docs',
|
||||
testConnection: 'Test Connection',
|
||||
@@ -703,6 +708,7 @@ export default {
|
||||
requestFailed: 'Request failed',
|
||||
cos: 'Tencent Cloud COS',
|
||||
tos: 'Volcengine TOS',
|
||||
oss: 'Alibaba Cloud OSS',
|
||||
},
|
||||
},
|
||||
webSearchSettings: {
|
||||
@@ -3117,6 +3123,8 @@ export default {
|
||||
engineTosDesc: 'Volcengine object storage, for public cloud deployment',
|
||||
engineS3: 'AWS S3',
|
||||
engineS3Desc: 'AWS S3 and compatible storage, for public cloud deployment',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineOssDesc: 'Alibaba Cloud Object Storage Service, for public cloud deployment',
|
||||
},
|
||||
parser: {
|
||||
title: 'Parser Engine',
|
||||
|
||||
@@ -516,6 +516,7 @@ export default {
|
||||
engineLocal: 'Local(로컬)',
|
||||
engineCos: 'Tencent Cloud COS',
|
||||
engineTos: 'Volcengine TOS',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineS3: 'AWS S3',
|
||||
localTitle: 'Local(로컬 스토리지)',
|
||||
localDesc: '서버 로컬 파일시스템에 파일을 저장합니다. 단일 노드 배포에만 적합합니다.',
|
||||
@@ -549,6 +550,10 @@ export default {
|
||||
s3Desc: 'AWS S3 및 호환 오브젝트 스토리지 서비스, 퍼블릭 클라우드 배포에 적합합니다.',
|
||||
s3AccessKeyPlaceholder: 'AWS Access Key',
|
||||
s3SecretKeyPlaceholder: 'AWS Secret Key',
|
||||
ossTitle: 'Alibaba Cloud OSS',
|
||||
ossDesc: 'Alibaba Cloud 오브젝트 스토리지 서비스(OSS), 퍼블릭 클라우드 배포에 적합합니다.',
|
||||
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
|
||||
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
|
||||
console: '콘솔',
|
||||
docs: '문서',
|
||||
testConnection: '연결 테스트',
|
||||
@@ -560,6 +565,7 @@ export default {
|
||||
requestFailed: '요청 실패',
|
||||
cos: 'Tencent Cloud COS',
|
||||
tos: 'Volcengine TOS',
|
||||
oss: 'Alibaba Cloud OSS',
|
||||
},
|
||||
},
|
||||
webSearchSettings: {
|
||||
@@ -3172,6 +3178,8 @@ export default {
|
||||
engineTosDesc: 'Volcengine 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
|
||||
engineS3: 'AWS S3',
|
||||
engineS3Desc: 'AWS S3 및 호환 스토리지, 퍼블릭 클라우드 배포에 적합',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineOssDesc: 'Alibaba Cloud 오브젝트 스토리지, 퍼블릭 클라우드 배포에 적합',
|
||||
},
|
||||
parser: {
|
||||
title: '파서 엔진',
|
||||
|
||||
@@ -640,6 +640,7 @@ export default {
|
||||
engineLocal: 'Локальное',
|
||||
engineCos: 'Tencent Cloud COS',
|
||||
engineTos: 'Volcengine TOS',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineS3: 'AWS S3',
|
||||
localTitle: 'Локальное хранилище',
|
||||
localDesc: 'Хранение файлов в локальной файловой системе сервера. Подходит только для однонодового развёртывания.',
|
||||
@@ -673,6 +674,10 @@ export default {
|
||||
s3Desc: 'AWS S3 и совместимые сервисы объектного хранилища для публичного облака.',
|
||||
s3AccessKeyPlaceholder: 'AWS Access Key',
|
||||
s3SecretKeyPlaceholder: 'AWS Secret Key',
|
||||
ossTitle: 'Alibaba Cloud OSS',
|
||||
ossDesc: 'Объектное хранилище Alibaba Cloud (OSS), подходит для публичного облака.',
|
||||
ossAccessKeyPlaceholder: 'Alibaba Cloud Access Key',
|
||||
ossSecretKeyPlaceholder: 'Alibaba Cloud Secret Key',
|
||||
console: 'Консоль',
|
||||
docs: 'Документация',
|
||||
testConnection: 'Тест подключения',
|
||||
@@ -683,7 +688,8 @@ export default {
|
||||
unknownError: 'Неизвестная ошибка',
|
||||
requestFailed: 'Ошибка запроса',
|
||||
cos: 'Tencent Cloud COS',
|
||||
tos: 'Volcengine TOS'
|
||||
tos: 'Volcengine TOS',
|
||||
oss: 'Alibaba Cloud OSS',
|
||||
}
|
||||
},
|
||||
webSearchSettings: {
|
||||
@@ -2787,6 +2793,8 @@ export default {
|
||||
engineTosDesc: 'Объектное хранилище Volcengine, для публичного облака',
|
||||
engineS3: 'AWS S3',
|
||||
engineS3Desc: 'AWS S3 и совместимые хранилища, для публичного облака',
|
||||
engineOss: 'Alibaba Cloud OSS',
|
||||
engineOssDesc: 'Объектное хранилище Alibaba Cloud, для публичного облака',
|
||||
},
|
||||
parser: {
|
||||
title: 'Парсер',
|
||||
|
||||
@@ -513,6 +513,7 @@ export default {
|
||||
engineLocal: "Local(本地)",
|
||||
engineCos: "腾讯云 COS",
|
||||
engineTos: "火山引擎 TOS",
|
||||
engineOss: "阿里云 OSS",
|
||||
engineS3: "AWS S3",
|
||||
localTitle: "Local(本地存储)",
|
||||
localDesc: "使用服务器本地文件系统存储文件,仅适合单机部署。",
|
||||
@@ -546,6 +547,10 @@ export default {
|
||||
s3Desc: "AWS S3 及兼容的对象存储服务,适合公有云部署。",
|
||||
s3AccessKeyPlaceholder: "AWS Access Key",
|
||||
s3SecretKeyPlaceholder: "AWS Secret Key",
|
||||
ossTitle: "阿里云 OSS",
|
||||
ossDesc: "阿里云对象存储服务(OSS),适合公有云部署。",
|
||||
ossAccessKeyPlaceholder: "阿里云 Access Key",
|
||||
ossSecretKeyPlaceholder: "阿里云 Secret Key",
|
||||
console: "控制台",
|
||||
docs: "文档",
|
||||
testConnection: "测试连接",
|
||||
@@ -557,6 +562,7 @@ export default {
|
||||
requestFailed: "请求失败",
|
||||
cos: "腾讯云 COS",
|
||||
tos: "火山引擎 TOS",
|
||||
oss: "阿里云 OSS",
|
||||
},
|
||||
},
|
||||
webSearchSettings: {
|
||||
@@ -3115,6 +3121,8 @@ export default {
|
||||
engineTosDesc: "火山引擎对象存储,适合公有云部署",
|
||||
engineS3: "AWS S3",
|
||||
engineS3Desc: "AWS S3 及兼容存储,适合公有云部署",
|
||||
engineOss: "阿里云 OSS",
|
||||
engineOssDesc: "阿里云对象存储,适合公有云部署",
|
||||
},
|
||||
parser: {
|
||||
title: "解析引擎",
|
||||
|
||||
@@ -41,7 +41,7 @@ const DOMPurifyConfig = {
|
||||
'display', 'pointer-events', 'cursor', 'data-emit', 'direction'
|
||||
],
|
||||
// 允许的协议
|
||||
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
|
||||
ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|(?:local|minio|cos|tos|s3|oss):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i,
|
||||
// 禁止的标签和属性
|
||||
FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'],
|
||||
FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'],
|
||||
@@ -119,7 +119,7 @@ function protectProviderImageSrcInHTML(html: string): string {
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"');
|
||||
return html.replace(
|
||||
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi,
|
||||
/<img\b([^>]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi,
|
||||
(_m, before, quote, provider, restPathRaw, after) => {
|
||||
const restPath = decodeProviderURL(restPathRaw);
|
||||
const protectedSrc = `${provider}://${restPath}`;
|
||||
@@ -173,7 +173,7 @@ export function isValidURL(url: string): boolean {
|
||||
}
|
||||
|
||||
// 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL
|
||||
if (/^(local|minio|cos|tos|s3):\/\/\S+$/i.test(trimmed)) {
|
||||
if (/^(local|minio|cos|tos|s3|oss):\/\/\S+$/i.test(trimmed)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -299,7 +299,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
|
||||
}
|
||||
|
||||
const images = root.querySelectorAll<HTMLImageElement>(
|
||||
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"]',
|
||||
'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"]',
|
||||
);
|
||||
if (!images.length) {
|
||||
return;
|
||||
@@ -319,7 +319,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef
|
||||
}
|
||||
img.dataset.authHydrated = '1';
|
||||
|
||||
const isProviderScheme = /^(local|minio|cos|tos|s3):\/\//.test(sourceURL);
|
||||
const isProviderScheme = /^(local|minio|cos|tos|s3|oss):\/\//.test(sourceURL);
|
||||
const requestURL = isProviderScheme
|
||||
? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}`
|
||||
: sourceURL;
|
||||
|
||||
@@ -1266,6 +1266,7 @@ const imageStorageOptions = computed(() => {
|
||||
{ value: 'cos', label: t('settings.storage.engineCos'), disabled: statusMap.cos === false },
|
||||
{ value: 'tos', label: t('settings.storage.engineTos'), disabled: statusMap.tos === false },
|
||||
{ value: 's3', label: 'Amazon S3', disabled: statusMap.s3 === false },
|
||||
{ value: 'oss', label: t('settings.storage.engineOss'), disabled: statusMap.oss === false },
|
||||
];
|
||||
});
|
||||
|
||||
|
||||
@@ -114,6 +114,13 @@ const engineOptions = computed(() => {
|
||||
available: statusMap.s3,
|
||||
disabled: statusMap.s3 === false,
|
||||
},
|
||||
{
|
||||
value: 'oss',
|
||||
label: t('kbSettings.storage.engineOss'),
|
||||
desc: t('kbSettings.storage.engineOssDesc'),
|
||||
available: statusMap.oss,
|
||||
disabled: statusMap.oss === false,
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
|
||||
@@ -201,6 +201,7 @@ const navItems = computed(() => [
|
||||
{ key: 'cos', label: t('settings.storage.cos') },
|
||||
{ key: 'tos', label: t('settings.storage.tos') },
|
||||
{ key: 's3', label: 'AWS S3' },
|
||||
{ key: 'oss', label: t('settings.storage.oss') },
|
||||
]
|
||||
},
|
||||
{ key: 'mcp', icon: 'tools', label: t('settings.mcpService') },
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
<t-option value="cos" :label="$t('settings.storage.engineCos')" />
|
||||
<t-option value="tos" :label="$t('settings.storage.engineTos')" />
|
||||
<t-option value="s3" label="AWS S3" />
|
||||
<t-option value="oss" :label="$t('settings.storage.engineOss')" />
|
||||
</t-select>
|
||||
</div>
|
||||
</div>
|
||||
@@ -422,6 +423,80 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- OSS -->
|
||||
<div class="engine-section" data-model-type="oss">
|
||||
<div class="engine-header">
|
||||
<div class="engine-header-info">
|
||||
<div class="engine-title-row">
|
||||
<h3>{{ $t('settings.storage.ossTitle') }}</h3>
|
||||
<t-tag theme="success" variant="light" size="small">{{ $t('settings.storage.configurable') }}</t-tag>
|
||||
</div>
|
||||
<p>
|
||||
{{ $t('settings.storage.ossDesc') }}
|
||||
<a class="engine-link" href="https://oss.console.aliyun.com/" target="_blank" rel="noopener">{{ $t('settings.storage.console') }} ↗</a>
|
||||
<a class="engine-link" href="https://help.aliyun.com/zh/oss/" target="_blank" rel="noopener">{{ $t('settings.storage.docs') }} ↗</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="engine-form">
|
||||
<div class="form-field">
|
||||
<label>Endpoint</label>
|
||||
<t-input
|
||||
v-model="config.oss.endpoint"
|
||||
placeholder="e.g. https://oss-cn-hangzhou.aliyuncs.com"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="form-field">
|
||||
<label>Region</label>
|
||||
<t-input
|
||||
v-model="config.oss.region"
|
||||
placeholder="e.g. cn-hangzhou"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="form-field">
|
||||
<label>Access Key</label>
|
||||
<t-input
|
||||
v-model="config.oss.access_key"
|
||||
:placeholder="$t('settings.storage.ossAccessKeyPlaceholder')"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="form-field">
|
||||
<label>Secret Key</label>
|
||||
<t-input
|
||||
v-model="config.oss.secret_key"
|
||||
type="password"
|
||||
:placeholder="$t('settings.storage.ossSecretKeyPlaceholder')"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="form-field">
|
||||
<label>{{ $t('settings.storage.bucketName') }}</label>
|
||||
<t-input
|
||||
v-model="config.oss.bucket_name"
|
||||
:placeholder="$t('settings.storage.bucketPlaceholder')"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
<div class="form-field">
|
||||
<label>{{ $t('settings.storage.pathPrefix') }}</label>
|
||||
<t-input
|
||||
v-model="config.oss.path_prefix"
|
||||
:placeholder="$t('settings.storage.prefixPlaceholder')"
|
||||
clearable
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="test-bar">
|
||||
<t-button size="small" variant="outline" :loading="checkingOss" @click="onCheckOss">{{ $t('settings.storage.testConnection') }}</t-button>
|
||||
<span v-if="ossCheckResult" :class="['test-msg', ossCheckResult.ok ? 'success' : 'error']">
|
||||
{{ ossCheckResult.message }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Save -->
|
||||
<div class="save-bar">
|
||||
<t-button theme="primary" :loading="saving" @click="onSave">{{ $t('settings.storage.saveConfig') }}</t-button>
|
||||
@@ -476,6 +551,17 @@ const defaultConfig = (): StorageEngineConfig => ({
|
||||
bucket_name: '',
|
||||
path_prefix: '',
|
||||
},
|
||||
oss: {
|
||||
endpoint: '',
|
||||
region: '',
|
||||
access_key: '',
|
||||
secret_key: '',
|
||||
bucket_name: '',
|
||||
path_prefix: '',
|
||||
use_temp_bucket: false,
|
||||
temp_bucket_name: '',
|
||||
temp_region: '',
|
||||
},
|
||||
})
|
||||
|
||||
const loading = ref(true)
|
||||
@@ -501,6 +587,8 @@ const checkingTos = ref(false)
|
||||
const tosCheckResult = ref<{ ok: boolean; message: string } | null>(null)
|
||||
const checkingS3 = ref(false)
|
||||
const s3CheckResult = ref<{ ok: boolean; message: string } | null>(null)
|
||||
const checkingOss = ref(false)
|
||||
const ossCheckResult = ref<{ ok: boolean; message: string } | null>(null)
|
||||
|
||||
const minioAvailable = computed(() => {
|
||||
if (config.value.minio?.mode === 'remote') {
|
||||
@@ -558,6 +646,19 @@ async function loadConfig() {
|
||||
path_prefix: d.s3.path_prefix || '',
|
||||
}
|
||||
: defaultConfig().s3!,
|
||||
oss: d.oss
|
||||
? {
|
||||
endpoint: d.oss.endpoint || '',
|
||||
region: d.oss.region || '',
|
||||
access_key: d.oss.access_key || '',
|
||||
secret_key: d.oss.secret_key || '',
|
||||
bucket_name: d.oss.bucket_name || '',
|
||||
path_prefix: d.oss.path_prefix || '',
|
||||
use_temp_bucket: d.oss.use_temp_bucket ?? false,
|
||||
temp_bucket_name: d.oss.temp_bucket_name || '',
|
||||
temp_region: d.oss.temp_region || '',
|
||||
}
|
||||
: defaultConfig().oss!,
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -649,6 +750,18 @@ function buildPayload(): StorageEngineConfig {
|
||||
bucket_name: (config.value.s3?.bucket_name || '').trim(),
|
||||
path_prefix: (config.value.s3?.path_prefix || '').trim(),
|
||||
},
|
||||
oss: {
|
||||
endpoint: (config.value.oss?.endpoint || '').trim(),
|
||||
region: (config.value.oss?.region || '').trim(),
|
||||
access_key: (config.value.oss?.access_key || '').trim(),
|
||||
secret_key: (config.value.oss?.secret_key || '').trim(),
|
||||
bucket_name: (config.value.oss?.bucket_name || '').trim(),
|
||||
path_prefix: (config.value.oss?.path_prefix || '').trim(),
|
||||
// Temp bucket fields: not exposed in UI; server manages these independently
|
||||
use_temp_bucket: config.value.oss?.use_temp_bucket ?? false,
|
||||
temp_bucket_name: (config.value.oss?.temp_bucket_name || '').trim(),
|
||||
temp_region: (config.value.oss?.temp_region || '').trim(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -728,6 +841,20 @@ async function onCheckS3() {
|
||||
}
|
||||
}
|
||||
|
||||
async function onCheckOss() {
|
||||
checkingOss.value = true
|
||||
ossCheckResult.value = null
|
||||
try {
|
||||
const payload = buildPayload()
|
||||
const res = await checkStorageEngine({ provider: 'oss', oss: payload.oss })
|
||||
ossCheckResult.value = res?.data ?? { ok: false, message: t('settings.storage.unknownError') }
|
||||
} catch (e: unknown) {
|
||||
ossCheckResult.value = { ok: false, message: e instanceof Error ? e.message : t('settings.storage.requestFailed') }
|
||||
} finally {
|
||||
checkingOss.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(loadAll)
|
||||
</script>
|
||||
|
||||
@@ -872,7 +999,7 @@ onMounted(loadAll)
|
||||
label {
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
color: var(--td-text-color-secondary)555;
|
||||
color: var(--td-text-color-secondary);
|
||||
}
|
||||
|
||||
&--inline {
|
||||
|
||||
@@ -101,6 +101,30 @@ func NewFileServiceFromStorageConfig(
|
||||
svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix)
|
||||
return svc, p, err
|
||||
|
||||
case "oss":
|
||||
if sec == nil || sec.OSS == nil || sec.OSS.Endpoint == "" || sec.OSS.Region == "" || sec.OSS.AccessKey == "" || sec.OSS.SecretKey == "" || sec.OSS.BucketName == "" {
|
||||
return nil, p, fmt.Errorf("incomplete oss config")
|
||||
}
|
||||
pathPrefix := strings.TrimSpace(sec.OSS.PathPrefix)
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
var svc interfaces.FileService
|
||||
var err error
|
||||
if sec.OSS.UseTempBucket && sec.OSS.TempBucketName != "" {
|
||||
svc, err = NewOssFileServiceWithTempBucket(
|
||||
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
|
||||
sec.OSS.BucketName, pathPrefix,
|
||||
sec.OSS.TempBucketName, sec.OSS.TempRegion,
|
||||
)
|
||||
} else {
|
||||
svc, err = NewOssFileService(
|
||||
sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey,
|
||||
sec.OSS.BucketName, pathPrefix,
|
||||
)
|
||||
}
|
||||
return svc, p, err
|
||||
|
||||
default:
|
||||
return nil, p, fmt.Errorf("unsupported provider %q", p)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// ossFileService implements the FileService interface for Aliyun OSS
|
||||
// using the S3-compatible protocol with virtual-hosted style addressing.
|
||||
type ossFileService struct {
|
||||
client *s3.Client
|
||||
tempClient *s3.Client
|
||||
pathPrefix string
|
||||
bucketName string
|
||||
tempBucketName string
|
||||
}
|
||||
|
||||
const ossScheme = "oss://"
|
||||
|
||||
// newOSSClient creates a bare s3.Client configured for OSS S3-compatible mode.
|
||||
// OSS uses virtual-hosted style addressing and does not support aws-chunked encoding.
|
||||
func newOSSClient(endpoint, region, accessKey, secretKey string) (*s3.Client, error) {
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
// Disable automatic aws-chunked encoding — OSS does not support it.
|
||||
config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
|
||||
config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = false
|
||||
})
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// ossBucketExists checks if the bucket exists using the provided client.
|
||||
func ossBucketExists(ctx context.Context, client *s3.Client, bucketName string) (bool, error) {
|
||||
_, err := client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// ossCreateBucket creates a new bucket using the provided client.
|
||||
func ossCreateBucket(ctx context.Context, client *s3.Client, bucketName string) error {
|
||||
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// ossEnsureBucket checks if the bucket exists and creates it if missing.
|
||||
func ossEnsureBucket(ctx context.Context, client *s3.Client, bucketName string) error {
|
||||
exists, err := ossBucketExists(ctx, client, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := ossCreateBucket(ctx, client, bucketName); err != nil {
|
||||
return fmt.Errorf("failed to create bucket: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewOssFileService creates an Aliyun OSS file service.
|
||||
// It verifies that the bucket exists and creates it if missing.
|
||||
func NewOssFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) {
|
||||
return NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewOssFileServiceWithTempBucket creates an Aliyun OSS file service with optional temp bucket.
|
||||
func NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ossEnsureBucket(context.Background(), client, bucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tempClient *s3.Client
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region
|
||||
}
|
||||
tempClient, err = newOSSClient(endpoint, tempRegion, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize OSS temp client: %w", err)
|
||||
}
|
||||
if err := ossEnsureBucket(context.Background(), tempClient, tempBucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize pathPrefix: ensure it ends with '/' if not empty
|
||||
if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") {
|
||||
pathPrefix += "/"
|
||||
}
|
||||
|
||||
return &ossFileService{
|
||||
client: client,
|
||||
tempClient: tempClient,
|
||||
pathPrefix: pathPrefix,
|
||||
bucketName: bucketName,
|
||||
tempBucketName: tempBucketName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CheckOssConnectivity tests OSS connectivity using the provided credentials.
|
||||
func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
|
||||
client, err := newOSSClient(endpoint, region, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
exists, err := ossBucketExists(checkCtx, client, bucketName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check bucket: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseOssFilePath extracts bucket and object key from: oss://{bucket}/{objectKey}
|
||||
func parseOssFilePath(filePath string) (bucketName string, objectKey string, err error) {
|
||||
if !strings.HasPrefix(filePath, ossScheme) {
|
||||
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
|
||||
}
|
||||
|
||||
rest := strings.TrimPrefix(filePath, ossScheme)
|
||||
parts := strings.SplitN(rest, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("invalid OSS file path: %s", filePath)
|
||||
}
|
||||
return parts[0], parts[1], nil
|
||||
}
|
||||
|
||||
// CheckConnectivity verifies OSS is reachable and the main bucket exists.
|
||||
func (s *ossFileService) CheckConnectivity(ctx context.Context) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
exists, err := ossBucketExists(checkCtx, s.client, s.bucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to OSS.
|
||||
func (s *ossFileService) SaveFile(ctx context.Context,
|
||||
file *multipart.FileHeader, tenantID uint64, knowledgeID string,
|
||||
) (string, error) {
|
||||
ext := filepath.Ext(file.Filename)
|
||||
objectName := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
|
||||
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
contentType := file.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = getContentTypeByExt(ext)
|
||||
}
|
||||
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
Body: src,
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to OSS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to OSS.
|
||||
// If temp is true and temp bucket is configured, saves to temp bucket.
|
||||
// Otherwise saves to main bucket.
|
||||
func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
safeName, err := utils.SafeFileName(fileName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid file name: %w", err)
|
||||
}
|
||||
ext := filepath.Ext(safeName)
|
||||
|
||||
// If requesting temp bucket and it is configured, use it
|
||||
if temp && s.tempClient != nil {
|
||||
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
_, err := s.tempClient.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.tempBucketName),
|
||||
Key: aws.String(objectName),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: aws.String("text/csv; charset=utf-8"),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to OSS temp bucket: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("oss://%s/%s", s.tempBucketName, objectName), nil
|
||||
}
|
||||
|
||||
// Save to main bucket
|
||||
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucketName),
|
||||
Key: aws.String(objectName),
|
||||
Body: bytes.NewReader(data),
|
||||
ContentType: aws.String("text/csv; charset=utf-8"),
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to OSS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from OSS by its path.
|
||||
func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// DeleteFile removes a file from OSS.
|
||||
func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectName),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete file from OSS: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file.
|
||||
func (s *ossFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
bucketName, objectName, err := parseOssFilePath(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := utils.SafeObjectKey(objectName); err != nil {
|
||||
return "", fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
// Determine which client to use
|
||||
var client *s3.Client
|
||||
if bucketName == s.tempBucketName && s.tempClient != nil {
|
||||
client = s.tempClient
|
||||
} else {
|
||||
client = s.client
|
||||
}
|
||||
|
||||
presignClient := s3.NewPresignClient(client)
|
||||
|
||||
presignedReq, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectName),
|
||||
}, s3.WithPresignExpires(24*time.Hour))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate OSS presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedReq.URL, nil
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseOssFilePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantBucket string
|
||||
wantKey string
|
||||
wantErr bool
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "valid path with nested key",
|
||||
input: "oss://my-bucket/123/exports/abc123.csv",
|
||||
wantBucket: "my-bucket",
|
||||
wantKey: "123/exports/abc123.csv",
|
||||
},
|
||||
{
|
||||
name: "valid path with simple key",
|
||||
input: "oss://test-bucket/key",
|
||||
wantBucket: "test-bucket",
|
||||
wantKey: "key",
|
||||
},
|
||||
{
|
||||
name: "valid path with deep nesting",
|
||||
input: "oss://bucket/prefix/tenant/exports/uuid.png",
|
||||
wantBucket: "bucket",
|
||||
wantKey: "prefix/tenant/exports/uuid.png",
|
||||
},
|
||||
{
|
||||
name: "invalid scheme",
|
||||
input: "s3://bucket/key",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "bucket only no key",
|
||||
input: "oss://bucket/",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "scheme only",
|
||||
input: "oss://",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "no slash after bucket",
|
||||
input: "oss://bucket",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
{
|
||||
name: "empty bucket name",
|
||||
input: "oss:///some-key",
|
||||
wantErr: true,
|
||||
errContains: "invalid OSS file path",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
bucket, key, err := parseOssFilePath(tt.input)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("parseOssFilePath(%q) expected error, got bucket=%q key=%q", tt.input, bucket, key)
|
||||
}
|
||||
if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) {
|
||||
t.Errorf("parseOssFilePath(%q) error = %v, want containing %q", tt.input, err, tt.errContains)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("parseOssFilePath(%q) unexpected error: %v", tt.input, err)
|
||||
return
|
||||
}
|
||||
if bucket != tt.wantBucket {
|
||||
t.Errorf("parseOssFilePath(%q) bucket = %q, want %q", tt.input, bucket, tt.wantBucket)
|
||||
}
|
||||
if key != tt.wantKey {
|
||||
t.Errorf("parseOssFilePath(%q) key = %q, want %q", tt.input, key, tt.wantKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewOSSClient(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
region string
|
||||
accessKey string
|
||||
secretKey string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid parameters create client",
|
||||
endpoint: "https://oss-cn-hangzhou.aliyuncs.com",
|
||||
region: "cn-hangzhou",
|
||||
accessKey: "test-access-key",
|
||||
secretKey: "test-secret-key",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "custom endpoint",
|
||||
endpoint: "https://custom-oss-endpoint.com",
|
||||
region: "cn-shanghai",
|
||||
accessKey: "ak",
|
||||
secretKey: "sk",
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client, err := newOSSClient(tt.endpoint, tt.region, tt.accessKey, tt.secretKey)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Error("expected error but got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("newOSSClient() unexpected error: %v", err)
|
||||
return
|
||||
}
|
||||
if client == nil {
|
||||
t.Error("expected non-nil client")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Should fail with an invalid/unreachable endpoint
|
||||
err := CheckOssConnectivity(ctx,
|
||||
"https://invalid-oss-endpoint-that-does-not-exist.local",
|
||||
"cn-hangzhou",
|
||||
"invalid-access-key",
|
||||
"invalid-secret-key",
|
||||
"nonexistent-bucket",
|
||||
)
|
||||
|
||||
if err == nil {
|
||||
t.Error("CheckOssConnectivity with invalid endpoint should return an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOssBucketExists_NonExistent(t *testing.T) {
|
||||
client, err := newOSSClient(
|
||||
"https://oss-cn-hangzhou.aliyuncs.com",
|
||||
"cn-hangzhou",
|
||||
"test-invalid-key",
|
||||
"test-invalid-secret",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newOSSClient() error: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Bucket that definitely doesn't exist - should return false, not error
|
||||
// (auth may fail first, but we verify the function doesn't crash)
|
||||
exists, err := ossBucketExists(ctx, client, "this-bucket-definitely-does-not-exist-12345")
|
||||
// We don't assert on the result because it depends on network/auth,
|
||||
// but the function should not panic or hang.
|
||||
_ = exists
|
||||
_ = err
|
||||
}
|
||||
|
||||
func TestOssCreateBucket(t *testing.T) {
|
||||
client, err := newOSSClient(
|
||||
"https://oss-cn-hangzhou.aliyuncs.com",
|
||||
"cn-hangzhou",
|
||||
"test-invalid-key",
|
||||
"test-invalid-secret",
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("newOSSClient() error: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Should fail with invalid credentials
|
||||
err = ossCreateBucket(ctx, client, "test-bucket")
|
||||
if err == nil {
|
||||
t.Error("ossCreateBucket with invalid credentials should return an error")
|
||||
}
|
||||
}
|
||||
@@ -647,6 +647,28 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
os.Getenv("S3_REGION"),
|
||||
pathPrefix,
|
||||
)
|
||||
case "oss":
|
||||
if os.Getenv("OSS_ENDPOINT") == "" ||
|
||||
os.Getenv("OSS_REGION") == "" ||
|
||||
os.Getenv("OSS_ACCESS_KEY") == "" ||
|
||||
os.Getenv("OSS_SECRET_KEY") == "" ||
|
||||
os.Getenv("OSS_BUCKET_NAME") == "" {
|
||||
return nil, fmt.Errorf("missing OSS configuration")
|
||||
}
|
||||
pathPrefix := os.Getenv("OSS_PATH_PREFIX")
|
||||
if pathPrefix == "" {
|
||||
pathPrefix = "weknora/"
|
||||
}
|
||||
return file.NewOssFileServiceWithTempBucket(
|
||||
os.Getenv("OSS_ENDPOINT"),
|
||||
os.Getenv("OSS_REGION"),
|
||||
os.Getenv("OSS_ACCESS_KEY"),
|
||||
os.Getenv("OSS_SECRET_KEY"),
|
||||
os.Getenv("OSS_BUCKET_NAME"),
|
||||
pathPrefix,
|
||||
os.Getenv("OSS_TEMP_BUCKET_NAME"),
|
||||
os.Getenv("OSS_TEMP_REGION"),
|
||||
)
|
||||
case "local":
|
||||
baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR")
|
||||
if baseDir == "" {
|
||||
|
||||
@@ -396,6 +396,17 @@ func (h *SystemHandler) isTOSConfigured(c *gin.Context) bool {
|
||||
return h.isTOSEnvAvailable()
|
||||
}
|
||||
|
||||
// isOSSConfigured checks whether OSS connection info is available from tenant config.
|
||||
func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool {
|
||||
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
|
||||
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.OSS != nil {
|
||||
ossConf := tenant.StorageEngineConfig.OSS
|
||||
return ossConf.Endpoint != "" && ossConf.Region != "" && ossConf.AccessKey != "" && ossConf.SecretKey != "" && ossConf.BucketName != ""
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isTOSEnvAvailable checks whether TOS env vars are set.
|
||||
func (h *SystemHandler) isTOSEnvAvailable() bool {
|
||||
return os.Getenv("TOS_ENDPOINT") != "" &&
|
||||
@@ -442,11 +453,13 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
|
||||
minioEnvAvailable := h.isMinioEnvAvailable()
|
||||
cosConfigured := h.isCOSConfigured(c)
|
||||
tosConfigured := h.isTOSConfigured(c)
|
||||
ossConfigured := h.isOSSConfigured(c)
|
||||
engines := []StorageEngineStatusItem{
|
||||
{Name: "local", Available: true, Description: "本地文件系统存储,仅适合单机部署"},
|
||||
{Name: "minio", Available: minioConfigured || minioEnvAvailable, Description: "S3 兼容的自托管对象存储,适合内网和私有云部署"},
|
||||
{Name: "cos", Available: cosConfigured, Description: "腾讯云对象存储服务,适合公有云部署,支持 CDN 加速"},
|
||||
{Name: "tos", Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"},
|
||||
{Name: "oss", Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"},
|
||||
}
|
||||
c.JSON(200, gin.H{
|
||||
"code": 0,
|
||||
@@ -640,6 +653,9 @@ func hasGetObjectAction(action interface{}) bool {
|
||||
// cosFieldPattern validates COS region and bucket name format to prevent URL injection.
|
||||
var cosFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
|
||||
|
||||
// ossFieldPattern validates OSS region and bucket name format to prevent URL injection.
|
||||
var ossFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
|
||||
|
||||
// sanitizeStorageCheckError converts a raw storage connectivity error into a safe
|
||||
// user-facing message that does not leak internal network details (hostnames, IPs, ports).
|
||||
func sanitizeStorageCheckError(err error) string {
|
||||
@@ -727,11 +743,12 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) {
|
||||
|
||||
// StorageCheckRequest is the body for POST /system/storage-engine-check.
|
||||
type StorageCheckRequest struct {
|
||||
Provider string `json:"provider"` // "minio", "cos", "tos", or "s3"
|
||||
Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss"
|
||||
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
|
||||
COS *types.COSEngineConfig `json:"cos,omitempty"`
|
||||
TOS *types.TOSEngineConfig `json:"tos,omitempty"`
|
||||
S3 *types.S3EngineConfig `json:"s3,omitempty"`
|
||||
OSS *types.OSSEngineConfig `json:"oss,omitempty"`
|
||||
}
|
||||
|
||||
// StorageCheckResponse is the response for a single-engine connectivity check.
|
||||
@@ -768,6 +785,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
|
||||
h.checkTOS(c, ctx, req.TOS)
|
||||
case "s3":
|
||||
h.checkS3(c, ctx, req.S3)
|
||||
case "oss":
|
||||
h.checkOSS(c, ctx, req.OSS)
|
||||
default:
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
|
||||
}
|
||||
@@ -960,6 +979,55 @@ func (h *SystemHandler) checkS3(c *gin.Context, ctx context.Context, cfg *types.
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
|
||||
}
|
||||
|
||||
func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types.OSSEngineConfig) {
|
||||
if cfg == nil {
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 OSS 配置"}})
|
||||
return
|
||||
}
|
||||
|
||||
endpoint, accessKey, secretKey := cfg.Endpoint, cfg.AccessKey, cfg.SecretKey
|
||||
if endpoint == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" {
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Access Key、Secret Key、Bucket Name 不能为空"}})
|
||||
return
|
||||
}
|
||||
|
||||
// Strip URL scheme before SSRF check — OSS endpoint may include http:// or https://
|
||||
ssrfEndpoint := strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
|
||||
if blocked, reason := isBlockedStorageEndpoint(ssrfEndpoint); blocked {
|
||||
logger.Warnf(ctx, "Storage check: OSS endpoint blocked by SSRF protection", "endpoint", endpoint)
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
|
||||
return
|
||||
}
|
||||
if !ossFieldPattern.MatchString(cfg.Region) {
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Region 格式不正确,仅允许字母、数字、点、连字符"}})
|
||||
return
|
||||
}
|
||||
if !ossFieldPattern.MatchString(cfg.BucketName) {
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Bucket 名称格式不正确,仅允许字母、数字、点、连字符"}})
|
||||
return
|
||||
}
|
||||
|
||||
err := file.CheckOssConnectivity(ctx, endpoint, cfg.Region, accessKey, secretKey, cfg.BucketName)
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") {
|
||||
logger.Errorf(ctx, "Storage check: OSS auth failed", "endpoint", endpoint, "bucket", cfg.BucketName)
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}})
|
||||
return
|
||||
}
|
||||
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
|
||||
logger.Errorf(ctx, "Storage check: OSS bucket not found", "bucket", cfg.BucketName)
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在", cfg.BucketName)}})
|
||||
return
|
||||
}
|
||||
logger.Errorf(ctx, "Storage check: OSS connectivity failed", "endpoint", endpoint, "bucket", cfg.BucketName, "error", err)
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("OSS 连通性检测失败: %s", sanitizeStorageCheckError(err))}})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
|
||||
}
|
||||
|
||||
func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader {
|
||||
if addr == "" {
|
||||
return h.documentReader
|
||||
|
||||
@@ -152,7 +152,7 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string {
|
||||
// StorageProviderConfig stores the KB-level storage provider selection.
|
||||
// Credentials are managed at the tenant level (StorageEngineConfig).
|
||||
type StorageProviderConfig struct {
|
||||
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos"
|
||||
Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss"
|
||||
}
|
||||
|
||||
func (c StorageProviderConfig) Value() (driver.Value, error) {
|
||||
@@ -266,7 +266,7 @@ func InferStorageFromFilePath(filePath string) string {
|
||||
// e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local"
|
||||
// Returns "" if the path does not use a known provider scheme.
|
||||
func ParseProviderScheme(filePath string) string {
|
||||
for _, provider := range []string{"local", "minio", "cos", "tos", "s3"} {
|
||||
for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss"} {
|
||||
if strings.HasPrefix(filePath, provider+"://") {
|
||||
return provider
|
||||
}
|
||||
|
||||
@@ -328,15 +328,16 @@ func (c *ParserEngineConfig) Scan(value interface{}) error {
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, and S3.
|
||||
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, and OSS.
|
||||
// Knowledge bases select which provider to use; parameters are read from here.
|
||||
type StorageEngineConfig struct {
|
||||
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3"
|
||||
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss"
|
||||
Local *LocalEngineConfig `json:"local,omitempty"`
|
||||
MinIO *MinIOEngineConfig `json:"minio,omitempty"`
|
||||
COS *COSEngineConfig `json:"cos,omitempty"`
|
||||
TOS *TOSEngineConfig `json:"tos,omitempty"`
|
||||
S3 *S3EngineConfig `json:"s3,omitempty"`
|
||||
OSS *OSSEngineConfig `json:"oss,omitempty"`
|
||||
}
|
||||
|
||||
// LocalEngineConfig is for local file system storage (single-machine deployment only).
|
||||
@@ -386,6 +387,19 @@ type S3EngineConfig struct {
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
}
|
||||
|
||||
// OSSEngineConfig is for Alibaba Cloud OSS (对象存储服务).
|
||||
type OSSEngineConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Region string `json:"region"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
UseTempBucket bool `json:"use_temp_bucket"`
|
||||
TempBucketName string `json:"temp_bucket_name"`
|
||||
TempRegion string `json:"temp_region"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface for StorageEngineConfig
|
||||
func (c *StorageEngineConfig) Value() (driver.Value, error) {
|
||||
if c == nil {
|
||||
|
||||
Reference in New Issue
Block a user