diff --git a/README.md b/README.md index e42816b3..81d0e268 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Fully modular pipeline from document parsing, vectorization, and retrieval to LL | LLMs | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama | | Embeddings | Ollama / BGE / GTE / OpenAI-compatible APIs | | Vector DBs | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant | -| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS | +| Object Storage | Local / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS | | IM Channels | WeCom / Feishu / Slack / Telegram / DingTalk / Mattermost | | Web Search | DuckDuckGo / Bing / Google / Tavily | diff --git a/README_CN.md b/README_CN.md index 61900914..3cabb884 100644 --- a/README_CN.md +++ b/README_CN.md @@ -180,7 +180,7 @@ WeKnora 提供**快速问答**与**智能推理**两种问答模式。快速问 |------|------| | 模型厂商 | OpenAI / DeepSeek / Qwen(阿里云)/ 智谱 / 混元 / 豆包(火山引擎)/ Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama | | 向量数据库 | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant | -| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 | +| 对象存储 | 本地 / 腾讯云COS / 火山引擎 TOS / MinIO / AWS S3 / 阿里云 OSS | | IM 集成 | 企业微信 / 飞书 / Slack / Telegram / 钉钉 / Mattermost | | 网络搜索 | DuckDuckGo / Bing / Google / Tavily | diff --git a/README_JA.md b/README_JA.md index 875a8e2a..6375076e 100644 --- a/README_JA.md +++ b/README_JA.md @@ -182,7 +182,7 @@ Feishuなどの外部プラットフォームからのナレッジ自動同期 | 大規模モデル | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama | | Embedding | Ollama / BGE / GTE / OpenAI 互換 API | | ベクトル DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant | -| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS | +| オブジェクトストレージ | ローカル / MinIO / AWS S3 / 火山引擎 TOS / Alibaba Cloud OSS | | IM 統合 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost | | Web 検索 | DuckDuckGo / Bing / Google / Tavily | diff --git a/README_KO.md b/README_KO.md index e7a8b612..44e0a91d 100644 --- a/README_KO.md +++ b/README_KO.md @@ -191,7 +191,7 @@ Feishu 등 외부 플랫폼에서 지식 자동 동기화를 지원하며(추가 | LLM | OpenAI / DeepSeek / Qwen (Alibaba Cloud) / Zhipu / Hunyuan / Doubao (Volcengine) / Gemini / MiniMax / NVIDIA / Novita AI / SiliconFlow / OpenRouter / Ollama | | Embedding | Ollama / BGE / GTE / OpenAI 호환 API | | 벡터 DB | PostgreSQL (pgvector) / Elasticsearch / Milvus / Weaviate / Qdrant | -| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS | +| 오브젝트 스토리지 | 로컬 / MinIO / AWS S3 / Volcengine TOS / Alibaba Cloud OSS | | IM 통합 | WeChat Work / Feishu / Slack / Telegram / DingTalk / Mattermost | | 웹 검색 | DuckDuckGo / Bing / Google / Tavily | diff --git a/docreader/README.md b/docreader/README.md index 071e2e5b..19f49541 100644 --- a/docreader/README.md +++ b/docreader/README.md @@ -126,6 +126,16 @@ DocReader 支持多种存储后端: - `COS_PATH_PREFIX`: 文件路径前缀 - `COS_ENABLE_OLD_DOMAIN`: 是否使用旧域名(默认:true) +#### 阿里云 OSS 存储 + +- `STORAGE_TYPE`: 设置为 `oss` +- `OSS_ACCESS_KEY_ID`: OSS 访问密钥 ID +- `OSS_ACCESS_KEY_SECRET`: OSS 访问密钥 +- `OSS_ENDPOINT`: OSS 端点(如 `oss-cn-hangzhou.aliyuncs.com`) +- `OSS_BUCKET_NAME`: OSS 存储桶名称 +- `OSS_REGION`: OSS 区域(如 `cn-hangzhou`) +- `OSS_PATH_PREFIX`: 文件路径前缀 + ### 代理配置 如果需要通过代理访问外部服务: @@ -178,6 +188,20 @@ docreader: - MAX_FILE_SIZE_MB=50 ``` +### 使用阿里云 OSS + +```yaml +docreader: + environment: + - STORAGE_TYPE=oss + - OSS_ACCESS_KEY_ID=your_access_key_id + - OSS_ACCESS_KEY_SECRET=your_access_key_secret + - OSS_ENDPOINT=oss-cn-hangzhou.aliyuncs.com + - OSS_BUCKET_NAME=your-bucket + - OSS_REGION=cn-hangzhou + - MAX_FILE_SIZE_MB=50 +``` + ## 常见问题 ### 1. DocReader 服务无法启动? diff --git a/docreader/parser/storage.py b/docreader/parser/storage.py index 7f62896f..6e97838b 100644 --- a/docreader/parser/storage.py +++ b/docreader/parser/storage.py @@ -10,6 +10,11 @@ from typing import Dict, Optional from minio import Minio from qcloud_cos import CosConfig, CosS3Client +try: + import oss2 +except ImportError: + oss2 = None # type: ignore[assignment] + from docreader.utils import endecode logger = logging.getLogger(__name__) @@ -232,6 +237,99 @@ class MinioStorage(Storage): return "" +class OssStorage(Storage): + """Alibaba Cloud OSS storage implementation""" + + def __init__(self, storage_config: Optional[Dict] = None): + self.storage_config = storage_config + self.bucket, self.endpoint, self.prefix = self._init_oss_bucket() + + def _init_oss_bucket(self): + if oss2 is None: + logger.error("oss2 package not installed. Install with: pip install oss2") + return None, None, None + try: + sc = self.storage_config + access_key = _cfg(sc, "access_key", "OSS_ACCESS_KEY") + secret_key = _cfg(sc, "secret_key", "OSS_SECRET_KEY") + endpoint_raw = _cfg(sc, "endpoint", "OSS_ENDPOINT") + bucket_name = _cfg(sc, "bucket_name", "OSS_BUCKET_NAME") + prefix_raw = _cfg(sc, "path_prefix", "OSS_PATH_PREFIX") + + if not all([access_key, secret_key, endpoint_raw, bucket_name]): + logger.error( + "Incomplete OSS configuration: " + "access_key=%s, endpoint=%s, bucket=%s", + bool(access_key), endpoint_raw, bucket_name, + ) + return None, None, None + + # oss2 requires endpoint to include the scheme (http:// or https://) + if not endpoint_raw.startswith(("http://", "https://")): + endpoint_raw = f"https://{endpoint_raw}" + + auth = oss2.Auth(access_key, secret_key) + bucket = oss2.Bucket(auth, endpoint_raw, bucket_name) + + # Verify bucket exists; do NOT auto-create to avoid unintended public-read buckets + try: + bucket.get_bucket_info() + except oss2.exceptions.NoSuchBucket: + logger.error( + "OSS bucket %s does not exist. Please create it manually in the console.", + bucket_name, + ) + return None, None, None + + prefix = prefix_raw.strip().strip("/") if prefix_raw else "" + return bucket, endpoint_raw, prefix + + except Exception as e: + logger.error("Failed to initialize OSS client: %s", e) + return None, None, None + + def _get_download_url(self, object_key: str) -> str: + # Use virtual-hosted style URL: https://{bucket}.{endpoint_without_scheme}/{key} + endpoint_no_scheme = self.endpoint.removeprefix("https://").removeprefix("http://") + return f"https://{self.bucket.bucket_name}.{endpoint_no_scheme}/{object_key}" + + def upload_file(self, file_path: str) -> str: + try: + if not self.bucket: + return "" + file_ext = os.path.splitext(file_path)[1] + object_key = ( + f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" + if self.prefix + else f"images/{uuid.uuid4().hex}{file_ext}" + ) + self.bucket.put_object_from_file(object_key, file_path) + file_url = self._get_download_url(object_key) + logger.info("OSS upload_file ok: %s", file_url) + return file_url + except Exception as e: + logger.error("OSS upload_file failed: %s", e) + return "" + + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + try: + if not self.bucket: + return "" + object_key = ( + f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" + if self.prefix + else f"images/{uuid.uuid4().hex}{file_ext}" + ) + self.bucket.put_object(object_key, content) + file_url = self._get_download_url(object_key) + logger.info("OSS upload_bytes ok: %s", file_url) + return file_url + except Exception as e: + logger.error("OSS upload_bytes failed: %s", e) + traceback.print_exc() + return "" + + class LocalStorage(Storage): """Local file system storage implementation. @@ -297,7 +395,7 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage: """Create a storage instance based on storage_config dict. The ``provider`` key in storage_config determines the backend: - minio, cos, local, base64. + minio, cos, oss, local, base64. Falls back to STORAGE_TYPE env var, then ``local``. """ storage_type = "" @@ -315,6 +413,8 @@ def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage: return MinioStorage(storage_config) elif storage_type == "cos": return CosStorage(storage_config) + elif storage_type == "oss": + return OssStorage(storage_config) elif storage_type == "local": return LocalStorage(storage_config) elif storage_type == "base64": diff --git a/docreader/pyproject.toml b/docreader/pyproject.toml index bc6e846c..31005e91 100644 --- a/docreader/pyproject.toml +++ b/docreader/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "mistletoe>=1.5.0", "ollama>=0.6.0", "openai>=2.7.1", + "oss2>=2.18.0", "paddleocr>=2.10.0,<3.0.0", "paddlepaddle>=3.0.0,<4.0.0", "pdfplumber>=0.11.7", diff --git a/docreader/uv.lock b/docreader/uv.lock index 796919ff..3f138646 100644 --- a/docreader/uv.lock +++ b/docreader/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10.18" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'darwin'", @@ -59,6 +59,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/64/013409c451a44b61310fb757af4527f3de57fc98a00f40448de28b864290/albumentations-2.0.8-py3-none-any.whl", hash = "sha256:c4c4259aaf04a7386ad85c7fdcb73c6c7146ca3057446b745cc035805acb1017", size = 369423, upload-time = "2025-05-27T21:23:15.609Z" }, ] +[[package]] +name = "aliyun-python-sdk-core" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jmespath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/09/da9f58eb38b4fdb97ba6523274fbf445ef6a06be64b433693da8307b4bec/aliyun-python-sdk-core-2.16.0.tar.gz", hash = "sha256:651caad597eb39d4fad6cf85133dffe92837d53bdf62db9d8f37dab6508bb8f9", size = 449555, upload-time = "2024-10-09T06:01:01.762Z" } + +[[package]] +name = "aliyun-python-sdk-kms" +version = "2.16.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aliyun-python-sdk-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/2c/9877d0e6b18ecf246df671ac65a5d1d9fecbf85bdcb5d43efbde0d4662eb/aliyun-python-sdk-kms-2.16.5.tar.gz", hash = "sha256:f328a8a19d83ecbb965ffce0ec1e9930755216d104638cd95ecd362753b813b3", size = 12018, upload-time = "2024-08-30T09:01:20.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/5c/0132193d7da2c735669a1ed103b142fd63c9455984d48c5a88a1a516efaa/aliyun_python_sdk_kms-2.16.5-py2.py3-none-any.whl", hash = "sha256:24b6cdc4fd161d2942619479c8d050c63ea9cd22b044fe33b60bbb60153786f0", size = 99495, upload-time = "2024-08-30T09:01:18.462Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -690,6 +712,7 @@ dependencies = [ { name = "mistletoe" }, { name = "ollama" }, { name = "openai" }, + { name = "oss2" }, { name = "paddleocr" }, { name = "paddlepaddle" }, { name = "pdfplumber" }, @@ -724,6 +747,7 @@ requires-dist = [ { name = "mistletoe", specifier = ">=1.5.0" }, { name = "ollama", specifier = ">=0.6.0" }, { name = "openai", specifier = ">=2.7.1" }, + { name = "oss2", specifier = ">=2.18.0" }, { name = "paddleocr", specifier = ">=2.10.0,<3.0.0" }, { name = "paddlepaddle", specifier = ">=3.0.0,<4.0.0" }, { name = "pdfplumber", specifier = ">=0.11.7" }, @@ -1322,6 +1346,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/71/71408b02c6133153336d29fa3ba53000f1e1a3f78bb2fc2d1a1865d2e743/jiter-0.11.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18c77aaa9117510d5bdc6a946baf21b1f0cfa58ef04d31c8d016f206f2118960", size = 343697, upload-time = "2025-10-17T11:31:13.773Z" }, ] +[[package]] +name = "jmespath" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/56/3f325b1eef9791759784aa5046a8f6a1aff8f7c898a2e34506771d3b99d8/jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", size = 21607, upload-time = "2020-05-12T22:03:47.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f", size = 24489, upload-time = "2020-05-12T22:03:45.643Z" }, +] + [[package]] name = "joblib" version = "1.5.2" @@ -1984,21 +2017,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, { url = "https://files.pythonhosted.org/packages/db/db/81bf3d7cecfbfed9092b6b4052e857a769d62ed90561b410014e0aae18db/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:b28740f4ecef1738ea8f807461dd541b8287d5650b5be33bca7b474e3cbd1f36", size = 19153079, upload-time = "2025-10-27T23:05:57.686Z" }, { url = "https://files.pythonhosted.org/packages/2e/4d/a382452b17cf70a2313153c520ea4c96ab670c996cb3a95cc5d5ac7bfdac/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f7d1fe034090a1e371b7f3ca9d3ccae2fabae8c1d8844fb7371d1ea38e8e8d2", size = 15219883, upload-time = "2025-10-22T03:46:21.66Z" }, - { url = "https://files.pythonhosted.org/packages/fb/56/179bf90679984c85b417664c26aae4f427cba7514bd2d65c43b181b7b08b/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ca88747e708e5c67337b0f65eed4b7d0dd70d22ac332038c9fc4635760018f7", size = 17370357, upload-time = "2025-10-22T03:46:57.968Z" }, { url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113, upload-time = "2025-10-22T03:47:33.526Z" }, { url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857, upload-time = "2025-10-22T03:46:37.578Z" }, { url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095, upload-time = "2025-10-22T03:46:24.769Z" }, - { url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080, upload-time = "2025-10-22T03:47:00.265Z" }, { url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929, upload-time = "2025-10-22T03:47:36.24Z" }, { url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" }, { url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915, upload-time = "2025-10-22T03:46:27.773Z" }, - { url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649, upload-time = "2025-10-22T03:47:02.782Z" }, { url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337, upload-time = "2025-10-22T03:46:35.168Z" }, { url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" }, { url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898, upload-time = "2025-10-22T03:46:30.039Z" }, - { url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518, upload-time = "2025-10-22T03:47:05.407Z" }, { url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610, upload-time = "2025-10-22T03:46:32.239Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" }, ] [[package]] @@ -2099,6 +2127,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/19/404708a7e54ad2798907210462fd950c3442ea51acc8790f3da48d2bee8b/opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147", size = 65486, upload-time = "2020-07-19T22:40:30.301Z" }, ] +[[package]] +name = "oss2" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aliyun-python-sdk-core" }, + { name = "aliyun-python-sdk-kms" }, + { name = "crcmod" }, + { name = "pycryptodome" }, + { name = "requests" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/b5/f2cb1950dda46ac2284d6c950489fdacd0e743c2d79a347924d3cc44b86f/oss2-2.19.1.tar.gz", hash = "sha256:a8ab9ee7eb99e88a7e1382edc6ea641d219d585a7e074e3776e9dec9473e59c1", size = 298845, upload-time = "2024-10-25T11:37:46.638Z" } + [[package]] name = "packaging" version = "25.0" diff --git a/docs/api/agent.md b/docs/api/agent.md index 02640f58..95151eef 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -458,7 +458,7 @@ curl --location 'http://localhost:8080/api/v1/agents/placeholders' \ |------|------|--------|------| | `image_upload_enabled` | bool | false | 是否允许上传图片 | | `vlm_model_id` | string | - | 图片分析所用的 VLM 模型 ID | -| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`,为空使用全局默认 | +| `image_storage_provider` | string | - | 图片存储提供者:`local`/`minio`/`cos`/`tos`/`oss`,为空使用全局默认 | ### FAQ 策略设置 diff --git a/docs/api/system.md b/docs/api/system.md index eeab0cc0..dc1b1cdd 100644 --- a/docs/api/system.md +++ b/docs/api/system.md @@ -155,6 +155,11 @@ curl --location 'http://localhost:8080/api/v1/system/storage-engine-status' \ "name": "s3", "available": false, "description": "AWS S3 对象存储" + }, + { + "name": "oss", + "available": false, + "description": "阿里云 OSS 对象存储" } ], "minio_env_available": true diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d3fbc78c..ae6ca15c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -429,7 +429,7 @@ definitions: type: integer image_storage_provider: description: |- - Storage provider for image uploads: "local", "minio", "cos", "tos" + Storage provider for image uploads: "local", "minio", "cos", "tos", "s3", "oss" Empty means use the global/tenant default provider. type: string image_upload_enabled: @@ -2067,12 +2067,14 @@ definitions: cos: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.COSEngineConfig' default_provider: - description: '"local", "minio", "cos", "tos", "s3"' + description: '"local", "minio", "cos", "tos", "s3", "oss"' type: string local: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.LocalEngineConfig' minio: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig' + oss: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig' s3: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig' tos: @@ -2081,7 +2083,7 @@ definitions: github_com_Tencent_WeKnora_internal_types.StorageProviderConfig: properties: provider: - description: '"local", "minio", "cos", "tos"' + description: '"local", "minio", "cos", "tos", "s3", "oss"' type: string type: object github_com_Tencent_WeKnora_internal_types.SubmitJoinRequestRequest: @@ -2169,6 +2171,27 @@ definitions: secret_key: type: string type: object + github_com_Tencent_WeKnora_internal_types.OSSEngineConfig: + properties: + access_key: + type: string + bucket_name: + type: string + endpoint: + type: string + path_prefix: + type: string + region: + type: string + secret_key: + type: string + temp_bucket_name: + type: string + temp_region: + type: string + use_temp_bucket: + type: boolean + type: object github_com_Tencent_WeKnora_internal_types.Tenant: properties: agent_config: @@ -2671,8 +2694,10 @@ definitions: minio: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.MinIOEngineConfig' provider: - description: '"minio", "cos", "tos", or "s3"' + description: '"minio", "cos", "tos", "s3", or "oss"' type: string + oss: + $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.OSSEngineConfig' s3: $ref: '#/definitions/github_com_Tencent_WeKnora_internal_types.S3EngineConfig' tos: @@ -2696,7 +2721,7 @@ definitions: description: short description for UI type: string name: - description: '"local", "minio", "cos", "tos"' + description: '"local", "minio", "cos", "tos", "s3", "oss"' type: string type: object internal_handler.TextRelationExtractionRequest: diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts index 688e1323..35ad2855 100644 --- a/frontend/src/api/system/index.ts +++ b/frontend/src/api/system/index.ts @@ -183,7 +183,7 @@ export function reconnectDocReader(addr: string): Promise]*?)\ssrc=(["'])(local|minio|cos|tos|s3):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, + /]*?)\ssrc=(["'])(local|minio|cos|tos|s3|oss):(?:\/\/|//|//)([^"']+)\2([^>]*)>/gi, (_m, before, quote, provider, restPathRaw, after) => { const restPath = decodeProviderURL(restPathRaw); const protectedSrc = `${provider}://${restPath}`; @@ -173,7 +173,7 @@ export function isValidURL(url: string): boolean { } // 允许 provider:// 形式,由前端后续鉴权拉取并替换为 blob URL - if (/^(local|minio|cos|tos|s3):\/\/\S+$/i.test(trimmed)) { + if (/^(local|minio|cos|tos|s3|oss):\/\/\S+$/i.test(trimmed)) { return true; } @@ -299,7 +299,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } const images = root.querySelectorAll( - 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"]', + 'img[data-protected-src], img[src^="local://"], img[src^="minio://"], img[src^="cos://"], img[src^="tos://"], img[src^="s3://"], img[src^="oss://"]', ); if (!images.length) { return; @@ -319,7 +319,7 @@ export async function hydrateProtectedFileImages(root: ParentNode | null | undef } img.dataset.authHydrated = '1'; - const isProviderScheme = /^(local|minio|cos|tos|s3):\/\//.test(sourceURL); + const isProviderScheme = /^(local|minio|cos|tos|s3|oss):\/\//.test(sourceURL); const requestURL = isProviderScheme ? `/files?${new URLSearchParams({ file_path: sourceURL }).toString()}` : sourceURL; diff --git a/frontend/src/views/agent/AgentEditorModal.vue b/frontend/src/views/agent/AgentEditorModal.vue index b6846e4b..2e7aaea1 100644 --- a/frontend/src/views/agent/AgentEditorModal.vue +++ b/frontend/src/views/agent/AgentEditorModal.vue @@ -1266,6 +1266,7 @@ const imageStorageOptions = computed(() => { { value: 'cos', label: t('settings.storage.engineCos'), disabled: statusMap.cos === false }, { value: 'tos', label: t('settings.storage.engineTos'), disabled: statusMap.tos === false }, { value: 's3', label: 'Amazon S3', disabled: statusMap.s3 === false }, + { value: 'oss', label: t('settings.storage.engineOss'), disabled: statusMap.oss === false }, ]; }); diff --git a/frontend/src/views/knowledge/settings/KBStorageSettings.vue b/frontend/src/views/knowledge/settings/KBStorageSettings.vue index c3ec4c8b..035fb4ce 100644 --- a/frontend/src/views/knowledge/settings/KBStorageSettings.vue +++ b/frontend/src/views/knowledge/settings/KBStorageSettings.vue @@ -114,6 +114,13 @@ const engineOptions = computed(() => { available: statusMap.s3, disabled: statusMap.s3 === false, }, + { + value: 'oss', + label: t('kbSettings.storage.engineOss'), + desc: t('kbSettings.storage.engineOssDesc'), + available: statusMap.oss, + disabled: statusMap.oss === false, + }, ] }) diff --git a/frontend/src/views/settings/Settings.vue b/frontend/src/views/settings/Settings.vue index 86abcb0a..bdbfab36 100644 --- a/frontend/src/views/settings/Settings.vue +++ b/frontend/src/views/settings/Settings.vue @@ -201,6 +201,7 @@ const navItems = computed(() => [ { key: 'cos', label: t('settings.storage.cos') }, { key: 'tos', label: t('settings.storage.tos') }, { key: 's3', label: 'AWS S3' }, + { key: 'oss', label: t('settings.storage.oss') }, ] }, { key: 'mcp', icon: 'tools', label: t('settings.mcpService') }, diff --git a/frontend/src/views/settings/StorageEngineSettings.vue b/frontend/src/views/settings/StorageEngineSettings.vue index 405b74ce..694a5f45 100644 --- a/frontend/src/views/settings/StorageEngineSettings.vue +++ b/frontend/src/views/settings/StorageEngineSettings.vue @@ -34,6 +34,7 @@ + @@ -422,6 +423,80 @@ + +
+
+
+
+

{{ $t('settings.storage.ossTitle') }}

+ {{ $t('settings.storage.configurable') }} +
+

+ {{ $t('settings.storage.ossDesc') }} + {{ $t('settings.storage.console') }} ↗ + {{ $t('settings.storage.docs') }} ↗ +

+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ {{ $t('settings.storage.testConnection') }} + + {{ ossCheckResult.message }} + +
+
+
{{ $t('settings.storage.saveConfig') }} @@ -476,6 +551,17 @@ const defaultConfig = (): StorageEngineConfig => ({ bucket_name: '', path_prefix: '', }, + oss: { + endpoint: '', + region: '', + access_key: '', + secret_key: '', + bucket_name: '', + path_prefix: '', + use_temp_bucket: false, + temp_bucket_name: '', + temp_region: '', + }, }) const loading = ref(true) @@ -501,6 +587,8 @@ const checkingTos = ref(false) const tosCheckResult = ref<{ ok: boolean; message: string } | null>(null) const checkingS3 = ref(false) const s3CheckResult = ref<{ ok: boolean; message: string } | null>(null) +const checkingOss = ref(false) +const ossCheckResult = ref<{ ok: boolean; message: string } | null>(null) const minioAvailable = computed(() => { if (config.value.minio?.mode === 'remote') { @@ -558,6 +646,19 @@ async function loadConfig() { path_prefix: d.s3.path_prefix || '', } : defaultConfig().s3!, + oss: d.oss + ? { + endpoint: d.oss.endpoint || '', + region: d.oss.region || '', + access_key: d.oss.access_key || '', + secret_key: d.oss.secret_key || '', + bucket_name: d.oss.bucket_name || '', + path_prefix: d.oss.path_prefix || '', + use_temp_bucket: d.oss.use_temp_bucket ?? false, + temp_bucket_name: d.oss.temp_bucket_name || '', + temp_region: d.oss.temp_region || '', + } + : defaultConfig().oss!, } } } catch { @@ -649,6 +750,18 @@ function buildPayload(): StorageEngineConfig { bucket_name: (config.value.s3?.bucket_name || '').trim(), path_prefix: (config.value.s3?.path_prefix || '').trim(), }, + oss: { + endpoint: (config.value.oss?.endpoint || '').trim(), + region: (config.value.oss?.region || '').trim(), + access_key: (config.value.oss?.access_key || '').trim(), + secret_key: (config.value.oss?.secret_key || '').trim(), + bucket_name: (config.value.oss?.bucket_name || '').trim(), + path_prefix: (config.value.oss?.path_prefix || '').trim(), + // Temp bucket fields: not exposed in UI; server manages these independently + use_temp_bucket: config.value.oss?.use_temp_bucket ?? false, + temp_bucket_name: (config.value.oss?.temp_bucket_name || '').trim(), + temp_region: (config.value.oss?.temp_region || '').trim(), + }, } } @@ -728,6 +841,20 @@ async function onCheckS3() { } } +async function onCheckOss() { + checkingOss.value = true + ossCheckResult.value = null + try { + const payload = buildPayload() + const res = await checkStorageEngine({ provider: 'oss', oss: payload.oss }) + ossCheckResult.value = res?.data ?? { ok: false, message: t('settings.storage.unknownError') } + } catch (e: unknown) { + ossCheckResult.value = { ok: false, message: e instanceof Error ? e.message : t('settings.storage.requestFailed') } + } finally { + checkingOss.value = false + } +} + onMounted(loadAll) @@ -872,7 +999,7 @@ onMounted(loadAll) label { font-size: 13px; font-weight: 500; - color: var(--td-text-color-secondary)555; + color: var(--td-text-color-secondary); } &--inline { diff --git a/internal/application/service/file/factory.go b/internal/application/service/file/factory.go index 5cc8d008..405400bd 100644 --- a/internal/application/service/file/factory.go +++ b/internal/application/service/file/factory.go @@ -101,6 +101,30 @@ func NewFileServiceFromStorageConfig( svc, err := NewS3FileService(sec.S3.Endpoint, sec.S3.AccessKey, sec.S3.SecretKey, sec.S3.BucketName, sec.S3.Region, pathPrefix) return svc, p, err + case "oss": + if sec == nil || sec.OSS == nil || sec.OSS.Endpoint == "" || sec.OSS.Region == "" || sec.OSS.AccessKey == "" || sec.OSS.SecretKey == "" || sec.OSS.BucketName == "" { + return nil, p, fmt.Errorf("incomplete oss config") + } + pathPrefix := strings.TrimSpace(sec.OSS.PathPrefix) + if pathPrefix == "" { + pathPrefix = "weknora/" + } + var svc interfaces.FileService + var err error + if sec.OSS.UseTempBucket && sec.OSS.TempBucketName != "" { + svc, err = NewOssFileServiceWithTempBucket( + sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey, + sec.OSS.BucketName, pathPrefix, + sec.OSS.TempBucketName, sec.OSS.TempRegion, + ) + } else { + svc, err = NewOssFileService( + sec.OSS.Endpoint, sec.OSS.Region, sec.OSS.AccessKey, sec.OSS.SecretKey, + sec.OSS.BucketName, pathPrefix, + ) + } + return svc, p, err + default: return nil, p, fmt.Errorf("unsupported provider %q", p) } diff --git a/internal/application/service/file/oss.go b/internal/application/service/file/oss.go new file mode 100644 index 00000000..fdfb9017 --- /dev/null +++ b/internal/application/service/file/oss.go @@ -0,0 +1,331 @@ +package file + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "mime/multipart" + "path/filepath" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" +) + +// ossFileService implements the FileService interface for Aliyun OSS +// using the S3-compatible protocol with virtual-hosted style addressing. +type ossFileService struct { + client *s3.Client + tempClient *s3.Client + pathPrefix string + bucketName string + tempBucketName string +} + +const ossScheme = "oss://" + +// newOSSClient creates a bare s3.Client configured for OSS S3-compatible mode. +// OSS uses virtual-hosted style addressing and does not support aws-chunked encoding. +func newOSSClient(endpoint, region, accessKey, secretKey string) (*s3.Client, error) { + cfg, err := config.LoadDefaultConfig(context.Background(), + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")), + // Disable automatic aws-chunked encoding — OSS does not support it. + config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), + config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + ) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = false + }) + + return client, nil +} + +// ossBucketExists checks if the bucket exists using the provided client. +func ossBucketExists(ctx context.Context, client *s3.Client, bucketName string) (bool, error) { + _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }) + if err != nil { + var notFound *types.NotFound + if errors.As(err, ¬Found) { + return false, nil + } + return false, err + } + return true, nil +} + +// ossCreateBucket creates a new bucket using the provided client. +func ossCreateBucket(ctx context.Context, client *s3.Client, bucketName string) error { + _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + return err +} + +// ossEnsureBucket checks if the bucket exists and creates it if missing. +func ossEnsureBucket(ctx context.Context, client *s3.Client, bucketName string) error { + exists, err := ossBucketExists(ctx, client, bucketName) + if err != nil { + return fmt.Errorf("failed to check bucket: %w", err) + } + if !exists { + if err := ossCreateBucket(ctx, client, bucketName); err != nil { + return fmt.Errorf("failed to create bucket: %w", err) + } + } + return nil +} + +// NewOssFileService creates an Aliyun OSS file service. +// It verifies that the bucket exists and creates it if missing. +func NewOssFileService(endpoint, region, accessKey, secretKey, bucketName, pathPrefix string) (interfaces.FileService, error) { + return NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, "", "") +} + +// NewOssFileServiceWithTempBucket creates an Aliyun OSS file service with optional temp bucket. +func NewOssFileServiceWithTempBucket(endpoint, region, accessKey, secretKey, bucketName, pathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) { + client, err := newOSSClient(endpoint, region, accessKey, secretKey) + if err != nil { + return nil, err + } + + if err := ossEnsureBucket(context.Background(), client, bucketName); err != nil { + return nil, err + } + + var tempClient *s3.Client + if tempBucketName != "" { + if tempRegion == "" { + tempRegion = region + } + tempClient, err = newOSSClient(endpoint, tempRegion, accessKey, secretKey) + if err != nil { + return nil, fmt.Errorf("failed to initialize OSS temp client: %w", err) + } + if err := ossEnsureBucket(context.Background(), tempClient, tempBucketName); err != nil { + return nil, err + } + } + + // Normalize pathPrefix: ensure it ends with '/' if not empty + if pathPrefix != "" && !strings.HasSuffix(pathPrefix, "/") { + pathPrefix += "/" + } + + return &ossFileService{ + client: client, + tempClient: tempClient, + pathPrefix: pathPrefix, + bucketName: bucketName, + tempBucketName: tempBucketName, + }, nil +} + +// CheckOssConnectivity tests OSS connectivity using the provided credentials. +func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error { + client, err := newOSSClient(endpoint, region, accessKey, secretKey) + if err != nil { + return err + } + + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + exists, err := ossBucketExists(checkCtx, client, bucketName) + if err != nil { + return fmt.Errorf("failed to check bucket: %w", err) + } + if !exists { + return fmt.Errorf("bucket %q does not exist", bucketName) + } + return nil +} + +// parseOssFilePath extracts bucket and object key from: oss://{bucket}/{objectKey} +func parseOssFilePath(filePath string) (bucketName string, objectKey string, err error) { + if !strings.HasPrefix(filePath, ossScheme) { + return "", "", fmt.Errorf("invalid OSS file path: %s", filePath) + } + + rest := strings.TrimPrefix(filePath, ossScheme) + parts := strings.SplitN(rest, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid OSS file path: %s", filePath) + } + return parts[0], parts[1], nil +} + +// CheckConnectivity verifies OSS is reachable and the main bucket exists. +func (s *ossFileService) CheckConnectivity(ctx context.Context) error { + checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + exists, err := ossBucketExists(checkCtx, s.client, s.bucketName) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("bucket %q does not exist", s.bucketName) + } + return nil +} + +// SaveFile saves a file to OSS. +func (s *ossFileService) SaveFile(ctx context.Context, + file *multipart.FileHeader, tenantID uint64, knowledgeID string, +) (string, error) { + ext := filepath.Ext(file.Filename) + objectName := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext) + + src, err := file.Open() + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + + contentType := file.Header.Get("Content-Type") + if contentType == "" { + contentType = getContentTypeByExt(ext) + } + + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(objectName), + Body: src, + ContentType: aws.String(contentType), + }) + if err != nil { + return "", fmt.Errorf("failed to upload file to OSS: %w", err) + } + + return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil +} + +// SaveBytes saves bytes data to OSS. +// If temp is true and temp bucket is configured, saves to temp bucket. +// Otherwise saves to main bucket. +func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + safeName, err := utils.SafeFileName(fileName) + if err != nil { + return "", fmt.Errorf("invalid file name: %w", err) + } + ext := filepath.Ext(safeName) + + // If requesting temp bucket and it is configured, use it + if temp && s.tempClient != nil { + objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext) + _, err := s.tempClient.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.tempBucketName), + Key: aws.String(objectName), + Body: bytes.NewReader(data), + ContentType: aws.String("text/csv; charset=utf-8"), + }) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to OSS temp bucket: %w", err) + } + return fmt.Sprintf("oss://%s/%s", s.tempBucketName, objectName), nil + } + + // Save to main bucket + objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext) + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucketName), + Key: aws.String(objectName), + Body: bytes.NewReader(data), + ContentType: aws.String("text/csv; charset=utf-8"), + }) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to OSS: %w", err) + } + + return fmt.Sprintf("oss://%s/%s", s.bucketName, objectName), nil +} + +// GetFile retrieves a file from OSS by its path. +func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + bucketName, objectName, err := parseOssFilePath(filePath) + if err != nil { + return nil, err + } + if err := utils.SafeObjectKey(objectName); err != nil { + return nil, fmt.Errorf("invalid file path: %w", err) + } + + resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectName), + }) + if err != nil { + return nil, fmt.Errorf("failed to get file from OSS: %w", err) + } + + return resp.Body, nil +} + +// DeleteFile removes a file from OSS. +func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error { + bucketName, objectName, err := parseOssFilePath(filePath) + if err != nil { + return err + } + if err := utils.SafeObjectKey(objectName); err != nil { + return fmt.Errorf("invalid file path: %w", err) + } + + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectName), + }) + if err != nil { + return fmt.Errorf("failed to delete file from OSS: %w", err) + } + + return nil +} + +// GetFileURL returns a presigned download URL for the file. +func (s *ossFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + bucketName, objectName, err := parseOssFilePath(filePath) + if err != nil { + return "", err + } + if err := utils.SafeObjectKey(objectName); err != nil { + return "", fmt.Errorf("invalid file path: %w", err) + } + + // Determine which client to use + var client *s3.Client + if bucketName == s.tempBucketName && s.tempClient != nil { + client = s.tempClient + } else { + client = s.client + } + + presignClient := s3.NewPresignClient(client) + + presignedReq, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectName), + }, s3.WithPresignExpires(24*time.Hour)) + if err != nil { + return "", fmt.Errorf("failed to generate OSS presigned URL: %w", err) + } + + return presignedReq.URL, nil +} diff --git a/internal/application/service/file/oss_test.go b/internal/application/service/file/oss_test.go new file mode 100644 index 00000000..3a1b06b6 --- /dev/null +++ b/internal/application/service/file/oss_test.go @@ -0,0 +1,208 @@ +package file + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestParseOssFilePath(t *testing.T) { + tests := []struct { + name string + input string + wantBucket string + wantKey string + wantErr bool + errContains string + }{ + { + name: "valid path with nested key", + input: "oss://my-bucket/123/exports/abc123.csv", + wantBucket: "my-bucket", + wantKey: "123/exports/abc123.csv", + }, + { + name: "valid path with simple key", + input: "oss://test-bucket/key", + wantBucket: "test-bucket", + wantKey: "key", + }, + { + name: "valid path with deep nesting", + input: "oss://bucket/prefix/tenant/exports/uuid.png", + wantBucket: "bucket", + wantKey: "prefix/tenant/exports/uuid.png", + }, + { + name: "invalid scheme", + input: "s3://bucket/key", + wantErr: true, + errContains: "invalid OSS file path", + }, + { + name: "empty path", + input: "", + wantErr: true, + errContains: "invalid OSS file path", + }, + { + name: "bucket only no key", + input: "oss://bucket/", + wantErr: true, + errContains: "invalid OSS file path", + }, + { + name: "scheme only", + input: "oss://", + wantErr: true, + errContains: "invalid OSS file path", + }, + { + name: "no slash after bucket", + input: "oss://bucket", + wantErr: true, + errContains: "invalid OSS file path", + }, + { + name: "empty bucket name", + input: "oss:///some-key", + wantErr: true, + errContains: "invalid OSS file path", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bucket, key, err := parseOssFilePath(tt.input) + if tt.wantErr { + if err == nil { + t.Errorf("parseOssFilePath(%q) expected error, got bucket=%q key=%q", tt.input, bucket, key) + } + if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) { + t.Errorf("parseOssFilePath(%q) error = %v, want containing %q", tt.input, err, tt.errContains) + } + return + } + if err != nil { + t.Errorf("parseOssFilePath(%q) unexpected error: %v", tt.input, err) + return + } + if bucket != tt.wantBucket { + t.Errorf("parseOssFilePath(%q) bucket = %q, want %q", tt.input, bucket, tt.wantBucket) + } + if key != tt.wantKey { + t.Errorf("parseOssFilePath(%q) key = %q, want %q", tt.input, key, tt.wantKey) + } + }) + } +} + +func TestNewOSSClient(t *testing.T) { + tests := []struct { + name string + endpoint string + region string + accessKey string + secretKey string + wantErr bool + }{ + { + name: "valid parameters create client", + endpoint: "https://oss-cn-hangzhou.aliyuncs.com", + region: "cn-hangzhou", + accessKey: "test-access-key", + secretKey: "test-secret-key", + wantErr: false, + }, + { + name: "custom endpoint", + endpoint: "https://custom-oss-endpoint.com", + region: "cn-shanghai", + accessKey: "ak", + secretKey: "sk", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := newOSSClient(tt.endpoint, tt.region, tt.accessKey, tt.secretKey) + if tt.wantErr { + if err == nil { + t.Error("expected error but got nil") + } + return + } + if err != nil { + t.Errorf("newOSSClient() unexpected error: %v", err) + return + } + if client == nil { + t.Error("expected non-nil client") + } + }) + } +} + +func TestCheckOssConnectivity_InvalidEndpoint(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Should fail with an invalid/unreachable endpoint + err := CheckOssConnectivity(ctx, + "https://invalid-oss-endpoint-that-does-not-exist.local", + "cn-hangzhou", + "invalid-access-key", + "invalid-secret-key", + "nonexistent-bucket", + ) + + if err == nil { + t.Error("CheckOssConnectivity with invalid endpoint should return an error") + } +} + +func TestOssBucketExists_NonExistent(t *testing.T) { + client, err := newOSSClient( + "https://oss-cn-hangzhou.aliyuncs.com", + "cn-hangzhou", + "test-invalid-key", + "test-invalid-secret", + ) + if err != nil { + t.Fatalf("newOSSClient() error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Bucket that definitely doesn't exist - should return false, not error + // (auth may fail first, but we verify the function doesn't crash) + exists, err := ossBucketExists(ctx, client, "this-bucket-definitely-does-not-exist-12345") + // We don't assert on the result because it depends on network/auth, + // but the function should not panic or hang. + _ = exists + _ = err +} + +func TestOssCreateBucket(t *testing.T) { + client, err := newOSSClient( + "https://oss-cn-hangzhou.aliyuncs.com", + "cn-hangzhou", + "test-invalid-key", + "test-invalid-secret", + ) + if err != nil { + t.Fatalf("newOSSClient() error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Should fail with invalid credentials + err = ossCreateBucket(ctx, client, "test-bucket") + if err == nil { + t.Error("ossCreateBucket with invalid credentials should return an error") + } +} diff --git a/internal/container/container.go b/internal/container/container.go index bc56bcce..a6729a2a 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -647,6 +647,28 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) { os.Getenv("S3_REGION"), pathPrefix, ) + case "oss": + if os.Getenv("OSS_ENDPOINT") == "" || + os.Getenv("OSS_REGION") == "" || + os.Getenv("OSS_ACCESS_KEY") == "" || + os.Getenv("OSS_SECRET_KEY") == "" || + os.Getenv("OSS_BUCKET_NAME") == "" { + return nil, fmt.Errorf("missing OSS configuration") + } + pathPrefix := os.Getenv("OSS_PATH_PREFIX") + if pathPrefix == "" { + pathPrefix = "weknora/" + } + return file.NewOssFileServiceWithTempBucket( + os.Getenv("OSS_ENDPOINT"), + os.Getenv("OSS_REGION"), + os.Getenv("OSS_ACCESS_KEY"), + os.Getenv("OSS_SECRET_KEY"), + os.Getenv("OSS_BUCKET_NAME"), + pathPrefix, + os.Getenv("OSS_TEMP_BUCKET_NAME"), + os.Getenv("OSS_TEMP_REGION"), + ) case "local": baseDir := os.Getenv("LOCAL_STORAGE_BASE_DIR") if baseDir == "" { diff --git a/internal/handler/system.go b/internal/handler/system.go index e1dbd323..7cdeaa5a 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -396,6 +396,17 @@ func (h *SystemHandler) isTOSConfigured(c *gin.Context) bool { return h.isTOSEnvAvailable() } +// isOSSConfigured checks whether OSS connection info is available from tenant config. +func (h *SystemHandler) isOSSConfigured(c *gin.Context) bool { + if v, exists := c.Get(types.TenantInfoContextKey.String()); exists { + if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.OSS != nil { + ossConf := tenant.StorageEngineConfig.OSS + return ossConf.Endpoint != "" && ossConf.Region != "" && ossConf.AccessKey != "" && ossConf.SecretKey != "" && ossConf.BucketName != "" + } + } + return false +} + // isTOSEnvAvailable checks whether TOS env vars are set. func (h *SystemHandler) isTOSEnvAvailable() bool { return os.Getenv("TOS_ENDPOINT") != "" && @@ -442,11 +453,13 @@ func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) { minioEnvAvailable := h.isMinioEnvAvailable() cosConfigured := h.isCOSConfigured(c) tosConfigured := h.isTOSConfigured(c) + ossConfigured := h.isOSSConfigured(c) engines := []StorageEngineStatusItem{ {Name: "local", Available: true, Description: "本地文件系统存储,仅适合单机部署"}, {Name: "minio", Available: minioConfigured || minioEnvAvailable, Description: "S3 兼容的自托管对象存储,适合内网和私有云部署"}, {Name: "cos", Available: cosConfigured, Description: "腾讯云对象存储服务,适合公有云部署,支持 CDN 加速"}, {Name: "tos", Available: tosConfigured, Description: "火山引擎对象存储服务,适合公有云部署"}, + {Name: "oss", Available: ossConfigured, Description: "阿里云对象存储服务,适合公有云部署,支持 S3 兼容协议"}, } c.JSON(200, gin.H{ "code": 0, @@ -640,6 +653,9 @@ func hasGetObjectAction(action interface{}) bool { // cosFieldPattern validates COS region and bucket name format to prevent URL injection. var cosFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) +// ossFieldPattern validates OSS region and bucket name format to prevent URL injection. +var ossFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) + // sanitizeStorageCheckError converts a raw storage connectivity error into a safe // user-facing message that does not leak internal network details (hostnames, IPs, ports). func sanitizeStorageCheckError(err error) string { @@ -727,11 +743,12 @@ func isBlockedStorageEndpoint(endpoint string) (bool, string) { // StorageCheckRequest is the body for POST /system/storage-engine-check. type StorageCheckRequest struct { - Provider string `json:"provider"` // "minio", "cos", "tos", or "s3" + Provider string `json:"provider"` // "minio", "cos", "tos", "s3", "oss" MinIO *types.MinIOEngineConfig `json:"minio,omitempty"` COS *types.COSEngineConfig `json:"cos,omitempty"` TOS *types.TOSEngineConfig `json:"tos,omitempty"` S3 *types.S3EngineConfig `json:"s3,omitempty"` + OSS *types.OSSEngineConfig `json:"oss,omitempty"` } // StorageCheckResponse is the response for a single-engine connectivity check. @@ -768,6 +785,8 @@ func (h *SystemHandler) CheckStorageEngine(c *gin.Context) { h.checkTOS(c, ctx, req.TOS) case "s3": h.checkS3(c, ctx, req.S3) + case "oss": + h.checkOSS(c, ctx, req.OSS) default: c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}}) } @@ -960,6 +979,55 @@ func (h *SystemHandler) checkS3(c *gin.Context, ctx context.Context, cfg *types. c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) } +func (h *SystemHandler) checkOSS(c *gin.Context, ctx context.Context, cfg *types.OSSEngineConfig) { + if cfg == nil { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 OSS 配置"}}) + return + } + + endpoint, accessKey, secretKey := cfg.Endpoint, cfg.AccessKey, cfg.SecretKey + if endpoint == "" || accessKey == "" || secretKey == "" || cfg.BucketName == "" { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Access Key、Secret Key、Bucket Name 不能为空"}}) + return + } + + // Strip URL scheme before SSRF check — OSS endpoint may include http:// or https:// + ssrfEndpoint := strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://") + if blocked, reason := isBlockedStorageEndpoint(ssrfEndpoint); blocked { + logger.Warnf(ctx, "Storage check: OSS endpoint blocked by SSRF protection", "endpoint", endpoint) + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}}) + return + } + if !ossFieldPattern.MatchString(cfg.Region) { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Region 格式不正确,仅允许字母、数字、点、连字符"}}) + return + } + if !ossFieldPattern.MatchString(cfg.BucketName) { + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Bucket 名称格式不正确,仅允许字母、数字、点、连字符"}}) + return + } + + err := file.CheckOssConnectivity(ctx, endpoint, cfg.Region, accessKey, secretKey, cfg.BucketName) + if err != nil { + errMsg := err.Error() + if strings.Contains(errMsg, "403") || strings.Contains(errMsg, "AccessDenied") { + logger.Errorf(ctx, "Storage check: OSS auth failed", "endpoint", endpoint, "bucket", cfg.BucketName) + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Access Key / Secret Key 是否正确"}}) + return + } + if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") { + logger.Errorf(ctx, "Storage check: OSS bucket not found", "bucket", cfg.BucketName) + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在", cfg.BucketName)}}) + return + } + logger.Errorf(ctx, "Storage check: OSS connectivity failed", "endpoint", endpoint, "bucket", cfg.BucketName, "error", err) + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("OSS 连通性检测失败: %s", sanitizeStorageCheckError(err))}}) + return + } + + c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) +} + func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader { if addr == "" { return h.documentReader diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index e280f7c8..462e5829 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -152,7 +152,7 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string { // StorageProviderConfig stores the KB-level storage provider selection. // Credentials are managed at the tenant level (StorageEngineConfig). type StorageProviderConfig struct { - Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos" + Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos", "s3", "oss" } func (c StorageProviderConfig) Value() (driver.Value, error) { @@ -266,7 +266,7 @@ func InferStorageFromFilePath(filePath string) string { // e.g. "minio://bucket/key" → "minio", "local://tenant/file.pdf" → "local" // Returns "" if the path does not use a known provider scheme. func ParseProviderScheme(filePath string) string { - for _, provider := range []string{"local", "minio", "cos", "tos", "s3"} { + for _, provider := range []string{"local", "minio", "cos", "tos", "s3", "oss"} { if strings.HasPrefix(filePath, provider+"://") { return provider } diff --git a/internal/types/tenant.go b/internal/types/tenant.go index b6081842..53f6240c 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -328,15 +328,16 @@ func (c *ParserEngineConfig) Scan(value interface{}) error { return json.Unmarshal(b, c) } -// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, and S3. +// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, COS, TOS, S3, and OSS. // Knowledge bases select which provider to use; parameters are read from here. type StorageEngineConfig struct { - DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3" + DefaultProvider string `json:"default_provider"` // "local", "minio", "cos", "tos", "s3", "oss" Local *LocalEngineConfig `json:"local,omitempty"` MinIO *MinIOEngineConfig `json:"minio,omitempty"` COS *COSEngineConfig `json:"cos,omitempty"` TOS *TOSEngineConfig `json:"tos,omitempty"` S3 *S3EngineConfig `json:"s3,omitempty"` + OSS *OSSEngineConfig `json:"oss,omitempty"` } // LocalEngineConfig is for local file system storage (single-machine deployment only). @@ -386,6 +387,19 @@ type S3EngineConfig struct { PathPrefix string `json:"path_prefix"` } +// OSSEngineConfig is for Alibaba Cloud OSS (对象存储服务). +type OSSEngineConfig struct { + Endpoint string `json:"endpoint"` + Region string `json:"region"` + AccessKey string `json:"access_key"` + SecretKey string `json:"secret_key"` + BucketName string `json:"bucket_name"` + PathPrefix string `json:"path_prefix"` + UseTempBucket bool `json:"use_temp_bucket"` + TempBucketName string `json:"temp_bucket_name"` + TempRegion string `json:"temp_region"` +} + // Value implements the driver.Valuer interface for StorageEngineConfig func (c *StorageEngineConfig) Value() (driver.Value, error) { if c == nil {