feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
+149
-205
@@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
|
||||
|
||||
from docreader import config
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser import Parser
|
||||
from docreader.proto import docreader_pb2_grpc
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.proto.docreader_pb2 import (
|
||||
Chunk,
|
||||
Image,
|
||||
ReadConfig,
|
||||
ReadFromFileRequest,
|
||||
ReadFromURLRequest,
|
||||
ReadRequest,
|
||||
ReadResponse,
|
||||
StorageProvider,
|
||||
ImageRef,
|
||||
ListEnginesResponse,
|
||||
ParserEngineInfo,
|
||||
)
|
||||
from docreader.utils.request import init_logging_request_id, request_id_context
|
||||
|
||||
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
|
||||
# cannot be encoded to UTF-8
|
||||
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
|
||||
|
||||
|
||||
def to_valid_utf8_text(s: Optional[str]) -> str:
|
||||
"""Return a UTF-8 safe string for protobuf.
|
||||
|
||||
- Replace any surrogate code points with U+FFFD
|
||||
- Re-encode with errors='replace' to ensure valid UTF-8
|
||||
"""
|
||||
if not s:
|
||||
return ""
|
||||
s = _SURROGATE_RE.sub("\ufffd", s)
|
||||
return s.encode("utf-8", errors="replace").decode("utf-8")
|
||||
|
||||
|
||||
# Ensure no existing handlers
|
||||
for handler in logging.root.handlers[:]:
|
||||
logging.root.removeHandler(handler)
|
||||
|
||||
# Configure logging - use stdout
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
logging.root.addHandler(handler)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Initializing server logging")
|
||||
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
|
||||
_level = getattr(logging, _level_name, logging.INFO)
|
||||
logging.root.setLevel(_level)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info("Initializing server logging, level=%s", _level_name)
|
||||
|
||||
# Initialize request ID logging
|
||||
init_logging_request_id()
|
||||
|
||||
|
||||
parser = Parser()
|
||||
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
|
||||
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
|
||||
|
||||
``images`` is a dict of {relative_path: raw_data} where raw_data is
|
||||
base64-encoded string or raw bytes.
|
||||
|
||||
def create_chunking_config(read_config: ReadConfig):
|
||||
"""Create ChunkingConfig from ReadConfig request.
|
||||
``storage_map`` is a dict from the request's config.image_storage map.
|
||||
When it contains a valid "provider" key, images are uploaded to shared object
|
||||
storage and storage_key is set (no inline bytes, saving gRPC message size).
|
||||
|
||||
Args:
|
||||
read_config: The read_config from the gRPC request
|
||||
|
||||
Returns:
|
||||
ChunkingConfig: Configured chunking configuration object
|
||||
Returns (image_dir_path, list[ImageRef]).
|
||||
"""
|
||||
# Extract chunking parameters
|
||||
chunk_size = read_config.chunk_size or 512
|
||||
chunk_overlap = read_config.chunk_overlap or 50
|
||||
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
|
||||
separators = (
|
||||
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
|
||||
)
|
||||
enable_multimodal = read_config.enable_multimodal or False
|
||||
import base64
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
|
||||
)
|
||||
if not images:
|
||||
return "", []
|
||||
|
||||
# Extract storage config
|
||||
sc = read_config.storage_config
|
||||
storage_config = {
|
||||
"provider": StorageProvider.Name(sc.provider),
|
||||
"region": sc.region,
|
||||
"bucket_name": sc.bucket_name,
|
||||
"access_key_id": sc.access_key_id,
|
||||
"secret_access_key": sc.secret_access_key,
|
||||
"app_id": sc.app_id,
|
||||
"path_prefix": sc.path_prefix,
|
||||
}
|
||||
logger.info(
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, "
|
||||
f"bucket={storage_config['bucket_name']}"
|
||||
)
|
||||
|
||||
# Extract VLM config
|
||||
vlm_config = {
|
||||
"model_name": read_config.vlm_config.model_name,
|
||||
"base_url": read_config.vlm_config.base_url,
|
||||
"api_key": read_config.vlm_config.api_key,
|
||||
"interface_type": read_config.vlm_config.interface_type or "openai",
|
||||
mime_map = {
|
||||
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"Using VLM config: model={vlm_config['model_name']}, "
|
||||
f"base_url={vlm_config['base_url']}, "
|
||||
f"interface_type={vlm_config['interface_type']}"
|
||||
)
|
||||
storage_client = None
|
||||
use_storage = False
|
||||
provider = (storage_map or {}).get("provider", "")
|
||||
if provider:
|
||||
try:
|
||||
from docreader.parser.storage import create_storage
|
||||
storage_client = create_storage(storage_map)
|
||||
use_storage = True
|
||||
logger.info("Using shared storage (%s) for image upload", provider)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
|
||||
|
||||
# Create and return ChunkingConfig
|
||||
return ChunkingConfig(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=separators,
|
||||
enable_multimodal=enable_multimodal,
|
||||
storage_config=storage_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
base_dir = CONFIG.image_output_dir
|
||||
output_dir = os.path.join(base_dir, request_id, "images")
|
||||
wrote_to_dir = False
|
||||
if not use_storage:
|
||||
try:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
wrote_to_dir = True
|
||||
except OSError:
|
||||
logger.warning("Cannot write to image output dir %s", output_dir)
|
||||
|
||||
refs = []
|
||||
for ref_path, b64data in images.items():
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64data)
|
||||
except Exception:
|
||||
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
|
||||
|
||||
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
mime = mime_map.get(ext, "application/octet-stream")
|
||||
|
||||
if use_storage and storage_client:
|
||||
storage_key = storage_client.upload_bytes(img_bytes, ext)
|
||||
if storage_key:
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
storage_key=storage_key,
|
||||
))
|
||||
continue
|
||||
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
|
||||
|
||||
if wrote_to_dir:
|
||||
dest = os.path.join(output_dir, fname)
|
||||
try:
|
||||
with open(dest, "wb") as f:
|
||||
f.write(img_bytes)
|
||||
except OSError as e:
|
||||
logger.warning("Failed to write image %s: %s", dest, e)
|
||||
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
image_data=img_bytes,
|
||||
))
|
||||
|
||||
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
|
||||
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
|
||||
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
|
||||
return image_dir, refs
|
||||
|
||||
|
||||
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
@@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
super().__init__()
|
||||
self.parser = Parser()
|
||||
|
||||
def ReadFromFile(self, request: ReadFromFileRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
def Read(self, request: ReadRequest, context):
|
||||
"""Unified read: file mode (file_content set) or URL mode (url set)."""
|
||||
request_id = request.request_id or str(uuid.uuid4())
|
||||
is_url = bool(request.url)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
# Get file type
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
|
||||
)
|
||||
logger.info(f"File content size: {len(request.file_content)} bytes")
|
||||
cfg = request.config
|
||||
parser_engine = cfg.parser_engine if cfg else ""
|
||||
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
|
||||
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
if is_url:
|
||||
logger.info("Read(URL): url=%s", request.url)
|
||||
result = self.parser.parse_url(
|
||||
request.url,
|
||||
request.title,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.url
|
||||
else:
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
"Read(File): file=%s, type=%s, size=%d bytes",
|
||||
request.file_name, file_type, len(request.file_content),
|
||||
)
|
||||
result = self.parser.parse_file(
|
||||
request.file_name,
|
||||
file_type,
|
||||
request.file_content,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.file_name
|
||||
|
||||
# Parse file
|
||||
logger.info("Starting file parsing process")
|
||||
result = self.parser.parse_file(
|
||||
request.file_name, file_type, request.file_content, chunking_config
|
||||
)
|
||||
|
||||
if not result:
|
||||
error_msg = "Failed to parse file"
|
||||
if not result or not result.content:
|
||||
error_msg = f"Failed to parse: {source_desc}"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse()
|
||||
|
||||
# Convert to protobuf message
|
||||
logger.info(
|
||||
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
# Build response, including image info
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading file: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def ReadFromURL(self, request: ReadFromURLRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
logger.info(f"Received ReadFromURL request for URL: {request.url}")
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
|
||||
# Parse URL
|
||||
logger.info("Starting URL parsing process")
|
||||
result = self.parser.parse_url(
|
||||
request.url, request.title, chunking_config
|
||||
)
|
||||
if not result:
|
||||
error_msg = "Failed to parse URL"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse(error=error_msg)
|
||||
|
||||
# Convert to protobuf message, including image info
|
||||
logger.info(
|
||||
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
_c = to_valid_utf8_text
|
||||
image_dir, image_refs = _resolve_images(
|
||||
result.images, request_id, storage_map=storage_map
|
||||
)
|
||||
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
markdown_content=_c(result.content),
|
||||
image_refs=image_refs,
|
||||
image_dir_path=image_dir,
|
||||
)
|
||||
logger.info(
|
||||
"Read response: content_len=%d, images=%d",
|
||||
len(result.content), len(image_refs),
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading URL: {str(e)}"
|
||||
error_msg = f"Error reading document: {e}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
logger.info("Traceback: %s", traceback.format_exc())
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def _convert_chunk_to_proto(self, chunk):
|
||||
"""Convert internal Chunk object to protobuf Chunk message
|
||||
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
|
||||
"""
|
||||
# Clean helper for strings
|
||||
_c = to_valid_utf8_text
|
||||
|
||||
proto_chunk = Chunk(
|
||||
content=_c(getattr(chunk, "content", None)),
|
||||
seq=getattr(chunk, "seq", 0),
|
||||
start=getattr(chunk, "start", 0),
|
||||
end=getattr(chunk, "end", 0),
|
||||
)
|
||||
|
||||
# If chunk has images attribute and is not empty, add image info
|
||||
if hasattr(chunk, "images") and chunk.images:
|
||||
logger.info(
|
||||
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
|
||||
def ListEngines(self, request, context):
|
||||
overrides = dict(getattr(request, "config_overrides", None) or {})
|
||||
engines_data = registry.list_engines(overrides=overrides or None)
|
||||
engines = [
|
||||
ParserEngineInfo(
|
||||
name=e["name"],
|
||||
description=e["description"],
|
||||
file_types=e["file_types"],
|
||||
available=e.get("available", True),
|
||||
unavailable_reason=e.get("unavailable_reason", ""),
|
||||
)
|
||||
for img_info in chunk.images:
|
||||
# img_info expected as dict
|
||||
proto_image = Image(
|
||||
url=_c(img_info.get("cos_url", "")),
|
||||
caption=_c(img_info.get("caption", "")),
|
||||
ocr_text=_c(img_info.get("ocr_text", "")),
|
||||
original_url=_c(img_info.get("original_url", "")),
|
||||
start=int(img_info.get("start", 0) or 0),
|
||||
end=int(img_info.get("end", 0) or 0),
|
||||
)
|
||||
proto_chunk.images.append(proto_image)
|
||||
|
||||
return proto_chunk
|
||||
for e in engines_data
|
||||
]
|
||||
return ListEnginesResponse(engines=engines)
|
||||
|
||||
|
||||
def main():
|
||||
# Print effective env/config at startup
|
||||
config.print_config()
|
||||
|
||||
# Create server
|
||||
server = grpc.server(
|
||||
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
|
||||
options=[
|
||||
@@ -286,24 +236,18 @@ def main():
|
||||
],
|
||||
)
|
||||
|
||||
# Register services
|
||||
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
|
||||
|
||||
# Register health check service
|
||||
health_servicer = HealthServicer()
|
||||
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
|
||||
|
||||
# Set listen address
|
||||
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
|
||||
|
||||
# Start service
|
||||
server.start()
|
||||
|
||||
logger.info(f"Server started on port {CONFIG.grpc_port}")
|
||||
logger.info("Server started on port %d", CONFIG.grpc_port)
|
||||
logger.info("Server is ready to accept connections")
|
||||
|
||||
try:
|
||||
# Wait for service termination
|
||||
server.wait_for_termination()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received termination signal, shutting down server")
|
||||
|
||||
Reference in New Issue
Block a user