feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+149 -205
View File
@@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
from docreader import config
from docreader.config import CONFIG
from docreader.models.read_config import ChunkingConfig
from docreader.parser import Parser
from docreader.proto import docreader_pb2_grpc
from docreader.parser.registry import registry
from docreader.proto.docreader_pb2 import (
Chunk,
Image,
ReadConfig,
ReadFromFileRequest,
ReadFromURLRequest,
ReadRequest,
ReadResponse,
StorageProvider,
ImageRef,
ListEnginesResponse,
ParserEngineInfo,
)
from docreader.utils.request import init_logging_request_id, request_id_context
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
# cannot be encoded to UTF-8
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
def to_valid_utf8_text(s: Optional[str]) -> str:
"""Return a UTF-8 safe string for protobuf.
- Replace any surrogate code points with U+FFFD
- Re-encode with errors='replace' to ensure valid UTF-8
"""
if not s:
return ""
s = _SURROGATE_RE.sub("\ufffd", s)
return s.encode("utf-8", errors="replace").decode("utf-8")
# Ensure no existing handlers
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
# Configure logging - use stdout
handler = logging.StreamHandler(sys.stdout)
logging.root.addHandler(handler)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.info("Initializing server logging")
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
_level = getattr(logging, _level_name, logging.INFO)
logging.root.setLevel(_level)
logger = logging.getLogger(__name__)
logger.info("Initializing server logging, level=%s", _level_name)
# Initialize request ID logging
init_logging_request_id()
parser = Parser()
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
``images`` is a dict of {relative_path: raw_data} where raw_data is
base64-encoded string or raw bytes.
def create_chunking_config(read_config: ReadConfig):
"""Create ChunkingConfig from ReadConfig request.
``storage_map`` is a dict from the request's config.image_storage map.
When it contains a valid "provider" key, images are uploaded to shared object
storage and storage_key is set (no inline bytes, saving gRPC message size).
Args:
read_config: The read_config from the gRPC request
Returns:
ChunkingConfig: Configured chunking configuration object
Returns (image_dir_path, list[ImageRef]).
"""
# Extract chunking parameters
chunk_size = read_config.chunk_size or 512
chunk_overlap = read_config.chunk_overlap or 50
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
separators = (
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
)
enable_multimodal = read_config.enable_multimodal or False
import base64
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
if not images:
return "", []
# Extract storage config
sc = read_config.storage_config
storage_config = {
"provider": StorageProvider.Name(sc.provider),
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
# Extract VLM config
vlm_config = {
"model_name": read_config.vlm_config.model_name,
"base_url": read_config.vlm_config.base_url,
"api_key": read_config.vlm_config.api_key,
"interface_type": read_config.vlm_config.interface_type or "openai",
mime_map = {
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
storage_client = None
use_storage = False
provider = (storage_map or {}).get("provider", "")
if provider:
try:
from docreader.parser.storage import create_storage
storage_client = create_storage(storage_map)
use_storage = True
logger.info("Using shared storage (%s) for image upload", provider)
except Exception as e:
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
# Create and return ChunkingConfig
return ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
base_dir = CONFIG.image_output_dir
output_dir = os.path.join(base_dir, request_id, "images")
wrote_to_dir = False
if not use_storage:
try:
os.makedirs(output_dir, exist_ok=True)
wrote_to_dir = True
except OSError:
logger.warning("Cannot write to image output dir %s", output_dir)
refs = []
for ref_path, b64data in images.items():
try:
img_bytes = base64.b64decode(b64data)
except Exception:
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
ext = os.path.splitext(fname)[1].lower()
mime = mime_map.get(ext, "application/octet-stream")
if use_storage and storage_client:
storage_key = storage_client.upload_bytes(img_bytes, ext)
if storage_key:
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
storage_key=storage_key,
))
continue
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
if wrote_to_dir:
dest = os.path.join(output_dir, fname)
try:
with open(dest, "wb") as f:
f.write(img_bytes)
except OSError as e:
logger.warning("Failed to write image %s: %s", dest, e)
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
image_data=img_bytes,
))
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
return image_dir, refs
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
@@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
super().__init__()
self.parser = Parser()
def ReadFromFile(self, request: ReadFromFileRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
def Read(self, request: ReadRequest, context):
"""Unified read: file mode (file_content set) or URL mode (url set)."""
request_id = request.request_id or str(uuid.uuid4())
is_url = bool(request.url)
# Use request ID context
with request_id_context(request_id):
try:
# Get file type
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
)
logger.info(f"File content size: {len(request.file_content)} bytes")
cfg = request.config
parser_engine = cfg.parser_engine if cfg else ""
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
if is_url:
logger.info("Read(URL): url=%s", request.url)
result = self.parser.parse_url(
request.url,
request.title,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.url
else:
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
"Read(File): file=%s, type=%s, size=%d bytes",
request.file_name, file_type, len(request.file_content),
)
result = self.parser.parse_file(
request.file_name,
file_type,
request.file_content,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.file_name
# Parse file
logger.info("Starting file parsing process")
result = self.parser.parse_file(
request.file_name, file_type, request.file_content, chunking_config
)
if not result:
error_msg = "Failed to parse file"
if not result or not result.content:
error_msg = f"Failed to parse: {source_desc}"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse()
# Convert to protobuf message
logger.info(
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
)
# Build response, including image info
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading file: {str(e)}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
return ReadResponse(error=str(e))
def ReadFromURL(self, request: ReadFromURLRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
# Use request ID context
with request_id_context(request_id):
try:
logger.info(f"Received ReadFromURL request for URL: {request.url}")
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
# Parse URL
logger.info("Starting URL parsing process")
result = self.parser.parse_url(
request.url, request.title, chunking_config
)
if not result:
error_msg = "Failed to parse URL"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse(error=error_msg)
# Convert to protobuf message, including image info
logger.info(
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
_c = to_valid_utf8_text
image_dir, image_refs = _resolve_images(
result.images, request_id, storage_map=storage_map
)
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
markdown_content=_c(result.content),
image_refs=image_refs,
image_dir_path=image_dir,
)
logger.info(
"Read response: content_len=%d, images=%d",
len(result.content), len(image_refs),
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading URL: {str(e)}"
error_msg = f"Error reading document: {e}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
logger.info("Traceback: %s", traceback.format_exc())
return ReadResponse(error=str(e))
def _convert_chunk_to_proto(self, chunk):
"""Convert internal Chunk object to protobuf Chunk message
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
"""
# Clean helper for strings
_c = to_valid_utf8_text
proto_chunk = Chunk(
content=_c(getattr(chunk, "content", None)),
seq=getattr(chunk, "seq", 0),
start=getattr(chunk, "start", 0),
end=getattr(chunk, "end", 0),
)
# If chunk has images attribute and is not empty, add image info
if hasattr(chunk, "images") and chunk.images:
logger.info(
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
def ListEngines(self, request, context):
overrides = dict(getattr(request, "config_overrides", None) or {})
engines_data = registry.list_engines(overrides=overrides or None)
engines = [
ParserEngineInfo(
name=e["name"],
description=e["description"],
file_types=e["file_types"],
available=e.get("available", True),
unavailable_reason=e.get("unavailable_reason", ""),
)
for img_info in chunk.images:
# img_info expected as dict
proto_image = Image(
url=_c(img_info.get("cos_url", "")),
caption=_c(img_info.get("caption", "")),
ocr_text=_c(img_info.get("ocr_text", "")),
original_url=_c(img_info.get("original_url", "")),
start=int(img_info.get("start", 0) or 0),
end=int(img_info.get("end", 0) or 0),
)
proto_chunk.images.append(proto_image)
return proto_chunk
for e in engines_data
]
return ListEnginesResponse(engines=engines)
def main():
# Print effective env/config at startup
config.print_config()
# Create server
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
options=[
@@ -286,24 +236,18 @@ def main():
],
)
# Register services
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
# Register health check service
health_servicer = HealthServicer()
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
# Set listen address
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
# Start service
server.start()
logger.info(f"Server started on port {CONFIG.grpc_port}")
logger.info("Server started on port %d", CONFIG.grpc_port)
logger.info("Server is ready to accept connections")
try:
# Wait for service termination
server.wait_for_termination()
except KeyboardInterrupt:
logger.info("Received termination signal, shutting down server")