zheromiku
dcdc8d1ddd
全栈知识治理平台首次入库（P1-P10 + 发布链路模拟验证闭环）： - 治理核心：源目录/变更集/人工审核/发布批次/发布执行（fail-closed 闸口） - 测试通道：WeKnora 测试 KB 上传/轮询/验收；poll 完成时写入 parser_generation/index_generation，修复 manual/seed 路径发布闸口永久拦截 - 发布适配器 v071（本地 source=测试角色 / 云端 target=生产角色 模拟验证通过）： * ES 导出/导入 mapping 自适应（.keyword 子字段、可选 tenant_id） * OSS locator 支持嵌套 scheme；chunks.seq_id 由目标序列分配 * bundle 归档同身份重试原子替换；verifier 按 bundle chunk_type 计数 * 回滚 job payload 支持 prior_execution_id（strict 解码） - 三通道发布+回滚演练实测：publish 4/4 verified，检索命中， rollback 4/4 rolled_back，目标端 PG is_enabled=false、ES is_enabled=false - 前端 A·Data-console 风格全站统一，120 用例通过 - 门禁：go vet / go test 全包通过，gofmt 干净
133 lines
9.0 KiB
Python
133 lines
9.0 KiB
Python
"""Refresh one source live, build a safe snapshot, and activate it.
|
|
|
|
Runs only inside the ordinary governance worker. It reads restricted credentials,
|
|
uses allowlisted read-only APIs, writes temporary files under /tmp, and never
|
|
downloads AList content or invokes source write/delete APIs.
|
|
"""
|
|
import argparse,csv,io,json,os,pathlib,subprocess,sys,tempfile
|
|
from collections import deque
|
|
sys.path.insert(0,'/app/scripts')
|
|
from source_catalog_import import build_alist_source,build_dingtalk_sources,build_pp_live_source,catalog_sql,validate_source
|
|
class RefreshBlocked(RuntimeError):pass
|
|
|
|
_source_name='';_last_directory=''
|
|
|
|
def emit(stage,progress,message,processed=0,total=0,current_directory='',scanned_directories=0,discovered_directories=0,scanned_files=0,estimated_total_files=0,total_is_estimate=False):
|
|
global _last_directory
|
|
if current_directory:_last_directory=current_directory
|
|
payload={'stage':stage,'progress':progress,'message':message,'processed':processed,'total':total}
|
|
if _source_name:payload['source_name']=_source_name
|
|
if _last_directory:payload['current_directory']=_last_directory
|
|
if scanned_directories:payload['scanned_directories']=scanned_directories
|
|
if discovered_directories:payload['discovered_directories']=discovered_directories
|
|
if scanned_files:payload['scanned_files']=scanned_files
|
|
if estimated_total_files:payload['estimated_total_files']=estimated_total_files
|
|
if total_is_estimate:payload['total_is_estimate']=True
|
|
print(json.dumps(payload,ensure_ascii=False),flush=True)
|
|
|
|
def sanitize_directory(path):
|
|
if not path:return ''
|
|
cleaned=''.join('␉' if char=='\t' else char for char in str(path) if char.isprintable() or char=='\t')
|
|
for forbidden in ('http://','https://','token','secret','password','authorization','bearer','cookie','/run/source-secrets','api-key','apikey'):
|
|
if forbidden in cleaned.lower():return ''
|
|
return cleaned[:1024]
|
|
|
|
def run_psql(sql):
|
|
url=os.environ.get('DATABASE_URL','');
|
|
if not url:raise RuntimeError('database unavailable')
|
|
result=subprocess.run(['psql','-X','-v','ON_ERROR_STOP=1','-d',url,'-At'],input=sql,text=True,capture_output=True,timeout=180)
|
|
if result.returncode:raise RuntimeError('catalog activation failed')
|
|
|
|
def refresh_pp():
|
|
sys.path.insert(0,'/sources/pp');from download_scope_a import login,fetch_inventory
|
|
emit('connecting',10,'正在登录PP资料下载中心')
|
|
try:token=login('/run/source-secrets/pp_credentials.json')
|
|
except Exception as error:raise RefreshBlocked('PP登录失败，请在来源链接更新凭据') from error
|
|
emit('scanning',35,'正在实时读取PP公开资料清单')
|
|
inventory=fetch_inventory(token)
|
|
emit('building',75,'正在构建PP八类目录快照',len(inventory.get('items',[])),len(inventory.get('items',[])))
|
|
manifest={}
|
|
try:
|
|
from pathlib import Path as _P
|
|
manifest=json.loads(_P('/sources/pp/data/public-documents/manifest.json').read_text())
|
|
except Exception:
|
|
emit('building',78,'本地内容清单暂不可用，将记录为新资料待校验')
|
|
return build_pp_live_source(inventory,manifest)
|
|
|
|
def refresh_dingtalk(name):
|
|
root=pathlib.Path('/sources/dingtalk/data/dingtalk-internal/inventory-state.json');state=json.loads(root.read_text())
|
|
matches=[(key,value) for key,value in state['roots'].items() if value.get('live_space_name')==name]
|
|
if len(matches)!=1:raise RuntimeError('DingTalk source root missing')
|
|
key,current=matches[0];output=pathlib.Path('/tmp')/('dingtalk-'+key+'-live.json')
|
|
if output.exists():output.unlink()
|
|
command=['python3','/sources/dingtalk/browser_dingtalk_scan.py','--capture','/run/source-secrets/dingtalk_capture.json','--output',str(output),'--root',f"{key}={current['configured_root_id']}",'--checkpoint-every','10']
|
|
emit('connecting',8,'正在验证钉钉专用浏览器登录')
|
|
process=subprocess.Popen(command,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True)
|
|
assert process.stdout is not None
|
|
last=10
|
|
for line in process.stdout:
|
|
if line.startswith('progress root='):
|
|
values={part.split('=',1)[0]:part.split('=',1)[1] for part in line.strip().split() if '=' in part}
|
|
done=int(values.get('folders','0'));pending=int(values.get('pending','0'));total=done+pending;files=int(values.get('files','0'))
|
|
current_progress=10+(65*done//total if total else 65);last=max(last,current_progress)
|
|
emit('scanning',last,f"正在扫描目录：已完成 {done} 个文件夹，待处理 {pending} 个",done,total,scanned_directories=done,discovered_directories=total,scanned_files=files,estimated_total_files=files,total_is_estimate=True)
|
|
if process.wait()!=0 or not output.is_file():raise RefreshBlocked('钉钉登录会话已失效，请在来源链接重新扫码')
|
|
payload=json.loads(output.read_text());sources=build_dingtalk_sources(payload);match=[source for source in sources if source['name']==name]
|
|
if len(match)!=1:raise RuntimeError('DingTalk refreshed source missing')
|
|
emit('building',80,'正在校验钉钉父子目录和文件数量')
|
|
return match[0]
|
|
|
|
def refresh_alist(name):
|
|
sys.path.insert(0,'/sources/alist');from alist_readonly_inventory import AListReadOnlyClient,join_path,read_connections
|
|
connections=[item for item in read_connections(pathlib.Path('/run/source-secrets/alist_connections.xlsx'),require_token=False) if item['category']==name]
|
|
if len(connections)!=1:raise RuntimeError('AList source configuration missing')
|
|
token_path=pathlib.Path('/run/source-secrets/alist_api_token')
|
|
try:
|
|
token=token_path.read_text(encoding='utf-8').strip()
|
|
if not token:raise ValueError('empty token')
|
|
except (OSError,ValueError) as error:raise RefreshBlocked('AList API Token 未配置，请联系管理员更新来源凭据') from error
|
|
connection=connections[0];client=AListReadOnlyClient(connection['base_url'],token);queue=deque(['/']);visited=set();directories=[];files=[];last=10
|
|
emit('connecting',8,'正在验证AList直连只读凭据')
|
|
while queue:
|
|
path=queue.popleft()
|
|
if path in visited:continue
|
|
visited.add(path)
|
|
try:items=client.list_directory(path)
|
|
except Exception as error:raise RefreshBlocked('AList直连API访问失败，请检查只读Token或来源可用性') from error
|
|
direct_dirs=direct_files=direct_bytes=0
|
|
for item in items:
|
|
item_name=str(item.get('name','')).strip()
|
|
if not item_name or item_name in ('.','..'):continue
|
|
child=join_path(path,item_name)
|
|
if bool(item.get('is_dir')):
|
|
direct_dirs+=1
|
|
if child not in visited:queue.append(child)
|
|
else:
|
|
size=max(0,int(item.get('size') or 0));direct_files+=1;direct_bytes+=size
|
|
files.append({'path':child,'parent_path':path,'name':item_name,'size':size,'modified':str(item.get('modified') or ''),'extension':pathlib.PurePosixPath(item_name).suffix.lower()})
|
|
directories.append({'path':path,'parent_path':'' if path=='/' else str(pathlib.PurePosixPath(path).parent) or '/','depth':0 if path=='/' else path.count('/'),'status':'done','direct_directories':direct_dirs,'direct_files':direct_files,'direct_bytes':direct_bytes,'subtree_files':0,'subtree_bytes':0,'last_error':''})
|
|
total=len(visited)+len(queue);progress=10+(65*len(visited)//total if total else 65);last=max(last,progress)
|
|
if len(visited)%25==0 or not queue:emit('scanning',last,f"正在扫描目录：已完成 {len(visited)} 个文件夹，待处理 {len(queue)} 个",len(visited),total,current_directory=sanitize_directory(path),scanned_directories=len(visited),discovered_directories=total,scanned_files=len(files),estimated_total_files=len(files),total_is_estimate=True)
|
|
dir_buf=io.StringIO();file_buf=io.StringIO();dw=csv.DictWriter(dir_buf,fieldnames=list(directories[0]));dw.writeheader();dw.writerows(directories);fw=csv.DictWriter(file_buf,fieldnames=['path','parent_path','name','size','modified','extension']);fw.writeheader();fw.writerows(files);dir_buf.seek(0);file_buf.seek(0)
|
|
emit('building',80,'正在校验AList父子目录和文件数量',len(visited),len(visited))
|
|
slug={'国内工业':'domestic-industrial','海外工业':'global-industrial','国内教育':'domestic-education','海外教育':'global-education'}[name]
|
|
return build_alist_source(slug,name,dir_buf,file_buf)
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser();parser.add_argument('--connection-id',required=True);parser.add_argument('--source-type',choices=('pp','dingtalk','alist'),required=True);parser.add_argument('--source-name',required=True);args=parser.parse_args()
|
|
global _source_name;_source_name=args.source_name
|
|
try:
|
|
if args.source_type=='pp':source=refresh_pp()
|
|
elif args.source_type=='dingtalk':source=refresh_dingtalk(args.source_name)
|
|
else:source=refresh_alist(args.source_name)
|
|
validate_source(source);emit('activating',92,'正在写入不可变快照并切换当前版本',source['node_count'],source['node_count'])
|
|
run_psql(catalog_sql(source,args.connection_id,apply=True));emit('completed',100,f"实时快照完成：{source['folder_count']-1} 个目录，{source['file_count']} 份文件",source['node_count'],source['node_count'])
|
|
except RefreshBlocked as error:
|
|
emit('failed',0,f"{error}；旧快照仍然保留")
|
|
return 1
|
|
except Exception:
|
|
emit('failed',0,'实时刷新发生内部错误；旧快照仍然保留')
|
|
return 1
|
|
return 0
|
|
if __name__=='__main__':raise SystemExit(main())
|