commit c000c31c2237b63f0eab6d757756a42ad64c6690 Author: SMNET Studio Date: Mon Aug 10 17:22:45 2026 +0800 chore: initial public release diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..03b14d9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +node_modules +**/node_modules +**/dist +.git +.github +.vscode +.idea +.env +.env.* +!.env.example +data +**/data +**/*.db +**/*.db-wal +**/*.db-shm +**/*.log +coverage +.turbo +**/.DS_Store +.ilink +tokens +**/*.bak +apps/api/data +packages/db/data +*.md +!docs/docker.md +!docs/upstash-redis.md +!docs/oauth-linuxdo.md diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..411c5db --- /dev/null +++ b/.env.example @@ -0,0 +1,282 @@ +# ───────────────────────────────────────────────────────────── +# 这里的值是「默认值」。除 REDIS_URL / LLM_* / LLM_PROVIDER_SECRET / +# WECHAT_AI_TOKEN / LINUXDO_ADMIN_IDS / COOKIE 与 URL 相关项之外， +# 绝大多数都可以在 /admin → 设置 页直接改，覆盖存在 Redis， +# 各节点 5 秒内生效，不必改这个文件再重启。 +# 详见 docs/runtime-settings.md +# ───────────────────────────────────────────────────────────── + +# Server（env-only：监听地址在 listen() 时固定） +# 本地开发用 127.0.0.1；Docker / 公网部署用 0.0.0.0 +WECHAT_AI_HOST=127.0.0.1 +WECHAT_AI_PORT=8787 +# 公网地址（OAuth 回调与文档用），Docker 请改成 https://你的域名 +PUBLIC_BASE_URL=http://127.0.0.1:8787 +# 跨域白名单（逗号分隔 Origin）。默认已含 PUBLIC_BASE_URL 的 origin +# CORS_ORIGINS=https://your.domain.com +# optional legacy/script token (OAuth is primary) +WECHAT_AI_TOKEN=change-me-long-random +COOKIE_SECURE=false +SESSION_COOKIE_NAME=wa_session + +# Process L1 cache over Redis (session/user/prompt). Set false to disable. +# REDIS_L1_CACHE=true +# REDIS_L1_SESSION_MS=45000 +# REDIS_L1_USER_MS=30000 +# REDIS_L1_PROMPT_MS=60000 +# 默认人设（每次开场都要读，改人设后最多滞后这个时间生效） +# REDIS_L1_DEFAULT_PERSONA_MS=60000 +# 广场列表快照（人设/表情包）。翻页、搜索、排序共用一份，免去每次 SMEMBERS+MGET 全量。 +# 多节点下，别的节点发布的新内容最多滞后这个时间才出现在广场。 +# REDIS_L1_SQUARE_MS=10000 +# 管理台仪表盘统计快照（doctorSnapshot 会扫全量 bot/peer/persona） +# REDIS_L1_DOCTOR_MS=15000 +# 仪表盘的 assignments / messages / memories 三项每个 peer 约多读 4 个 key。 +# peer 数超过这个上限就整体跳过，快照返回 deepStats=false（诊断里显示「未统计」 +# 而不是 0）。设 0 表示永不统计。 +# DOCTOR_DEEP_STATS_MAX_PEERS=5000 +# 超管身份缓存（解析一次要扫全部用户；授权/删号会立即失效） +# REDIS_L1_SUPERADMIN_MS=120000 + +# Remote Redis (required) — Upstash example (TLS = rediss://) +# Console → your DB → Connect → copy "Redis URL" for ioredis / Node +REDIS_URL=rediss://default:YOUR_UPSTASH_PASSWORD@YOUR_ENDPOINT.upstash.io:6379 +# REDIS_CONNECT_TIMEOUT_MS=15000 +# REDIS_TLS=true +# 并发命令合并进同一次往返（远端 Redis 必开）。REDIS_AUTO_PIPELINE=false 可关闭 +# REDIS_AUTO_PIPELINE=true +# REDIS_KEEPALIVE_MS=10000 +# 单命令重试次数。宁可快速失败让上层恢复，也不要盲目重试堆积 +# REDIS_MAX_RETRIES=2 +# 4=IPv4 / 6=IPv6 / 0=系统默认。不设时 Upstash 自动用 4（双栈 DNS 会多一跳） +# REDIS_FAMILY=4 + +# 静态页（app/admin/docs 外壳）启动时预压缩 br+gzip，避免每次请求现压。 +# STATIC_PRECOMPRESS=true + +# 出站超时（毫秒）。没有它，卡死的上游会一直占着一个 reply consumer +# 和该 bot:peer 的串行链，直到 OpenAI SDK 默认的 600s × 3 次重试跑完。 +# LLM_TIMEOUT_MS=120000 +# CHATFLOW_HTTP_TIMEOUT_MS=15000 + +# Multi-node labels (optional) +# WORKER_ID=node-01 +# NODE_LABEL=cn-east-1a +# NODE_REGION=cn-east +# APP_VERSION=0.2.0 +# OTA fleet update (file-diff + auto restart; default on) +# OTA_ENABLED=true +# OTA_ALLOW_INSTALL=true +# OTA_STAGING_DIR=.wa-update-staging + +# Bot tokens: Redis key wa:bot:{id}:creds (no local token files) + +# Stickers: meta + image blob in Redis (wa:sticker:* / wa:sticker:{id}:blob) +# STICKER_MAX_BYTES=2097152 +# STICKER_SEND_ENABLED=true +# MAX_STICKERS_PER_REPLY=2 + +# ── Inbound media (入站图片与语音) ── +# WeChat's own speech-to-text. Default ON and deliberately independent of +# VISION_ENABLED: the transcript arrives inside the inbound message, so using it +# costs nothing and needs no model at all. Set false to have voice notes +# answered with "didn't catch that" instead. (The audio itself is SILK/AMR, +# which no OpenAI-compatible endpoint accepts, so there is no ASR path here.) +# VOICE_TRANSCRIPT_ENABLED=true +# +# Image understanding. Master switch, default OFF — off means images are never +# downloaded and the bot simply says it cannot see them. +# VISION_ENABLED=false +# +# How the image reaches the conversation: +# caption (default) — a vision endpoint describes the image, and only that +# TEXT goes to the roleplay model. **The roleplay model needs no vision +# support at all**, so this is the mode to use with deepseek and friends. +# Bonus: the description is stored in history, so the model can still talk +# about the image several turns later. +# direct — hand the image to the roleplay model as content parts. Requires +# that model to be vision-capable; it errors outright otherwise. +# VISION_MODE=caption +# +# The vision endpoint. Admin-level creds, dialed directly like the platform LLM +# (a captioner is not user-supplied, so it does not need the tools gateway). +# Leave base/key empty to reuse LLM_BASE_URL / LLM_API_KEY — that works when the +# provider hosts a vision model alongside the chat model. +# VISION_MODEL is REQUIRED; without it every image is reported as unreadable. +# VISION_BASE_URL= +# VISION_API_KEY= +# VISION_MODEL= +# Caption length cap — a description, not an essay. +# VISION_CAPTION_MAX_TOKENS=300 +# Each image costs real tokens, so cap how many one message may spend. +# VISION_MAX_IMAGES=2 +# Per-attachment download cap (decrypted bytes; base64 adds ~1/3 on top). +# INBOUND_MEDIA_MAX_BYTES=4194304 + +# ── Platform LLM (admin) — main site connects DIRECTLY ── +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=sk-... +LLM_MODEL=gpt-4o-mini + +# ── Tools gateway (HF wechat-ai-tools) ───────────────── +# User custom OpenAI-compatible APIs + web search egress ONLY via this service. +# Main site never dials user base_url or search engines. +# Local: cd huggingface/wechat-ai-tools && docker build -t wechat-ai-tools . && docker run -p 7860:7860 --env-file .env wechat-ai-tools +# TOOLS_BASE_URL=http://127.0.0.1:7860 +# TOOLS_API_KEY=change-me-shared-with-tools-service +# TOOLS_TIMEOUT_MS=60000 +# WEB_SEARCH_ENABLED=true +# WEB_SEARCH_MAX_RESULTS=5 +# Encrypt user-stored custom API keys (required when users add providers) +# LLM_PROVIDER_SECRET=long-random-secret-at-least-16-chars + +# Chatflow: http nodes only hit tools host + this allowlist (comma hosts) +# Exact hostnames, no wildcards — api.example.com does not cover sub.api.example.com. +# Set to a single `*` to allow any PUBLIC host instead. `*` is not a blanket +# bypass: loopback, RFC1918, link-local (169.254.169.254), CGNAT +# (100.100.100.200), unique-local IPv6 and single-label/intranet names stay +# blocked, and every redirect hop is re-checked. Graphs are authored by any +# persona owner, so `*` hands every registered user a server-side fetch. +# CHATFLOW_HTTP_ALLOWLIST= +# CHATFLOW_MAX_STEPS=32 +# CHATFLOW_MAX_NODES=40 + +# LINUX DO OAuth (https://connect.linux.do) +LINUXDO_CLIENT_ID= +LINUXDO_CLIENT_SECRET= +# must match app registration callback, e.g. http://127.0.0.1:8787/api/v1/auth/callback +LINUXDO_REDIRECT_URI=http://127.0.0.1:8787/api/v1/auth/callback +# Comma-separated LINUX DO user ids and/or usernames who become admins +LINUXDO_ADMIN_IDS=12345,your_username +# Optional overrides: +# LINUXDO_AUTHORIZE_URL=https://connect.linux.do/oauth2/authorize +# LINUXDO_TOKEN_URL=https://connect.linux.do/oauth2/token +# LINUXDO_USERINFO_URL=https://connect.linux.do/api/user +# LINUXDO_SCOPE=openid profile +# First successful signup becomes admin when LINUXDO_ADMIN_IDS is empty +# FIRST_USER_IS_ADMIN=true + +# Local username+password auth (+ invite-only local register) +# LOCAL_AUTH_ENABLED=true +# PASSWORD_MIN_LENGTH=8 +# INVITE_REQUIRED_FOR_LOCAL=true +# INVITE_CODE_TTL_SEC=604800 +# INVITE_CODE_LENGTH=10 +# INVITE_MAX_PENDING_PER_USER=20 +# Per user: max INVITE_QUOTA_MAX codes every INVITE_QUOTA_WINDOW_HOURS hours +# INVITE_QUOTA_WINDOW_HOURS=24 +# INVITE_QUOTA_MAX=3 + +# Product +DEFAULT_PERSONA_SLUG=catgirl +ALLOW_UNAPPROVED_USERS=false +SHORT_HISTORY_LIMIT=20 +MEMORY_EXTRACT_EVERY_N=8 +# Memory retrieval (no embedding API): text overlap top-K when over FULL_INJECT_MAX +# MEMORY_TOP_K=12 +# MEMORY_FULL_INJECT_MAX=20 +# MEMORY_MAX_ITEMS=100 +# get_current_time tool (no external key). Set TIME_TOOL_ENABLED=false to disable. +# TIME_TOOL_ENABLED=true +# TIME_TOOL_TIMEZONE=Asia/Shanghai +PEER_RATE_PER_MINUTE=20 +# Human-like multi-bubble replies +# MULTI_BUBBLE_JSON: primary model must return {"messages":["..."]} (and sticker objects when catalog is present) +# REPLY_FILTER_ENABLED: optional second-pass AI that reformats primary reply into JSON (extra latency/cost; default off) +# SPLIT_REPLY: if JSON parse fails, fallback to punctuation/paragraph split +REPLY_FILTER_ENABLED=false +MULTI_BUBBLE_JSON=true +SPLIT_REPLY=true +MAX_REPLY_CHUNKS=5 +MAX_CHUNK_CHARS=72 +# Delay between bubbles (ms) — higher = more human typing feel +REPLY_DELAY_MS_PER_CHAR=90 +REPLY_DELAY_MIN_MS=1400 +REPLY_DELAY_MAX_MS=5500 +REPLY_DELAY_FIRST_MIN_MS=900 +REPLY_DELAY_FIRST_MAX_MS=2200 +REPLY_DELAY_THINK_EXTRA_MS=400 +# User-to-user chat via @{LINUX DO username} (bind code in /app first) +# P2P_ENABLED=true +# P2P_BIND_CODE_TTL_SEC=600 +# P2P_REQUEST_TTL_SEC=300 +# P2P_SESSION_IDLE_SEC=1800 +# P2P_RELAY_MAX_CHARS=500 +# P2P_MAX_REQUESTS_PER_DAY=20 +# Proactive outreach (idle → LLM message). Global hard off by default. +# Requires: PROACTIVE_ENABLED=true + bot switch + per-peer switch + prior context_token +# PROACTIVE_ENABLED=false +# PROACTIVE_IDLE_HOURS=12 +# PROACTIVE_MIN_INTERVAL_HOURS=24 +# PROACTIVE_MAX_PER_DAY=1 +# PROACTIVE_QUIET_HOURS=0-8 +# PROACTIVE_SCAN_INTERVAL_SEC=300 +# PROACTIVE_MAX_PER_SCAN=10 +# PROACTIVE_LOCK_TTL_SEC=180 +# PROACTIVE_ATTEMPT_COOLDOWN_HOURS=1 +# Admin broadcast (plain text to WeChat peers via /admin). Requires WORKER_ENABLED. +# Interval between messages (ms). Lower = faster; raise if iLink rate-limits. +# BROADCAST_INTERVAL_MS=200 +# BROADCAST_MAX_TEXT=2000 +# BROADCAST_HISTORY=100 +# Web try-chat (persona preview in /app, no WeChat). Set TRY_CHAT_ENABLED=false to disable. +# TRY_CHAT_ENABLED=true +# TRY_CHAT_MAX_USER_MSGS_PER_DAY=40 +# TRY_CHAT_MAX_USER_MSGS_PER_SESSION=20 +# TRY_CHAT_SESSION_TTL_SEC=3600 +# TRY_CHAT_MAX_HISTORY=40 +# Persona fork from square (default on) +# PERSONA_FORK_ENABLED=true + +# Worker (same process as API — one image / one container) +WORKER_ENABLED=true +# Cap concurrent iLink long-polls in this process (raise for more bots) +MAX_BOTS_PER_WORKER=500 +# Lease TTL for multi-replica of the same image (optional scale-out) +LEASE_TTL_SEC=45 +LEASE_RENEW_SEC=15 +# Multi-node: overloaded workers release excess leases so idle nodes can claim. +# Default is an even split; per-node load weights (0-500%, default 100) are set +# at runtime in /admin -> 节点 -> 负载权重 and stored in Redis, not here. +# REBALANCE_ENABLED=false freezes existing leases, so weights then only affect +# newly claimed bots. +# REBALANCE_ENABLED=true +# REBALANCE_INTERVAL_SEC=60 +# REBALANCE_SLACK=2 +# REBALANCE_MAX_PER_TICK=50 +# How long a node's load weight survives after its heartbeat stops, before the +# fleet deletes it automatically. Must outlast a restart / OTA apply. +# WORKER_WEIGHT_TTL_SEC=3600 +# Pin WORKER_ID per node to keep its load weight across restarts — without it +# every restart gets a fresh random id and starts back at the default 100%. +# WORKER_ID=node-a +# Concurrent LLM + WeChat send jobs (poll is decoupled from reply) +REPLY_CONCURRENCY=16 +INBOX_MAX_LEN=20000 +# silent | fatal | error | warn | info | debug | trace (anything else → info). +# Drives Fastify's logger, which emits one line per request plus framework +# errors. Successful /health probes are skipped so the container healthcheck +# does not bury everything else. +LOG_LEVEL=info +# Successful requests slower than this are logged at warn. +# LOG_SLOW_REQUEST_MS=1000 + +# ── Admin live activity stream (SSE, /admin) ── +# DATA_STREAM_ENABLED=true +# Redis command sample rate (0–1) — every command would flood the stream +# DATA_STREAM_REDIS_SAMPLE=0.08 +# Process-local cap on stream events per second +# DATA_STREAM_MAX_EPS=80 + +# Legacy sqlite path — read only by the db migrate CLI, not at runtime +# WECHAT_AI_DB_PATH=data/wechat-ai.db + +# ── Multi-node (homogeneous replicas + Cloudflare Worker LB) ── +# Every node shares the same REDIS_URL and PUBLIC_BASE_URL (main domain). +# Unique per process (recommended on multi-node): node-01, node-02, … +# WORKER_ID=node-01 +# Optional ops labels shown in admin「节点」(not public URLs) +# NODE_LABEL=cn-east-1a +# NODE_REGION=cn-east +# APP_VERSION=0.2.0 +# Source node IPs live only in cloudflare-worker ORIGINS — see cloudflare-worker/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..915e9e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,43 @@ +node_modules/ +**/node_modules/ +dist/ +.wa-version +.wa-update-staging/ +.wa-backup/ +data/ +**/data/ +.env +.env.local +*.db +*.db-journal +*.db-shm +*.db-wal +.DS_Store +*.log +.turbo/ +coverage/ +apps/web/dist/ +packages/*/dist/ +.ilink/ +tokens/ +*.bak + +# Claude / editor +.claude/ +.idea/ +.vscode/ +*.swp + +# Python (huggingface/wechat-ai-tools) +__pycache__/ +*.pyc +.venv/ +.venv +.pytest_cache/ +.hf/ + +# Wrangler (Cloudflare Worker) +cloudflare-worker/.wrangler/ + +# Local UI preview shots +scripts/ui-shots/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c3da39b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,75 @@ +# syntax=docker/dockerfile:1 +# WeChat-AI (iLink + Redis + LINUX DO OAuth) +# +# Prefer host wrappers (bump + OTA pack + docker): +# pnpm docker:build -- -- docker build -t e51l6pwpe/wxai:latest . +# pnpm docker:up +# Publish channel: /admin → 上传通道包 (dist/release//files.json) +# Plain `docker build` does NOT bump or pack. +# +# 必填环境变量见 .env.example / docs/docker.md + +FROM node:22-bookworm-slim + +LABEL org.opencontainers.image.title="wechat-ai" \ + org.opencontainers.image.description="WeChat roleplay bots via iLink, Redis, LINUX DO OAuth" + +# Keep corepack/pnpm under /pnpm so the non-root runtime user can use them +# without writing to /home/appuser/.cache (which caused EACCES). +ENV PNPM_HOME=/pnpm \ + COREPACK_HOME=/pnpm/corepack \ + PATH=/pnpm:$PATH \ + NODE_ENV=production \ + WECHAT_AI_HOST=0.0.0.0 \ + WECHAT_AI_PORT=8787 \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + +RUN corepack enable \ + && corepack prepare pnpm@11.15.0 --activate \ + && apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install deps first (better layer cache) +COPY package.json pnpm-workspace.yaml pnpm-lock.yaml tsconfig.base.json ./ +COPY apps/api/package.json ./apps/api/ +COPY packages/core/package.json ./packages/core/ +COPY packages/db/package.json ./packages/db/ +COPY packages/ilink/package.json ./packages/ilink/ +COPY packages/llm/package.json ./packages/llm/ + +# tsx is a runtime dependency of @wechat-ai/api; install all workspace packages +RUN pnpm install --frozen-lockfile + +# Application source (TypeScript run via tsx) +COPY apps/api ./apps/api +COPY packages ./packages +COPY scripts ./scripts + +# Non-root user + writable home for tools +RUN mkdir -p /home/appuser \ + && groupadd --system --gid 1001 nodejs \ + && useradd --system --uid 1001 --gid nodejs --home-dir /home/appuser --no-create-home appuser \ + && chown -R appuser:nodejs /app /pnpm /home/appuser + +USER appuser + +EXPOSE 8787 + +# scrypt (login + provider-secret KDF) runs on the libuv pool; the default 4 +# threads let a login burst queue behind itself. +ENV UV_THREADPOOL_SIZE=16 + +HEALTHCHECK --interval=30s --timeout=8s --start-period=50s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.WECHAT_AI_PORT||8787)+'/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" + +# Run node as PID 1 so SIGTERM reaches the process and the graceful shutdown +# in index.ts actually runs (releasing bot leases). `sh -c` swallowed it, so +# every deploy left up to LEASE_TTL_SEC of bots that no node was polling. +# The separate seed step is gone — index.ts already calls seedPersonas(). +# WORKDIR is apps/api because tsx is linked under apps/api/node_modules; +# resolveRepoRoot() still walks up to /app for .env and OTA paths. +WORKDIR /app/apps/api +CMD ["node", "--import", "tsx", "src/index.ts"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e097c68 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 WeChat-AI Contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..cd6be0a --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +
+ +# WeChat-AI + +**自托管微信角色扮演对话服务** · Self-hosted WeChat roleplay chatbot service + +直连腾讯 **iLink**，数据存 **远端 Redis**，登录用 **LINUX DO OAuth**。 +Connects directly to Tencent **iLink**, stores data in **remote Redis**, and authenticates via **LINUX DO OAuth**. + +[功能 Features](#功能-features) · [架构 Architecture](#架构-architecture) · [快速开始 Quick Start](#快速开始-quick-start) · [文档 Docs](#文档-docs) · [许可证 License](#许可证-license) + +[![community](https://github.com/user-attachments/assets/653f2b6b-ee32-4f0f-abe0-1ba96e4bb473)](https://linux.do/) [![Telegram Group](https://img.shields.io/badge/Telegram-Group-blue?logo=telegram&style=flat)](https://t.me/smnet_group/105727) + +
+ +--- + +## 功能 Features + +| 中文 | English | +|------|---------| +| LINUX DO OAuth 登录（用户 / 管理员） | LINUX DO OAuth login (user / admin) | +| 用户中心：扫码添加/删除微信机器人、批准私聊用户、分配人设 | User center: scan-QR add/remove WeChat bots, approve chat peers, assign personas | +| 用户对话：绑定 LINUX DO 后微信内 `@用户名` 请求对话，对方 `/同意` 后双向中继 | P2P chat: after binding, `@username` in WeChat to request a chat, peer replies `/agree` to relay messages both ways | +| 管理后台：数据面板、Token 用量、用户与机器人、部署节点、表情包审核 | Admin dashboard: stats, token usage, users & bots, deploy nodes, sticker moderation | +| 表情包广场：投稿 / 我的库 / 公开需审核；恶意图过滤 | Sticker square: submit / my library / public needs review; malicious-image filtering | +| 回复文字与图片表情（主人表情库 → 模型引用 slug → iLink CDN 发图） | Text + image sticker replies (owner sticker library → model references slug → sent via iLink CDN) | +| 输入状态指示（getconfig + sendtyping，回复送达即消失） | Typing indicator (getconfig + sendtyping, disappears when reply is delivered) | +| 入站图片理解（`VISION_ENABLED`，默认关闭；caption 模式人设模型无需视觉） | Inbound image understanding (`VISION_ENABLED`, off by default; `caption` mode needs no vision-capable roleplay model) | +| 入站语音转写（微信自带转写，默认开启） | Inbound voice transcription (WeChat built-in STT, on by default) | +| 远端 Redis 存储（bot token 与表情包均在 Redis） | Remote Redis storage (bot tokens & stickers in Redis) | +| OpenAI 兼容 LLM + 按日 Token 统计 | OpenAI-compatible LLM + daily token stats | +| 用户自定义模型 + 联网搜索（经 HF 工具网关出站，主站不直连用户 API） | User custom models + web search (egress only via HF tools gateway) | +| Chatflow：可视化编排（`/chatflow`），人设可选 prompt / chatflow 模式 | Chatflow: visual orchestration (`/chatflow`), personas support prompt / chatflow mode | +| 多节点同构部署 + Cloudflare Worker 负载均衡 | Multi-node homogeneous deployment + Cloudflare Worker LB | +| OTA 增量更新（文件差量 + 自动重启） | OTA incremental updates (file diff + auto restart) | + +## 架构 Architecture + +``` +微信用户 ──► 腾讯 iLink ──► 本系统多节点 (收消息 / 人设+记忆 / LLM / 回消息) +WeChat user ──► Tencent iLink ──► multi-node system (receive / persona+memory / LLM / reply) + +浏览器 ──► 主域名 CF Worker LB ──► Node-1…N (同一镜像, 共享 Redis) +Browser ──► main domain CF Worker LB ──► Node-1…N (same image, shared Redis) +``` + +## 快速开始 Quick Start + +### 本地开发 Local Development + +```bash +pnpm install +cp .env.example .env +# 必填 Required: REDIS_URL（Upstash 用 rediss://）、LLM_API_KEY（平台）、LINUXDO_* 、LINUXDO_ADMIN_IDS +# 用户自定义 LLM / 联网搜索：部署 huggingface/wechat-ai-tools，配置 TOOLS_BASE_URL + TOOLS_API_KEY + +pnpm db:seed +pnpm diag +pnpm dev +``` + +页面 / Pages： + +| 路径 Path | 说明 Description | +|-----------|------------------| +| `/` | 功能落地页 + OG 分享图 Landing page | +| `/app` | 用户中心 User center (LINUX DO login) | +| `/docs` | 使用文档 Documentation | +| `/admin` | 管理后台 Admin dashboard | +| `/chatflow` | Chatflow 编辑器 Chatflow editor | + +### Docker 一键部署 One-Click Deploy + +```bash +# 配置好 .env 后 After configuring .env +docker compose up -d --build +``` + +详见 / See `docs/docker.md`。 + +### 多节点 Multi-Node + +每台服务器运行**同一镜像**，共享同一个 Upstash Redis，用户只访问主域名。Cloudflare Worker 负责健康检查与轮询分流，源站地址只写在 Worker 的 `ORIGINS` 中。详见 / See `cloudflare-worker/README.md`。 + +Each server runs the **same image**, shares one Upstash Redis; users only visit the main domain. A Cloudflare Worker handles health checks and round-robin, origin addresses live only in the Worker's `ORIGINS`. + +## 文档 Docs + +| 文档 Doc | 内容 Content | +|----------|-------------| +| [docs/upstash-redis.md](docs/upstash-redis.md) | Upstash Redis 配置 Redis setup | +| [docs/oauth-linuxdo.md](docs/oauth-linuxdo.md) | LINUX DO OAuth 配置 | +| [docs/docker.md](docs/docker.md) | Docker / 多节点部署 Multi-node deploy | +| [docs/cloudflare.md](docs/cloudflare.md) | Cloudflare 缓存 Cache | +| [cloudflare-worker/README.md](cloudflare-worker/README.md) | CF Worker 多源站 LB | +| [docs/ai-gateway.md](docs/ai-gateway.md) | AI 网关（主站 ↔ HF）AI gateway | +| [docs/chatflow.md](docs/chatflow.md) | Chatflow 编排 | +| [docs/admin-api.md](docs/admin-api.md) | 管理 API | +| [docs/runbook.md](docs/runbook.md) | 运维手册 Ops runbook | +| [docs/e2e-checklist.md](docs/e2e-checklist.md) | 真机验收清单 E2E checklist | + +## 仓库结构 Repository Structure + +``` +apps/api # REST + iLink worker + Admin/App/Chatflow UI (public/*.html) +packages/ilink # iLink HTTP 客户端 iLink HTTP client +packages/db # Redis 仓储 / seed / 人设与模型连接 Redis repos / seed +packages/llm # OpenAI 兼容 chat + tools 网关客户端 +packages/core # 会话、人设、记忆、路由、chatflow 引擎 +huggingface/ # wechat-ai-tools：唯一外网 AI/搜索出口（可独立部署） +cloudflare-worker # 主域名负载均衡 Cloudflare Worker LB +docs/ # runbook、E2E 清单、ADR +scripts/ # 构建 / 打包 / 验收脚本 build / pack / accept scripts +``` + +## 验收 Acceptance + +```bash +pnpm accept # 离线自动化门禁 offline automated gate +``` + +真机清单 / Real-device checklist：`docs/e2e-checklist.md` · 状态 / Status：`docs/ACCEPTANCE.md` + +## 合规与风险 Compliance & Risk + +- 使用腾讯 **微信 ClawBot / iLink** 能力，须遵守相关使用条款。 + Using Tencent **WeChat ClawBot / iLink** capabilities requires compliance with the applicable terms. +- 个人 Bot 存在限流与处置风险；默认 **白名单用户** 才可对话。 + Personal bots face rate-limit and takedown risks; by default only **approved users** can chat. +- 角色扮演内容会经 LLM API 出机；请自行评估隐私。 + Roleplay content leaves the machine via LLM APIs; assess your own privacy posture. +- iLink 协议以实测为准，字段可能变更；适配层见 `packages/ilink`。 + The iLink protocol is based on observed behavior and may change; the adapter lives in `packages/ilink`. + +## 许可证 License + +[Apache-2.0](LICENSE) diff --git a/apps/api/package.json b/apps/api/package.json new file mode 100644 index 0000000..495fe77 --- /dev/null +++ b/apps/api/package.json @@ -0,0 +1,30 @@ +{ + "name": "@wechat-ai/api", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/index.ts", + "start": "tsx src/index.ts", + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "ilink:login": "tsx src/cli-login.ts", + "diag": "tsx src/cli-doctor.ts", + "test": "node --import tsx --test src/**/*.test.ts" + }, + "dependencies": { + "@fastify/compress": "^8.0.1", + "@wechat-ai/core": "workspace:*", + "@wechat-ai/db": "workspace:*", + "@wechat-ai/ilink": "workspace:*", + "@wechat-ai/llm": "workspace:*", + "dotenv": "^16.4.7", + "fastify": "^5.2.1", + "tsx": "^4.19.3", + "zod": "^3.24.2" + }, + "devDependencies": { + "@types/node": "^22.13.10", + "typescript": "^5.8.2" + } +} diff --git a/apps/api/public/admin.html b/apps/api/public/admin.html new file mode 100644 index 0000000..45303a7 --- /dev/null +++ b/apps/api/public/admin.html @@ -0,0 +1,14235 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + WeChat-AI 管理后台 + + + + + + +
网络已断开，部分操作可能失败
+
+
+ + WeChat-AI 管理 +
+
+ + + + + 用户中心 + + +
+
+ +
+
+ +

管理后台

+

管理员登录。可用用户名密码或 LINUX DO。

+
+ + + +
+

+ +

仅管理员账号可进入 · 登录后跳转回本页

+

返回用户中心

+
+
+ +
+ +
+
下拉刷新…
+
+
+
+

仪表盘

+

系统概览 · 今日 Token · 待批私聊

+
+
+
+ — + +
+
+ 数据可能已过期 + +
+
+ + + + + + + + +
+
+
+ +
+
+
+
+

近 7 日 Token

+

点击柱跳转到该日用量

+
+
+
+
+
+
+

今日 Token

+ +
+
+

按用户（今日）

+
+ + + + + + + +
用户Token请求
+
+

按机器人（今日）

+
+ + + + + + + +
机器人Token请求
+
+
+
+
+
+

待批准私聊

+

全站未批准的 WeChat peer（管理员可代批）

+
+
+ + + + + +
+
+
+
+ + +
+ +
+ +
+ + + + + +
Peer机器人所有者操作
+
+
+
+
+ +
+
+
+

Workers

+

全舰队运行态 · 全部节点上正在轮询的机器人 · 批量运维

+
+
+ + + +
+
+
+ — + +
+
+
+
+

Worker 运行态

+

全舰队租约 / pollable · Inbox 等计数为当前应答节点本机

+
+
+
+

+
+
+
+
+

活跃轮询（全舰队）

+

所有节点正在 long-poll 的机器人 · 标注租约所在 WORKER_ID

+
+
+ + +
+
+
+
+ + +
+
+
+
+
+
+
+

批量运维

+

启停全部 Worker 在「系统」页；此处快速跳转

+
+
+
+ + +
+
+
+ +
+
+
+

部署节点

+

多机同构副本 · Redis 心跳注册 · 不含源站 URL（用户统一走主域名）

+
+
+ + + +
+
+
+ — + +
+
+

+
+
+
+

在线 / 注册节点

+

WORKER_ID · 版本 · hostname · 租约 bot 数 · 心跳时间

+
+
+ + + +
+
+

+

通道版本：—

+ + +
+ + +
+
+ + + + + + + + + + + + + + + + + + +
状态WORKER_ID版本Hostname标签租约目标负载权重上限心跳启动
+
+
+
+
+ +
+

用量

+

按日 Token 统计（保留约 90 天）

+

—

+
+ + + + + + + + + +
+ + +
+ +
+
+ + +
+

按用户

+
+ + + + + + + +
用户Token请求
+
+
+

按机器人

+
+ + + + + + + +
机器人Token请求
+
+
+
+
+
+

近 7 日概览

+ +
+
+
+ + + +
日期Token请求
+
+
+
+
+ +
+

用户

+

LINUX DO 登录用户 · 可授予/撤销管理员

+

—

+
+
+ + +
+
+ + + + + + +
+ + + + + + + + +
+
+
+ +
+
+ + + + + + + + + + + + + +
用户ID信任机器人角色注册操作
+
+
+ +
+
+ +
+

机器人

+

全部绑定账号 · 改名 / 停 Worker / 删除

+

—

+
+
+ + +
+
+ + + + + + + + +
+ + + + + + + + + + +
+
+
+ +
+
+ + + + + + + + + + + +
名称ID所有者状态操作
+
+
+ +
+
+ +
+

广播

+

向机器人私聊用户推送纯文本 · 异步任务 · 仅有 context_token 的用户可送达

+

—

+ +
+

撰写消息

+
+ +
+
发送范围
+
+ + + +
+
+ + +
+ + + +
+

+ 注意：从未给机器人发过消息的用户没有 context_token，无法触达。任务异步执行，可在下方查看进度与取消。 +

+
+
+ +
+ + +
+
+
+ + + + + + + + + + + + +
时间状态范围进度内容操作
+
+
+ +
+
+ +
+

人设管理

+

广场与官方人设 · 创建 / 下架 / 恢复 / 编辑

+

—

+ +
+
+ + +
+
+ + + + + + +
+ + + + + + + + +
+
+ +
+
+ + + + + + + + + + + + +
名称可见作者添加状态操作
+
+
+ +
+
+ +
+

表情包广场 · 审核

+

用户投稿需审核；官方可直传自动通过 · Redis 持久 · 上传自动过滤恶意图

+

—

+ +
+
+ + +
+ + + + + +
+ +
+
+ + + + + + + + + + + + + +
预览名称 / slug作者标签审核启用操作
+
+
+
+
+ +
+

审计日志

+

关键操作记录

+

—

+ +
+
+ + +
+ + + + + + + +
+
+
+
+
+
+
+ + + +
时间动作操作者详情
+
+
+
+
+ +
+
+
+

数据流

+

实时活动：消息、Redis 抽样、Worker / LLM · 仅超管

+
+
+
+ + + 未连接 + + — +
+
+
+ + +
+ + + + + + +
+
+
+
+
连接后显示实时事件…
+
+
+
+ +
+

系统

+

健康检查、Worker 批量操作与非敏感配置（不展示密钥）

+

—

+
+
+
+
+

邀请注册策略

+

每用户每 X 小时最多生成 N 个邀请码（仅本地用户名注册需要邀请；OAuth 不受限）

+
+
+
+ + + + + + +
+

+
+
+
+
+

Worker 运行态

+

进程内 poll 租约与回复队列（同 API 进程）

+
+
+
+

+
+
+
+
+

Worker 运维

+

批量启停轮询（仅 active 且 Redis 有 token 的 bot 会启动）

+
+
+
+ + + + + + + 打开用户中心 + +
+

部署节点（进程）

+
+ + +
+
+

配置摘要

+
+
+
+

Doctor 快照

+
+
+
+
+

最近请求错误

+ +
+
  • 暂无
+
+
+
+

本会话写操作

+
+ + +
+
+
  • 暂无（本页成功的写操作会出现在这里）
+
+
+
+
+

本机运维数据

+

固定机器人、备注、排序偏好（仅浏览器本地）

+
+
+
+ + + +
+ +

+ 常用快捷键：Ctrl/⌘K / / 跳转 · F 搜索 · + g+字母 切 Tab · R 刷新 · E 导出 · + C 复制 ID · [ ] 翻页 · 主题三态 · 密度三档。 +

+
+
+ +
+

设置

+

+ 运行时配置。.env 提供默认值，这里的改动写入 Redis 并覆盖它； + 留空/恢复默认即回到 .env。 +

+

—

+ +
+
+
+

概览

+

加载中…

+
+
+ + + +
+
+
+
+ +
+ +
+ +
+
+ +
+
+
+
+ +
+ +
+ + + + + +
+ + + + API — + — +
+ + + + diff --git a/apps/api/public/app.html b/apps/api/public/app.html new file mode 100644 index 0000000..00741c4 --- /dev/null +++ b/apps/api/public/app.html @@ -0,0 +1,6287 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + WeChat-AI — 微信角色扮演机器人 + + + + + + +
+
+ + WeChat-AI +
+
+ + +
+
+ +
网络已断开，部分操作可能失败
+ + + + + +
+

用户中心

+

机器人、用户对话、人设与表情，一站管理。

+ +
+ + + + + + + +
+ +
+
+
+
+

我的机器人

+

扫码绑定微信；token 失效时点「重新扫码」可刷新凭证并保留好友/人设/记忆

+
+
+
+ + + +
+
+

等待扫码…

+ + + +
+
+ + + + + +
机器人名字ID状态操作
+
+
+
+ +
+
+
+

机器人设置 · 微信用户

+

批准私聊用户、分配人设；可对单个用户开启「主动联系」（空闲后智能体找对方聊天）

+
+
+
+ + +
+
+ + + + + + + + + + + + +
微信用户机器人状态人设主动联系操作
+
+
+
+
+ +
+
+
+
+

微信身份绑定（用户对话）

+

绑定后可在微信发送 @LINUX_DO用户名 与其他已绑定用户对话。先生成绑定码，再给任意机器人发送 /绑定 验证码。

+
+
+
+
+ + + +
+ +
+
+
+
+

黑名单（用户对话）

+

拉黑后双方无法再通过 @用户名 建立对话；进行中的会话会结束。也可在微信发送 /拉黑 用户名。

+
+
+
+ + + +
+
+
+
+ +
+
+
+
+

邀请好友注册

+

生成一次性邀请码或链接，好友可用其注册用户名+密码账号。LINUX DO 登录无需邀请。生成受配额限制（每 X 小时 N 个）。

+
+
+

加载配额…

+
+ + +
+
+
+
+ +
+
+
+
+

人设广场

+

浏览公开人设：试聊体验 → 加入库分配，或 Fork 改编成自己的版本

+
+
+
+ + + +
+
+ +
+
+ +
+
+
+
+

表情包广场

+

浏览已审核公开表情，添加到你的库后机器人对话可引用发图。公开投稿需管理员审核。

+
+
+
+ + +
+
+ +
+
+
+
+

投稿 / 上传表情

+

公开需审核；私有仅自己机器人可用。自动过滤 SVG / 脚本等多语言混合危险图。

+
+
+
+
+ + + + + +
+
+
+ + +
+
+ +
+
+
+
+
+
+
+

我创建的表情

+

审核状态与私有/公开

+
+
+
+
+
+
+
+

我的表情库

+

用于你的机器人对话回图

+
+
+
+
+
+ +
+
+
+
+

人设编辑器

+

创作或修改人设；可用变量引用机器人名字，分配给不同机器人时自动替换

+
+
+
+ +
+ + + + + +
+ + +
+ 系统提示词（人设正文） +
+ 插入变量 + + +
+ +

+ 对话时系统会自动注入「智能体身份」段，并把 + {{bot_name}} / + {{机器人名字}} + 替换为该机器人的显示名。人设可复用到多个机器人，名字随绑定机器人变化。 +

+
+
+
+ + +
+
+ + +
+
+
+
+ +
+
+
+

我创建的

+

可改公开/私有，或删除

+
+
+
+
+ +
+
+
+

我的库

+

已添加的人设，可用于机器人分配

+
+
+
+
+
+ +
+
+
+
+

我的模型连接

+

自定义 OpenAI 兼容 API。主站不会直连你的地址，请求经 HuggingFace 工具服务（TOOLS）代发；密钥加密存储，日志不打印明文。

+
+
+

加载网关状态…

+
+
+ + +
+ + +
+
+ + +
+
+
+
+
+
+
+ +
+
+
+

确认操作

+

+
+ + +
+
+
+ + + + + + + + + + + + diff --git a/apps/api/public/chatflow.html b/apps/api/public/chatflow.html new file mode 100644 index 0000000..33071a3 --- /dev/null +++ b/apps/api/public/chatflow.html @@ -0,0 +1,2214 @@ + + + + + + + + + + + + Chatflow 编辑器 — WeChat-AI + + + + + + + + + + + + +
+ + + + Chatflow 编辑器 + 选择人设后编辑流程 + + +
+ + 返回 +
+
+ +
+
+ + + + + +
+ + +
+ +
+
+ +

需要登录

+

请先在用户中心登录，再打开 Chatflow 编辑器。

+ 前往用户中心登录 +
+
+ +
+ + +
+
+ + + +
+
选择一个人设以载入流程图
+ +
+ +
100%
+ + +
+
+ + +
+ + +
+ +
+ + + + diff --git a/apps/api/public/docs.html b/apps/api/public/docs.html new file mode 100644 index 0000000..4fba1f0 --- /dev/null +++ b/apps/api/public/docs.html @@ -0,0 +1,1204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 使用文档 — WeChat-AI + + + + + + + +
+ + + WeChat-AI + +
+ + 首页 + 用户中心 +
+
+ +
+
+ + +
+
+
使用文档
+

WeChat-AI 用户手册

+

+ 说明如何登录、绑定微信机器人、和好友角色扮演对话，以及通过 + @LINUX DO 用户名 与其他用户私聊。面向普通用户，无需写代码。 +

+
+ +
+

1 简介

+

+ WeChat-AI 是一个自托管微信角色扮演服务：你用 + LINUX DO 账号登录网页，扫码绑定自己的微信 ClawBot， + 批准好友后即可在微信里和 AI 角色聊天。系统还会支持已绑定用户之间的 + 用户对话（机器人中继，不经过 AI）。 +

+
+
+

网页端

+

用户中心 /app：机器人、用户对话、人设、表情、绑定码、黑名单。

+
+
+

微信端

+

给机器人发文字即可聊天；可用 /绑定、@用户名 等命令。

+
+
+
+ +
+

2 快速开始

+
+
+
1
+
+

登录

+

打开 用户中心，使用 LINUX DO 账号授权登录。

+
+
+
+
2
+
+

扫码添加机器人

+

在「我的机器人」点击扫码添加，用微信扫码完成 ClawBot 绑定。

+
+
+
+
3
+
+

让好友私聊机器人

+

好友在微信中找到该机器人并发一条消息；你在用户中心批准后即可对话。

+
+
+
+
4
+
+

（可选）分配人设

+

从人设广场添加角色，在用户列表中为人设分配给具体好友。

+
+
+
+
+
提示
+

默认仅已批准的微信用户可以和 AI 聊天。未批准时机器人会提示联系主人开通。

+
+
+ +
+

3 管理机器人

+
    +
  • 扫码添加：创建新的微信机器人账号。
    • +
  • 重新扫码：token 失效时刷新凭证；会保留已批准用户、人设分配与记忆。
    • +
  • 改名 / 删除：在机器人卡片或列表中操作。
    • +
+
+
注意
+

请遵守腾讯微信 / ClawBot 相关使用条款。个人 Bot 可能有限流与风控；建议控制开放范围（白名单批准）。

+
+
+ +
+

4 批准微信好友

+
    +
  1. 对方给机器人发任意消息后，会出现在「微信用户」列表。
    2. +
  2. 点击批准，可选同时分配人设。
    3. +
  3. 对方即可开始角色扮演对话（AI 可能分多条气泡回复）。
    4. +
+

可在列表中查看记忆、清空记忆，或开启「允许主动」（见下文主动联系）。

+
+ +
+

5 人设

+
    +
  • 人设广场：按热度浏览公开角色；可试聊（网页限量体验，不写入微信）、加入库或改编（复制为你的私有草稿，可自由改写）。
    • +
  • 我的人设：创建 / 编辑自己的角色提示词；可设为公开或私有；改编来的人设会标注「改编自」。
    • +
  • 分配：在微信用户行选择人设，仅对应该用户生效；分配次数会提升广场热度。
    • +
+

微信内发送 /角色 不会切换角色——角色只能由主人在后台分配。

+

试聊有每日与单会话条数上限，计入 Token 用量；管理员可通过环境变量关闭。

+
+ +
+

6 表情包

+
    +
  • 在表情包广场浏览、加入自己的库；也可上传投稿（公开需管理员审核）。
    • +
  • 对话中 AI 可能引用库中的 slug 以图片表情回复（需主人库中可用）。
    • +
+
+ +
+

7 用户对话（@LINUX DO 用户名）

+

+ 两个都完成「微信身份绑定」的用户，可以在微信里通过机器人 + 互相发消息（中继，不调用 AI）。用户名即 LINUX DO 的 + username。 +

+ +

7.1 绑定身份

+
+
+
1
+
+

生成绑定码

+

用户中心 → 用户对话 → 微信身份绑定 → 生成绑定码（约 10 分钟有效）。

+
+
+
+
2
+
+

微信确认

+

给任意已绑定的机器人发送：/绑定 ABC123（替换为你的码）。

+
+
+
+
3
+
+

先聊一句

+

绑定后请给机器人发过至少一条消息，系统才有推送凭证；否则别人 @ 你时会提示「不可达」。

+
+
+
+
+
查询身份
+

微信发送 /我的身份 可查看当前绑定的 LINUX DO 用户名与会话状态。

+
+ +

7.2 发起与聊天

+
    +
  1. 整条消息只发：@对方用户名（例如 @alice）。
    2. +
  2. 对方收到「对话请求」，回复 /同意 或 /拒绝。
    3. +
  3. 同意后双方直接发文字；对方会看到类似 [alice] 你好 的前缀。
    4. +
  4. 任一方发送 /断开 结束；长时间无消息（默认约 30 分钟）也会自动结束，并恢复与 AI 角色扮演。
    5. +
+
+
限制
+
    +
  • 会话中仅支持文字；图片 / 语音会提示不支持。
    • +
  • hello @alice 这类夹在句子里的写法不会发起请求。
    • +
  • 每个账号同一时间通常只能有一个进行中的请求或会话。
    • +
+
+
+ +
+

8 黑名单

+

不想被某人 @ 时可以拉黑（双向无法再建立用户对话）。

+
    +
  • 网页：用户中心 → 用户对话 → 黑名单 → 输入用户名拉黑 / 取消拉黑。
    • +
  • 微信：/拉黑 用户名、/取消拉黑 用户名、/黑名单。
    • +
+

拉黑会结束与对方进行中的用户对话请求或会话。

+
+ +
+

9 微信命令一览

+ + + + + + + + + + + + + + + + + + +
命令说明
/绑定 CODE认领用户中心生成的绑定码
/解绑解除微信与 LINUX DO 的绑定
/我的身份查看绑定用户名与当前状态
@用户名整条消息：向对方发起对话请求
/同意同意入站对话请求
/拒绝拒绝入站对话请求
/取消请求取消自己发出的请求
/断开结束当前用户对话，恢复 AI
/拉黑 用户名拉黑（可选前缀 @）
/取消拉黑 用户名移出黑名单
/黑名单查看黑名单列表
/角色 …不支持自助切换；由主人在后台分配
+

未匹配上述命令、且不在用户会话中时，消息会交给 AI 角色扮演回复（需已批准）。

+
+ +
+

10 主动联系（可选）

+

若站点开启了主动联系，空闲一段时间后角色可能主动给用户发消息。

+
    +
  1. 机器人卡片中打开「主动找用户聊天」并保存参数。
    2. +
  2. 对具体微信用户勾选「允许主动」。
    3. +
  3. 对方须曾经聊过机器人（有会话凭证）。
    4. +
+

具体是否开启由站点管理员配置；个人用户仅能控制自己的机器人与好友开关。

+
+ +
+

11 常见问题

+

对方收不到我的 @ 请求？

+
    +
  • 双方是否都完成了 /绑定？
    • +
  • 对方是否用绑定的微信给机器人发过消息（可达）？
    • +
  • 是否在黑名单中？用户名是否拼写正确（不区分大小写）？
    • +
  • 对方是否正忙（已有请求 / 会话）？
    • +
+

AI 不回复？

+
    +
  • 是否已在用户中心批准该微信用户？
    • +
  • 是否在用户对话会话中？会话内不会走 AI，需先 /断开。
    • +
  • 发得太快会触发限流提示。
    • +
+

能发图片 / 语音吗？

+
    +
  • + 语音：微信自带的语音转文字会一并送到机器人，所以带转写的语音就当文字聊。 +
    • +
  • + 图片：站点开启「图片理解」后，机器人能看图并据实描述；未开启时会直接告诉你看不了。 +
    • +
  • 视频 / 文件：暂时看不了，机器人会说明并请你用文字描述。
    • +
+

绑定码无效？

+

绑定码有时效且只能用一次。请在用户中心重新生成，再发 /绑定 新码。

+

机器人突然不回？

+

可能是微信会话过期。在用户中心对该机器人使用重新扫码刷新凭证（数据会保留）。

+
+
需要帮助
+

+ 进入 用户中心 管理机器人与绑定；站点首页见 + /。管理员后台为 /admin（需管理员权限）。 +

+
+
+
+
+
+ + + + + + diff --git a/apps/api/public/index.html b/apps/api/public/index.html new file mode 100644 index 0000000..24f9fa1 --- /dev/null +++ b/apps/api/public/index.html @@ -0,0 +1,987 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + WeChat-AI — 微信角色扮演机器人 + + + + + + +
+ + + WeChat-AI + +
+ + 使用文档 + 用户中心 + 开始使用 +
+
+ +
+
+
+
+
+ 自托管 + LINUX DO 登录 + iLink 直连 +
+

用微信，跑起你的角色扮演智能体

+

+ WeChat-AI 把扫码绑定、人设广场、表情包回图、长期记忆与私聊分配放在一个面板里。 + 登录后几分钟即可添加机器人，和微信好友开始对话。 +

+
+ + + 进入用户中心 + + + + 使用文档 + +
+

支持 LINUX DO OAuth · 多机器人 · 人设 / 表情广场

+
+
+ WeChat-AI 产品预览 +
+
+ +
+
+

核心能力

+

面向多用户的微信角色扮演服务，从绑定到运营一条链路打通。

+
+
+
+
+

扫码绑定机器人

+

微信扫码添加自己的机器人；token 失效可「重新扫码」，好友、人设与记忆会保留。

+
+
+
+

人设广场

+

浏览、投稿与收藏公开人设；为自己的机器人分配角色，随时切换性格与提示词。

+
+
+
+

表情包广场

+

投稿 / 收藏表情；模型可按 slug 引用，机器人通过 iLink 回发图片表情。

+
+
+
+

私聊批准与分配

+

默认白名单模式：批准微信用户后才可对话，并为每个 peer 指定人设。

+
+
+
+

长期记忆

+

跨会话记住关键事实与偏好，让角色扮演更连贯；可按人设分组查看与清理。

+
+
+
+

主动联系

+

空闲一段时间后，智能体可按配置主动找对方聊天，可设安静时段与每日上限。

+
+
+
+ +
+
+

如何开始

+

四步完成从登录到和微信好友对话。

+
+
+
+

LINUX DO 登录

+

在用户中心使用 LINUX DO 账号登录，创建个人空间。

+
+
+

扫码加机器人

+

在「机器人」页扫码绑定微信 ClawBot / iLink 账号。

+
+
+

选人设与表情

+

从广场添加人设与表情到自己的库，再分配给机器人。

+
+
+

批准好友对话

+

批准私聊用户后即可收消息、LLM 回复文字与表情。

+
+
+
+ +
+

准备好了？

+

登录用户中心，添加第一台机器人，开始角色扮演。

+
+ + + 进入用户中心 + + 使用文档 + 管理后台 +
+
+ +

+ 合规提示：使用腾讯微信 ClawBot / iLink 能力须遵守相关条款；个人 Bot 存在限流与处置风险。 + 默认仅白名单用户可对话。角色扮演内容会经 LLM API 出机，请自行评估隐私与内容安全。 +

+
+
+ + + + + + + + diff --git a/apps/api/public/og.jpg b/apps/api/public/og.jpg new file mode 100644 index 0000000..a73d2d9 Binary files /dev/null and b/apps/api/public/og.jpg differ diff --git a/apps/api/src/activity-stream.test.ts b/apps/api/src/activity-stream.test.ts new file mode 100644 index 0000000..5af0440 --- /dev/null +++ b/apps/api/src/activity-stream.test.ts @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + previewText, + redactRedisKey, + ActivityBus, +} from "./activity-stream.js"; + +describe("previewText", () => { + it("keeps short text", () => { + const r = previewText("hello", 48); + assert.equal(r.preview, "hello"); + assert.equal(r.len, 5); + assert.equal(r.truncated, false); + }); + + it("truncates long text", () => { + const s = "a".repeat(100); + const r = previewText(s, 48); + assert.equal(r.len, 100); + assert.equal(r.truncated, true); + assert.ok(r.preview.endsWith("…")); + assert.equal(r.preview.length, 49); + }); +}); + +describe("redactRedisKey", () => { + it("redacts creds keys", () => { + assert.equal( + redactRedisKey("wa:bot:abc:creds"), + "wa:bot:abc:*", + ); + }); + + it("passes normal keys", () => { + assert.equal(redactRedisKey("wa:msgs:b1:p1"), "wa:msgs:b1:p1"); + }); +}); + +describe("ActivityBus", () => { + it("rings and rate-limits", () => { + const fakeDb = { + redis: { + duplicate: () => ({ + on() {}, + subscribe: async () => {}, + unsubscribe: async () => {}, + disconnect() {}, + }), + pipeline: () => ({ + lpush() { + return this; + }, + ltrim() { + return this; + }, + publish() { + return this; + }, + exec: async () => [], + }), + publish: async () => 0, + lrange: async () => [], + }, + } as any; + + const bus = new ActivityBus({ + db: fakeDb, + source: "test", + enabled: true, + maxEps: 5, + ringSize: 10, + redisSample: 0, + }); + + const got: string[] = []; + bus.subscribe((ev) => got.push(ev.type)); + + for (let i = 0; i < 10; i++) { + bus.emit({ type: "worker.job", summary: `n=${i}` }, { fleet: false, persist: false }); + } + const jobs = got.filter((t) => t === "worker.job"); + assert.equal(jobs.length, 5); + // optional stream.dropped summary when over cap + assert.ok(got.length >= 5 && got.length <= 6); + assert.equal( + bus.recentLocal(20).filter((e) => e.type === "worker.job").length, + 5, + ); + + // redis samples stay local path + bus.noteRedisCmd({ op: "get", key: "wa:user:1", ms: 2, ok: true }); + // sample may or may not fire (random); just ensure no throw + }); +}); diff --git a/apps/api/src/activity-stream.ts b/apps/api/src/activity-stream.ts new file mode 100644 index 0000000..aa6c253 --- /dev/null +++ b/apps/api/src/activity-stream.ts @@ -0,0 +1,465 @@ +/** + * Admin live activity bus: in-process listeners + Redis Pub/Sub fan-in. + * + * redis.cmd samples stay local (never LPUSH) to avoid Upstash recursion. + * Important domain events (message / worker / llm) may PUBLISH + optional backlog. + */ +import { randomUUID } from "node:crypto"; +import type { Db } from "@wechat-ai/db"; +import { K } from "@wechat-ai/db"; + +export type StreamLevel = "info" | "warn" | "error"; + +export type StreamEvent = { + id: string; + ts: string; + type: string; + level?: StreamLevel; + source?: string; + summary: string; + data?: Record; +}; + +export type EmitInput = { + type: string; + summary: string; + level?: StreamLevel; + source?: string; + data?: Record; + /** Override auto id */ + id?: string; + /** Override ISO ts */ + ts?: string; +}; + +export type EmitOpts = { + /** LPUSH to Redis recent list (default: true for important domain types) */ + persist?: boolean; + /** PUBLISH to fleet channel (default: true except redis.cmd / stream.*) */ + fleet?: boolean; +}; + +export type ActivityBusOptions = { + db: Db; + source?: string; + enabled?: boolean; + /** Max events accepted per second (process-local) */ + maxEps?: number; + ringSize?: number; + backlogSize?: number; + /** Redis cmd sample rate 0..1 */ + redisSample?: number; + /** Max redis.cmd events per second */ + redisMaxEps?: number; +}; + +type Listener = (ev: StreamEvent) => void; + +const IMPORTANT_PREFIXES = ["message.", "worker.", "llm."]; + +function isImportantType(type: string): boolean { + return IMPORTANT_PREFIXES.some((p) => type.startsWith(p)); +} + +function shouldFleetDefault(type: string): boolean { + if (type.startsWith("redis.")) return false; + if (type.startsWith("stream.")) return false; + return isImportantType(type); +} + +function shouldPersistDefault(type: string): boolean { + return shouldFleetDefault(type); +} + +/** Truncate message body for stream privacy. */ +export function previewText( + text: string | null | undefined, + maxChars: number, +): { preview: string; len: number; truncated: boolean } { + const raw = text ?? ""; + const len = raw.length; + if (len <= maxChars) { + return { preview: raw, len, truncated: false }; + } + return { + preview: raw.slice(0, Math.max(0, maxChars)) + "…", + len, + truncated: true, + }; +} + +/** Redact Redis key for stream display (keep pattern, drop secrets-ish tails). */ +export function redactRedisKey(key: string | undefined | null): string { + if (!key) return ""; + const s = String(key); + // session / creds / blob: keep prefix only + if (/:creds$/i.test(s) || /:blob$/i.test(s) || /:session:/i.test(s)) { + const parts = s.split(":"); + return parts.slice(0, Math.min(3, parts.length)).join(":") + ":*"; + } + if (s.length > 96) return s.slice(0, 93) + "…"; + return s; +} + +export class ActivityBus { + private readonly db: Db; + private readonly source: string; + /** Admin-editable at runtime — see applyRuntimeOptions(). */ + private enabled: boolean; + private maxEps: number; + private readonly ringSize: number; + private readonly backlogSize: number; + private redisSample: number; + private readonly redisMaxEps: number; + + private ring: StreamEvent[] = []; + private listeners = new Set(); + private sub: ReturnType | null = null; + private started = false; + private closed = false; + + private windowStart = Date.now(); + private windowCount = 0; + private dropped = 0; + private lastDropReport = 0; + + private redisWindowStart = Date.now(); + private redisWindowCount = 0; + + /** Dedup fleet + local echoes (id → expiry ms) */ + private seen = new Map(); + private readonly seenTtlMs = 60_000; + + constructor(opts: ActivityBusOptions) { + this.db = opts.db; + this.source = opts.source || "api"; + this.enabled = opts.enabled !== false; + this.maxEps = Math.max(5, opts.maxEps ?? 80); + this.ringSize = Math.max(50, opts.ringSize ?? 500); + this.backlogSize = Math.max(50, opts.backlogSize ?? 300); + this.redisSample = Math.min(1, Math.max(0, opts.redisSample ?? 0.08)); + this.redisMaxEps = Math.max(1, opts.redisMaxEps ?? 15); + } + + /** + * Apply admin-editable settings in place (runtime settings reload). + * Turning the stream on lazily opens the Redis subscriber, which start() + * would otherwise only ever do at boot. + */ + applyRuntimeOptions(patch: { + enabled?: boolean; + maxEps?: number; + redisSample?: number; + }): void { + if (patch.maxEps !== undefined) this.maxEps = Math.max(5, patch.maxEps); + if (patch.redisSample !== undefined) { + this.redisSample = Math.min(1, Math.max(0, patch.redisSample)); + } + if (patch.enabled === undefined || patch.enabled === this.enabled) return; + this.enabled = patch.enabled; + if (this.enabled && !this.started && !this.closed) { + void this.start().catch(() => undefined); + } + } + + isEnabled(): boolean { + return this.enabled && !this.closed; + } + + getSource(): string { + return this.source; + } + + async start(): Promise { + if (!this.enabled || this.started || this.closed) return; + this.started = true; + try { + const sub = this.db.redis.duplicate(); + this.sub = sub; + sub.on("error", (err: Error) => { + if (process.env.LOG_LEVEL === "debug") { + console.error("[stream] sub error", err.message); + } + }); + await sub.subscribe(K.streamChannel); + sub.on("message", (_ch: string, raw: string) => { + try { + const ev = JSON.parse(raw) as StreamEvent; + if (!ev?.id || !ev?.type) return; + this.ingestRemote(ev); + } catch { + /* ignore bad payload */ + } + }); + } catch (err) { + console.warn( + "[stream] fleet subscribe unavailable:", + err instanceof Error ? err.message : err, + ); + } + } + + async stop(): Promise { + this.closed = true; + this.listeners.clear(); + if (this.sub) { + try { + await this.sub.unsubscribe(K.streamChannel); + this.sub.disconnect(); + } catch { + /* */ + } + this.sub = null; + } + } + + subscribe(fn: Listener): () => void { + this.listeners.add(fn); + return () => { + this.listeners.delete(fn); + }; + } + + /** Newest-first local ring snapshot. */ + recentLocal(limit = 100): StreamEvent[] { + const n = Math.max(1, Math.min(limit, this.ring.length)); + return this.ring.slice(0, n); + } + + async recentMerged(limit = 100): Promise { + const n = Math.max(1, Math.min(limit, 300)); + let remote: StreamEvent[] = []; + try { + const raw = await this.db.redis.lrange(K.streamRecent, 0, n - 1); + remote = raw + .map((r) => { + try { + return JSON.parse(r) as StreamEvent; + } catch { + return null; + } + }) + .filter((x): x is StreamEvent => !!x?.id); + } catch { + remote = []; + } + const local = this.recentLocal(n); + const map = new Map(); + for (const ev of [...remote, ...local]) { + if (!map.has(ev.id)) map.set(ev.id, ev); + } + return [...map.values()] + .sort((a, b) => (a.ts < b.ts ? 1 : a.ts > b.ts ? -1 : 0)) + .slice(0, n); + } + + /** + * Emit a stream event. Rate-limited; may drop under load. + */ + emit(input: EmitInput, opts: EmitOpts = {}): StreamEvent | null { + if (!this.isEnabled()) return null; + + const now = Date.now(); + if (now - this.windowStart >= 1000) { + this.windowStart = now; + this.windowCount = 0; + } + if (this.windowCount >= this.maxEps) { + this.dropped++; + this.maybeReportDrops(now); + return null; + } + this.windowCount++; + + const ev: StreamEvent = { + id: input.id || `sev_${randomUUID().replace(/-/g, "").slice(0, 16)}`, + ts: input.ts || new Date().toISOString(), + type: input.type, + level: input.level || "info", + source: input.source || this.source, + summary: input.summary, + data: input.data, + }; + + this.deliverLocal(ev); + + const fleet = opts.fleet ?? shouldFleetDefault(ev.type); + const persist = opts.persist ?? shouldPersistDefault(ev.type); + + if (fleet || persist) { + void this.fanOut(ev, { fleet, persist }); + } + + return ev; + } + + /** Sampled redis command hook (local only). */ + noteRedisCmd(info: { + op: string; + key?: string; + keys?: number; + ms?: number; + ok?: boolean; + }): void { + if (!this.isEnabled() || this.redisSample <= 0) return; + + const now = Date.now(); + if (now - this.redisWindowStart >= 1000) { + this.redisWindowStart = now; + this.redisWindowCount = 0; + } + if (this.redisWindowCount >= this.redisMaxEps) return; + if (Math.random() > this.redisSample) return; + this.redisWindowCount++; + + const keyLabel = info.key + ? redactRedisKey(info.key) + : info.keys != null + ? `${info.keys} key(s)` + : ""; + const msPart = info.ms != null ? ` ${Math.round(info.ms)}ms` : ""; + const okPart = info.ok === false ? " fail" : ""; + this.emit( + { + type: "redis.cmd", + level: info.ok === false ? "warn" : "info", + summary: `${info.op.toUpperCase()}${keyLabel ? " " + keyLabel : ""}${msPart}${okPart}`, + data: { + op: info.op, + key: keyLabel || undefined, + keys: info.keys, + ms: info.ms, + ok: info.ok !== false, + }, + }, + { fleet: false, persist: false }, + ); + } + + private maybeReportDrops(now: number): void { + if (this.dropped <= 0) return; + if (now - this.lastDropReport < 5000) return; + const n = this.dropped; + this.dropped = 0; + this.lastDropReport = now; + // Bypass rate limit for meta by delivering directly + const ev: StreamEvent = { + id: `sev_drop_${now.toString(36)}`, + ts: new Date().toISOString(), + type: "stream.dropped", + level: "warn", + source: this.source, + summary: `rate limit: dropped ${n} event(s) in last window`, + data: { dropped: n, maxEps: this.maxEps }, + }; + this.deliverLocal(ev); + } + + private ingestRemote(ev: StreamEvent): void { + if (this.markSeen(ev.id)) return; + // Do not re-publish remote events + this.pushRing(ev); + for (const fn of this.listeners) { + try { + fn(ev); + } catch { + /* listener errors must not break bus */ + } + } + } + + private deliverLocal(ev: StreamEvent): void { + if (this.markSeen(ev.id)) return; + this.pushRing(ev); + for (const fn of this.listeners) { + try { + fn(ev); + } catch { + /* */ + } + } + } + + private markSeen(id: string): boolean { + const now = Date.now(); + if (this.seen.size > 4000) { + for (const [k, exp] of this.seen) { + if (exp < now) this.seen.delete(k); + } + if (this.seen.size > 4000) { + // drop oldest ~20% + let i = 0; + const n = Math.ceil(this.seen.size * 0.2); + for (const k of this.seen.keys()) { + this.seen.delete(k); + if (++i >= n) break; + } + } + } + if (this.seen.has(id)) return true; + this.seen.set(id, now + this.seenTtlMs); + return false; + } + + private pushRing(ev: StreamEvent): void { + this.ring.unshift(ev); + if (this.ring.length > this.ringSize) { + this.ring.length = this.ringSize; + } + } + + private async fanOut( + ev: StreamEvent, + opts: { fleet: boolean; persist: boolean }, + ): Promise { + // Never persist/fleet full message bodies — local ring + SSE keep fullText + const fleetEv = stripSensitiveStreamData(ev); + const raw = JSON.stringify(fleetEv); + try { + if (opts.persist) { + const pipe = this.db.redis.pipeline(); + pipe.lpush(K.streamRecent, raw); + pipe.ltrim(K.streamRecent, 0, this.backlogSize - 1); + if (opts.fleet) pipe.publish(K.streamChannel, raw); + await pipe.exec(); + } else if (opts.fleet) { + await this.db.redis.publish(K.streamChannel, raw); + } + } catch { + /* non-fatal */ + } + } +} + +/** Drop fullText/text before Redis pub/backlog (privacy + size). */ +function stripSensitiveStreamData(ev: StreamEvent): StreamEvent { + if (!ev.data) return ev; + if (!("fullText" in ev.data) && !("text" in ev.data)) return ev; + const data = { ...ev.data }; + delete data.fullText; + delete data.text; + return { ...ev, data }; +} + +// ── Singleton ────────────────────────────────────────── + +let bus: ActivityBus | null = null; + +export function initActivityBus(opts: ActivityBusOptions): ActivityBus { + if (bus) { + void bus.stop(); + } + bus = new ActivityBus(opts); + return bus; +} + +export function getActivityBus(): ActivityBus | null { + return bus; +} + +export function emitActivity( + input: EmitInput, + opts?: EmitOpts, +): StreamEvent | null { + return bus?.emit(input, opts) ?? null; +} diff --git a/apps/api/src/bot-login-sessions.ts b/apps/api/src/bot-login-sessions.ts new file mode 100644 index 0000000..b9febeb --- /dev/null +++ b/apps/api/src/bot-login-sessions.ts @@ -0,0 +1,386 @@ +import { + ILinkClient, + ILinkError, + resolveQrOpenUrl, + type QrcodeStatusResponse, +} from "@wechat-ai/ilink"; +import { + type Db, + BOT_LOGIN_TTL_SEC, + type BotLoginSessionRecord, + getBotAccount, + getBotLoginSession, + markBotLoginCancelled, + saveBotLoginSession, + upsertBotAccount, + writeAudit, +} from "@wechat-ai/db"; +import type { BotWorkerManager } from "./worker.js"; + +export type LoginSessionStatus = BotLoginSessionRecord["status"]; +export type LoginMode = BotLoginSessionRecord["mode"]; +export type LoginSessionView = BotLoginSessionRecord; + +interface InternalSession { + view: LoginSessionView; + client: ILinkClient; + qrcode: string; + timer?: ReturnType; + stopped: boolean; +} + +/** + * QR bot login: poll loop is process-local (holds ILinkClient), + * but the session **view** is stored in Redis so any replica can + * serve GET status / cancel under load balancing. + */ +export class BotLoginSessionManager { + /** Local poll ownership only — status is authoritative in Redis. */ + private sessions = new Map(); + + constructor( + private db: Db, + private worker: BotWorkerManager, + ) {} + + /** + * Start QR login. + * - create (default): new bot id + * - rebind: update token on existing botId (peers / memories kept) + */ + async start( + ownerUserId: string, + displayName?: string, + opts?: { rebindBotId?: string }, + ): Promise { + const rebindBotId = opts?.rebindBotId?.trim(); + let mode: LoginMode = "create"; + let name = + (displayName?.trim() || "").slice(0, 64) || + `bot-${Date.now().toString(36).slice(-6)}`; + + if (rebindBotId) { + const bot = await getBotAccount(this.db, rebindBotId); + if (!bot) { + throw new Error("bot not found"); + } + // Ownership is enforced by the HTTP route before calling start() + mode = "rebind"; + name = (displayName?.trim() || bot.display_name || name).slice(0, 64); + } + + const client = new ILinkClient({ timeoutMs: 30_000 }); + const qr = await client.getBotQrcode(3); + if (!qr.qrcode) { + throw new ILinkError( + qr.errmsg ?? "get_bot_qrcode failed", + qr.ret, + undefined, + qr, + ); + } + + const sessionId = `login_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`; + const now = new Date().toISOString(); + const openUrl = resolveQrOpenUrl(qr); + + const view: LoginSessionView = { + sessionId, + displayName: name, + ownerUserId, + status: "wait_scan", + mode, + rebindBotId: rebindBotId || undefined, + qrcode: qr.qrcode, + openUrl, + message: + mode === "rebind" + ? `重新绑定「${name}」：请用微信扫码（好友/人设/记忆将保留）` + : "请用微信扫描二维码（或打开下方链接）", + createdAt: now, + updatedAt: now, + }; + + const session: InternalSession = { + client, + qrcode: qr.qrcode, + stopped: false, + view, + }; + this.sessions.set(sessionId, session); + await this.persist(view); + void this.pollLoop(sessionId); + session.timer = setTimeout(() => { + void this.expireLocal(sessionId); + }, BOT_LOGIN_TTL_SEC * 1000); + return { ...view }; + } + + /** Any node: read shared Redis view (falls back to local if still hot). */ + async get(sessionId: string): Promise { + const remote = await getBotLoginSession(this.db, sessionId); + if (remote) return { ...remote }; + const local = this.sessions.get(sessionId); + return local ? { ...local.view } : undefined; + } + + /** + * Cancel from any node: mark Redis cancelled so the poller exits; + * drop local resources if this process owns the poll. + */ + async cancel(sessionId: string, ownerUserId?: string): Promise { + if (!ownerUserId) { + const cur = await getBotLoginSession(this.db, sessionId); + if (!cur) { + const local = this.sessions.get(sessionId); + if (!local) return false; + local.stopped = true; + this.dropLocal(sessionId); + return true; + } + ownerUserId = cur.ownerUserId; + } + + const marked = await markBotLoginCancelled( + this.db, + sessionId, + ownerUserId, + ); + if (!marked) { + // Maybe only local (Redis miss) — try local ownership check + const local = this.sessions.get(sessionId); + if (!local) return false; + if (local.view.ownerUserId !== ownerUserId) return false; + local.stopped = true; + this.dropLocal(sessionId); + return true; + } + + const local = this.sessions.get(sessionId); + if (local) { + local.stopped = true; + local.view = { ...marked }; + this.dropLocal(sessionId); + } + return true; + } + + private async persist(view: LoginSessionView): Promise { + try { + await saveBotLoginSession(this.db, view, BOT_LOGIN_TTL_SEC); + } catch (err) { + console.error( + "[bot-login] persist failed:", + err instanceof Error ? err.message : err, + ); + } + } + + private async expireLocal(sessionId: string): Promise { + const s = this.sessions.get(sessionId); + if (s && s.view.status !== "confirmed") { + s.stopped = true; + s.view.status = "expired"; + s.view.message = "登录超时，请重新发起"; + s.view.updatedAt = new Date().toISOString(); + await this.persist(s.view); + } + this.dropLocal(sessionId); + } + + private dropLocal(sessionId: string): void { + const s = this.sessions.get(sessionId); + if (s?.timer) clearTimeout(s.timer); + this.sessions.delete(sessionId); + } + + private async pollLoop(sessionId: string): Promise { + const session = this.sessions.get(sessionId); + if (!session) return; + + const deadline = Date.now() + 8 * 60_000; + while (!session.stopped && Date.now() < deadline) { + // Cross-node cancel + try { + const remote = await getBotLoginSession(this.db, sessionId); + if (!remote || remote.status === "cancelled") { + session.stopped = true; + if (remote?.status === "cancelled") { + session.view = { ...remote }; + } + this.dropLocal(sessionId); + return; + } + } catch { + /* continue poll on redis blip */ + } + + try { + const status = await session.client.getQrcodeStatus(session.qrcode); + await this.applyStatus(session, status); + await this.persist(session.view); + if ( + session.view.status === "confirmed" || + session.view.status === "expired" || + session.view.status === "error" || + session.view.status === "cancelled" + ) { + // Keep confirmed/error view in Redis for clients; drop local poll resources + this.dropLocal(sessionId); + return; + } + } catch (err) { + if ( + err instanceof ILinkError && + (err.body as { aborted?: boolean })?.aborted + ) { + session.view.status = "wait_scan"; + session.view.updatedAt = new Date().toISOString(); + await this.persist(session.view); + continue; + } + const msg = err instanceof Error ? err.message : String(err); + if (/timed out|aborted|fetch failed|ECONNRESET|network/i.test(msg)) { + session.view.message = "等待扫码中…（网络重试）"; + session.view.updatedAt = new Date().toISOString(); + await this.persist(session.view); + await sleep(800); + continue; + } + session.view.status = "error"; + session.view.message = msg; + session.view.updatedAt = new Date().toISOString(); + await this.persist(session.view); + this.dropLocal(sessionId); + return; + } + await sleep(400); + } + + if (session.view.status !== "confirmed") { + session.view.status = "expired"; + session.view.message = "登录超时，请重新发起"; + session.view.updatedAt = new Date().toISOString(); + await this.persist(session.view); + } + this.dropLocal(sessionId); + } + + private async applyStatus( + session: InternalSession, + status: QrcodeStatusResponse, + ): Promise { + const st = (status.status ?? "").toLowerCase(); + session.view.updatedAt = new Date().toISOString(); + + if ( + st === "confirmed" || + st === "confirmed_login" || + st === "success" || + Boolean(status.bot_token) + ) { + if (!status.bot_token) { + session.view.status = "error"; + session.view.message = "扫码成功但未返回 bot_token"; + return; + } + try { + if (session.view.mode === "rebind" && session.view.rebindBotId) { + await this.finishRebind(session, status); + } else { + await this.finishCreate(session, status); + } + } catch (err) { + session.view.status = "error"; + session.view.message = + err instanceof Error ? err.message : "保存账号失败"; + } + return; + } + + if (st === "expired" || st === "cancel" || st === "cancelled") { + session.view.status = "expired"; + session.view.message = `二维码已${st === "expired" ? "过期" : "取消"}，请重新发起`; + session.stopped = true; + return; + } + + if (st.includes("scan") && !st.includes("wait")) { + session.view.status = "scanned"; + session.view.message = "已扫码，请在手机上确认登录"; + } else { + session.view.status = "wait_scan"; + session.view.message = + session.view.mode === "rebind" + ? "等待微信扫码以重新绑定…" + : "等待微信扫码…"; + } + } + + private async finishCreate( + session: InternalSession, + status: QrcodeStatusResponse, + ): Promise { + const botId = `bot_${Date.now().toString(36)}`; + const displayName = + session.view.displayName || + status.account_id || + status.ilink_bot_id || + botId; + await upsertBotAccount(this.db, { + id: botId, + ownerUserId: session.view.ownerUserId, + displayName, + accountRef: status.account_id ?? status.ilink_bot_id, + baseUrl: status.baseurl, + botToken: status.bot_token!, + }); + await writeAudit(this.db, "bot_login", session.view.ownerUserId, { + botId, + accountRef: status.account_id, + displayName, + mode: "create", + }); + this.worker.ensureLoop(botId); + session.view.status = "confirmed"; + session.view.botId = botId; + session.view.message = `登录成功：${displayName}`; + session.stopped = true; + } + + private async finishRebind( + session: InternalSession, + status: QrcodeStatusResponse, + ): Promise { + const botId = session.view.rebindBotId!; + const existing = await getBotAccount(this.db, botId); + if (!existing) { + throw new Error("bot not found during rebind"); + } + // Keep display name & ownership; refresh token + optional account refs + const displayName = existing.display_name || session.view.displayName; + await upsertBotAccount(this.db, { + id: botId, + ownerUserId: existing.owner_user_id || session.view.ownerUserId, + displayName, + accountRef: + status.account_id ?? status.ilink_bot_id ?? existing.account_ref ?? undefined, + baseUrl: status.baseurl ?? existing.base_url ?? undefined, + botToken: status.bot_token!, + }); + await writeAudit(this.db, "bot_rebind", session.view.ownerUserId, { + botId, + accountRef: status.account_id, + displayName, + }); + this.worker.restartBot(botId); + session.view.status = "confirmed"; + session.view.botId = botId; + session.view.message = `重新绑定成功：${displayName}（数据已保留）`; + session.stopped = true; + } +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} diff --git a/apps/api/src/broadcast-runner.ts b/apps/api/src/broadcast-runner.ts new file mode 100644 index 0000000..bda2cea --- /dev/null +++ b/apps/api/src/broadcast-runner.ts @@ -0,0 +1,355 @@ +import { + type BroadcastJob, + type Db, + cancelBroadcastJob, + findNextPendingBroadcast, + getBroadcastJob, + pushBroadcastFailure, + releaseBroadcastLock, + renewBroadcastLock, + saveBroadcastJob, + setBroadcastActive, + tryAcquireBroadcastLock, + writeAudit, +} from "@wechat-ai/db"; + +export type AdminSendResult = { + ok: boolean; + reason?: + | "empty" + | "no_context_token" + | "no_credentials" + | "ilink_error" + | "cancelled"; + error?: string; +}; + +export interface BroadcastRunnerOptions { + db: Db; + workerId: string; + /** Interval between send attempts (ms) */ + intervalMs: number; + /** Tick interval to look for new jobs (ms) */ + pollIntervalMs?: number; + lockTtlSec?: number; + adminSendText: ( + botId: string, + peerId: string, + text: string, + ) => Promise; + /** Serialize with inbound replies for the same peer when possible */ + runOnPeerChain?: ( + botId: string, + peerId: string, + fn: () => Promise, + ) => Promise; + log?: (msg: string, extra?: unknown) => void; +} + +/** + * Serial async processor for admin broadcast jobs stored in Redis. + * Only one job runs at a time fleet-wide (lock + active pointer). + */ +export class BroadcastRunner { + private stopped = true; + private timer: ReturnType | null = null; + private running = false; + private wakeRequested = false; + + constructor(private opts: BroadcastRunnerOptions) {} + + /** + * Apply admin-editable settings in place (runtime settings reload). + * Values are re-read on every tick, so a longer pollIntervalMs only takes + * hold after the currently armed timer fires. + */ + applyRuntimeOptions(patch: { + intervalMs?: number; + pollIntervalMs?: number; + lockTtlSec?: number; + }): void { + if (patch.intervalMs !== undefined) { + this.opts.intervalMs = Math.max(50, patch.intervalMs); + } + if (patch.pollIntervalMs !== undefined) { + this.opts.pollIntervalMs = Math.max(250, patch.pollIntervalMs); + } + if (patch.lockTtlSec !== undefined) { + this.opts.lockTtlSec = Math.max(10, patch.lockTtlSec); + } + } + + start(): void { + this.stopped = false; + this.opts.log?.( + `[broadcast] runner start interval=${this.opts.intervalMs}ms`, + ); + this.scheduleNext(1_500); + } + + stop(): void { + this.stopped = true; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + /** Call after creating a job so we don't wait for the next poll. */ + wake(): void { + this.wakeRequested = true; + if (this.stopped || this.running) return; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + this.scheduleNext(50); + } + + private scheduleNext(ms: number): void { + if (this.stopped) return; + this.timer = setTimeout(() => { + void this.tick() + .catch((err) => { + this.opts.log?.( + `[broadcast] tick error: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }) + .finally(() => { + const next = this.wakeRequested + ? 200 + : (this.opts.pollIntervalMs ?? 2_000); + this.wakeRequested = false; + this.scheduleNext(next); + }); + }, ms); + } + + /** Exposed for tests */ + async tick(): Promise { + if (this.stopped || this.running) return; + this.running = true; + try { + const job = await findNextPendingBroadcast(this.opts.db); + if (!job) return; + await this.processJob(job); + } finally { + this.running = false; + } + } + + private async processJob(initial: BroadcastJob): Promise { + const lockTtl = this.opts.lockTtlSec ?? 60; + const got = await tryAcquireBroadcastLock( + this.opts.db, + initial.id, + this.opts.workerId, + lockTtl, + ); + if (!got) { + this.opts.log?.( + `[broadcast] job=${initial.id} lock held by another worker`, + ); + return; + } + + let job = (await getBroadcastJob(this.opts.db, initial.id)) ?? initial; + if (job.status === "cancelled" || job.status === "completed") { + await releaseBroadcastLock( + this.opts.db, + job.id, + this.opts.workerId, + ); + return; + } + + try { + if (job.status === "pending") { + job.status = "running"; + job.startedAt = job.startedAt || new Date().toISOString(); + await saveBroadcastJob(this.opts.db, job); + } + await setBroadcastActive(this.opts.db, job.id); + + // Keep a local working copy. Never replace it wholesale with a Redis + // re-read — unflushed stats/cursor would be wiped and the job can + // finish as e.g. 1/N after only the last in-batch increment survives. + job.stats = job.stats ?? { total: 0, sent: 0, skipped: 0, failed: 0 }; + job.failures = job.failures ?? []; + const recipients = job.recipients ?? []; + const text = job.text; + let cursor = Math.max(0, job.cursor ?? 0); + let sinceFlush = 0; + let sinceCancelCheck = 0; + let lastRenew = Date.now(); + // With remote Redis, a GET every message dominates wall time. Check cancel + // every few sends (and always before first / after last). + const cancelCheckEvery = Math.max( + 1, + Math.min(5, Math.floor(1000 / Math.max(50, this.opts.intervalMs))), + ); + // Flush often enough that the admin UI progress is useful, but not every + // single send on a high-latency Redis. + const flushEvery = Math.max(1, Math.min(5, cancelCheckEvery)); + + this.opts.log?.( + `[broadcast] start job=${job.id} total=${recipients.length} cursor=${cursor}`, + ); + + while (cursor < recipients.length) { + if (this.stopped) break; + + // Periodic cancel poll — do NOT assign job = live (stale stats). + if (sinceCancelCheck === 0) { + const live = await getBroadcastJob(this.opts.db, job.id); + if (!live || live.status === "cancelled") { + job.status = "cancelled"; + job.finishedAt = new Date().toISOString(); + job.cursor = cursor; + await saveBroadcastJob(this.opts.db, job); + break; + } + } + sinceCancelCheck = + (sinceCancelCheck + 1) % cancelCheckEvery; + + const target = recipients[cursor]!; + const sendOnce = async () => { + const result = await this.opts.adminSendText( + target.botId, + target.peerId, + text, + ); + if (result.ok) { + job.stats.sent += 1; + } else if ( + result.reason === "no_context_token" || + result.reason === "no_credentials" || + result.reason === "empty" + ) { + job.stats.skipped += 1; + } else { + job.stats.failed += 1; + pushBroadcastFailure(job, { + botId: target.botId, + peerId: target.peerId, + error: result.error || result.reason || "send_failed", + }); + } + }; + + try { + if (this.opts.runOnPeerChain) { + await this.opts.runOnPeerChain( + target.botId, + target.peerId, + sendOnce, + ); + } else { + await sendOnce(); + } + } catch (err) { + job.stats.failed += 1; + pushBroadcastFailure(job, { + botId: target.botId, + peerId: target.peerId, + error: err instanceof Error ? err.message : String(err), + }); + } + + cursor += 1; + job.cursor = cursor; + sinceFlush += 1; + + const now = Date.now(); + if (now - lastRenew > 15_000) { + await renewBroadcastLock( + this.opts.db, + job.id, + this.opts.workerId, + lockTtl, + ); + lastRenew = now; + } + + if (sinceFlush >= flushEvery || cursor >= recipients.length) { + await saveBroadcastJob(this.opts.db, job); + sinceFlush = 0; + } + + if (cursor < recipients.length && this.opts.intervalMs > 0) { + await sleep(this.opts.intervalMs); + } + } + + // Finalize if not cancelled mid-way + const final = await getBroadcastJob(this.opts.db, job.id); + if (final && final.status === "running") { + // Prefer in-memory progress; Redis copy may lag between flushes. + final.cursor = cursor; + final.stats = job.stats; + final.failures = job.failures; + if (cursor >= recipients.length) { + final.status = "completed"; + final.finishedAt = new Date().toISOString(); + } else if (this.stopped) { + // leave as running for another worker / restart + final.status = "running"; + } + await saveBroadcastJob(this.opts.db, final); + job = final; + } else if ( + final && + final.status === "cancelled" && + job.status !== "cancelled" + ) { + // Cancel won the race after the loop; still persist latest counts. + final.cursor = cursor; + final.stats = job.stats; + final.failures = job.failures; + await saveBroadcastJob(this.opts.db, final); + job = final; + } + + if (job.status === "completed" || job.status === "cancelled") { + await writeAudit(this.opts.db, "admin_broadcast_finished", "system", { + jobId: job.id, + status: job.status, + stats: job.stats, + }); + await setBroadcastActive(this.opts.db, null); + this.opts.log?.( + `[broadcast] done job=${job.id} status=${job.status} ` + + `sent=${job.stats.sent} skipped=${job.stats.skipped} failed=${job.stats.failed}`, + ); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + job.status = "failed"; + job.error = message; + job.finishedAt = new Date().toISOString(); + await saveBroadcastJob(this.opts.db, job); + await setBroadcastActive(this.opts.db, null); + await writeAudit(this.opts.db, "admin_broadcast_failed", "system", { + jobId: job.id, + error: message, + }); + this.opts.log?.(`[broadcast] job=${job.id} failed: ${message}`); + } finally { + await releaseBroadcastLock( + this.opts.db, + initial.id, + this.opts.workerId, + ); + } + } +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +// re-export for routes that may cancel via runner-less path +export { cancelBroadcastJob }; diff --git a/apps/api/src/cache-headers.test.ts b/apps/api/src/cache-headers.test.ts new file mode 100644 index 0000000..fe826fa --- /dev/null +++ b/apps/api/src/cache-headers.test.ts @@ -0,0 +1,27 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + etagFromHash, + ifNoneMatchHits, + CC_PRIVATE_NO_STORE, + CC_CDN_STICKER, +} from "./cache-headers.js"; + +test("etagFromHash quotes hash", () => { + assert.equal(etagFromHash("abc123"), '"abc123"'); + assert.equal(etagFromHash("abc123", true), 'W/"abc123"'); +}); + +test("ifNoneMatchHits matches strong and weak", () => { + const etag = etagFromHash("deadbeef"); + assert.equal(ifNoneMatchHits(etag, etag), true); + assert.equal(ifNoneMatchHits(`W/${etag}`, etag), true); + assert.equal(ifNoneMatchHits('"other"', etag), false); + assert.equal(ifNoneMatchHits(undefined, etag), false); + assert.equal(ifNoneMatchHits("*", etag), true); +}); + +test("cache constants present", () => { + assert.match(CC_PRIVATE_NO_STORE, /no-store/); + assert.match(CC_CDN_STICKER, /immutable/); +}); diff --git a/apps/api/src/cache-headers.ts b/apps/api/src/cache-headers.ts new file mode 100644 index 0000000..f52afca --- /dev/null +++ b/apps/api/src/cache-headers.ts @@ -0,0 +1,95 @@ +/** + * Shared Cache-Control helpers for Cloudflare-friendly origin responses. + * + * Prefer Cloudflare-CDN-Cache-Control for longer edge TTL than browsers. + * Authenticated JSON stays private, no-store. + */ + +import type { FastifyReply } from "fastify"; + +/** Landing / docs HTML (browser short, edge longer). */ +export const CC_HTML_MARKETING = + "public, max-age=300" as const; +export const CDN_HTML_MARKETING = + "public, max-age=3600, stale-while-revalidate=86400" as const; + +/** App / admin SPA shells. */ +export const CC_HTML_APP = "public, max-age=60" as const; +export const CDN_HTML_APP = + "public, max-age=3600, stale-while-revalidate=86400" as const; + +/** OG share image. */ +export const CC_OG = "public, max-age=86400, immutable" as const; +export const CDN_OG = "public, max-age=604800" as const; + +/** Public approved sticker CDN. */ +export const CC_CDN_STICKER = + "public, max-age=31536000, immutable" as const; +export const CDN_CDN_STICKER = "public, max-age=31536000" as const; + +/** + * Private sticker bytes (own / pending / admin review). Content-addressed, so + * revalidate cheaply with an ETag instead of re-sending the blob every time. + */ +export const CC_PRIVATE_STICKER = + "private, max-age=300, must-revalidate" as const; + +/** + * Login QR image. Locally rendered from a link that embeds a login ticket, so + * it must never be shared or stored by an intermediary. + */ +export const CC_PRIVATE_QR = "private, no-store" as const; + +/** Auth config (static-ish). */ +export const CC_AUTH_CONFIG = "public, max-age=60" as const; +export const CDN_AUTH_CONFIG = "public, max-age=300" as const; + +/** Default for authenticated / dynamic APIs. */ +export const CC_PRIVATE_NO_STORE = "private, no-store" as const; + +/** Health checks — never cache. */ +export const CC_NO_STORE = "no-store" as const; + +export function setPublicCache( + reply: FastifyReply, + browser: string, + edge?: string, + extra?: { etag?: string; cacheTag?: string }, +): void { + reply.header("Cache-Control", browser); + if (edge) { + reply.header("Cloudflare-CDN-Cache-Control", edge); + } + if (extra?.etag) { + reply.header("ETag", extra.etag); + } + if (extra?.cacheTag) { + reply.header("Cache-Tag", extra.cacheTag); + } +} + +export function setPrivateNoStore(reply: FastifyReply): void { + reply.header("Cache-Control", CC_PRIVATE_NO_STORE); +} + +/** Quoted weak or strong ETag from a hex hash. */ +export function etagFromHash(hash: string, weak = false): string { + const h = hash.replace(/"/g, ""); + return weak ? `W/"${h}"` : `"${h}"`; +} + +export function ifNoneMatchHits( + ifNoneMatch: string | string[] | undefined, + etag: string, +): boolean { + if (!ifNoneMatch) return false; + const raw = Array.isArray(ifNoneMatch) ? ifNoneMatch.join(",") : ifNoneMatch; + const want = etag.replace(/^W\//, "").replace(/"/g, ""); + for (const part of raw.split(",")) { + const t = part.trim(); + if (t === "*") return true; + const got = t.replace(/^W\//, "").replace(/"/g, ""); + if (got === want) return true; + } + return false; +} diff --git a/apps/api/src/cli-doctor.ts b/apps/api/src/cli-doctor.ts new file mode 100644 index 0000000..ce7d290 --- /dev/null +++ b/apps/api/src/cli-doctor.ts @@ -0,0 +1,138 @@ +import { + doctorSnapshot, + getBotCredentials, + listBotAccounts, + openDatabase, + seedPersonas, +} from "@wechat-ai/db"; +import { probeToolsHealth } from "@wechat-ai/llm"; +import { loadConfig } from "./config.js"; +import { isPlaceholderRedisUrl, loadLinuxDoConfig } from "./oauth-linuxdo.js"; + +function ok(msg: string): void { + console.log(` ✓ ${msg}`); +} +function warn(msg: string): void { + console.log(` ! ${msg}`); +} +function fail(msg: string): void { + console.log(` ✗ ${msg}`); +} + +async function main(): Promise { + const cfg = loadConfig(); + let exitCode = 0; + + console.log("WeChat-AI doctor\n"); + console.log("Environment"); + // Mask password in redis URL for logs + const maskedRedis = cfg.redisUrl.replace( + /:\/\/([^:]+):([^@]+)@/, + "://$1:***@", + ); + ok(`REDIS_URL=${maskedRedis}`); + if (isPlaceholderRedisUrl(cfg.redisUrl)) { + fail( + "REDIS_URL 仍是占位符。请到 Upstash Console → Connect 复制 rediss:// 连接串写入 .env", + ); + process.exit(1); + } + if (/upstash\.io/i.test(cfg.redisUrl) && cfg.redisUrl.startsWith("redis://")) { + warn("Upstash 建议使用 rediss://（TLS），当前是 redis://"); + } + if (!cfg.llmApiKey) { + fail("LLM_API_KEY missing (platform / admin LLM)"); + exitCode = 1; + } else { + ok( + `platform LLM model=${cfg.llmModel} base=${cfg.llmBaseUrl.replace(/\/\/([^:]+):([^@]+)@/, "//$1:***@")}`, + ); + } + if (cfg.toolsBaseUrl) { + ok(`TOOLS_BASE_URL=${cfg.toolsBaseUrl}`); + if (!cfg.toolsApiKey) { + warn("TOOLS_API_KEY empty — tools gateway may reject requests"); + } + const probe = await probeToolsHealth(cfg.toolsBaseUrl); + if (probe.ok) { + ok(`tools gateway health: ${probe.detail.slice(0, 80)}`); + } else { + fail(`tools gateway unreachable: ${probe.detail}`); + exitCode = 1; + } + } else { + warn( + "TOOLS_BASE_URL 未配置 — 用户自定义 LLM 与联网搜索不可用（平台 LLM 仍可用）", + ); + } + if (cfg.webSearchEnabled && !cfg.toolsBaseUrl) { + fail("WEB_SEARCH_ENABLED=true 但未配置 TOOLS_BASE_URL"); + exitCode = 1; + } + const oauth = loadLinuxDoConfig(); + if (!oauth) { + fail("LINUX DO OAuth 未配置 LINUXDO_CLIENT_ID/SECRET/REDIRECT_URI"); + exitCode = 1; + } else { + ok(`OAuth redirect=${oauth.redirectUri}`); + } + if (cfg.adminIds.size === 0) { + warn("LINUXDO_ADMIN_IDS 为空 — 无人自动成为管理员"); + } else { + ok(`admin ids: ${[...cfg.adminIds].join(",")}`); + } + + console.log("\nRedis"); + const db = openDatabase(cfg.redisUrl); + try { + await db.ping(); + ok("PONG"); + } catch (err) { + fail(`连接失败: ${(err as Error).message}`); + process.exit(1); + } + await seedPersonas(db); + const snap = await doctorSnapshot(db); + ok( + `users=${snap.users} bots=${snap.bots} personas=${snap.personas} default=${snap.defaultPersona ?? "none"}`, + ); + ok( + `peers=${snap.peers} (approved=${snap.approvedPeers} pending=${snap.unapprovedPeers})`, + ); + if (snap.deepStats) { + ok( + `assignments=${snap.assignments} messages=${snap.messages} memories=${snap.memories}`, + ); + } else { + warn( + `assignments/messages/memories 未统计（peers 超过 DOCTOR_DEEP_STATS_MAX_PEERS）`, + ); + } + if (!snap.defaultPersona) { + fail("无默认人设"); + exitCode = 1; + } + if (snap.activeBots === 0) warn("尚无机器人 — 用户登录后扫码添加"); + + console.log("\nBot credentials (Redis)"); + for (const bot of await listBotAccounts(db)) { + const creds = await getBotCredentials(db, bot.id); + if (!creds?.botToken) { + fail( + `bot ${bot.id}: missing Redis token (wa:bot:${bot.id}:creds) — re-scan login`, + ); + exitCode = 1; + } else { + ok(`bot ${bot.id} owner=${bot.owner_user_id} token=redis`); + } + } + + console.log(exitCode === 0 ? "\nDoctor: PASS" : "\nDoctor: ISSUES FOUND"); + await db.close(); + process.exit(exitCode); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/api/src/cli-login.ts b/apps/api/src/cli-login.ts new file mode 100644 index 0000000..5353363 --- /dev/null +++ b/apps/api/src/cli-login.ts @@ -0,0 +1,82 @@ +import { loginWithQrcode, ILinkClient } from "@wechat-ai/ilink"; +import { + openDatabase, + upsertBotAccount, + writeAudit, +} from "@wechat-ai/db"; +import { loadConfig } from "./config.js"; + +function parseArgs(argv: string[]): { name?: string; owner?: string } { + const out: { name?: string; owner?: string } = {}; + const args = argv.filter((a) => a !== "--"); + for (let i = 0; i < args.length; i++) { + if (args[i] === "--name" && args[i + 1]) { + out.name = args[++i]; + } else if (args[i] === "--owner" && args[i + 1]) { + out.owner = args[++i]; + } + } + return out; +} + +async function main(): Promise { + const cfg = loadConfig(); + const args = parseArgs(process.argv.slice(2)); + const db = openDatabase(cfg.redisUrl); + + console.log("Requesting iLink QR code…"); + console.log("推荐：浏览器打开 /app → 机器人 → 扫码添加（token 写入 Redis）。"); + console.log("Open WeChat → scan the QR (ClawBot / 插件扫码).\n"); + if (args.name) console.log(`Display name: ${args.name}\n`); + + const client = new ILinkClient({ timeoutMs: 30_000 }); + const result = await loginWithQrcode({ + client, + timeoutMs: 8 * 60_000, + onQrcode: (info) => { + console.log("qrcode id:", info.qrcode); + const openUrl = info.qrcodeUrl || info.qrcodeImgContent; + if (openUrl?.startsWith("http")) { + console.log("\n>>> 请在手机微信中打开此链接完成扫码/授权：\n"); + console.log(openUrl); + console.log("\n"); + } + console.log("等待扫码（最长约 8 分钟，请勿关闭窗口）…\n"); + }, + onStatus: (st) => { + process.stdout.write(`\rstatus: ${st.padEnd(16)}`); + }, + }); + + console.log("\n\nLogin confirmed."); + + const botId = `bot_${Date.now().toString(36)}`; + const displayName = + args.name ?? result.accountId ?? `bot-${botId.slice(-6)}`; + const ownerUserId = args.owner ?? "cli"; + + await upsertBotAccount(db, { + id: botId, + ownerUserId, + displayName, + accountRef: result.accountId, + baseUrl: result.baseUrl, + botToken: result.botToken, + }); + await writeAudit(db, "bot_login", "cli", { + botId, + accountRef: result.accountId, + displayName, + }); + + console.log("Saved bot:", botId, `(${displayName})`); + console.log("Token: Redis key wa:bot:" + botId + ":creds"); + console.log("多 Bot：再次运行 pnpm ilink:login -- --name 第二个号"); + console.log("Start: pnpm dev → http://127.0.0.1:8787/app"); + await db.close(); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts new file mode 100644 index 0000000..0e4726c --- /dev/null +++ b/apps/api/src/config.ts @@ -0,0 +1,498 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { config as loadDotenv } from "dotenv"; +import { parseAdminIds } from "./oauth-linuxdo.js"; + +/** + * Levels pino accepts. Validated rather than passed through: pino throws at + * construction on an unknown level, so a typo in LOG_LEVEL would take the whole + * process down at boot. + */ +export const LOG_LEVELS = [ + "silent", + "fatal", + "error", + "warn", + "info", + "debug", + "trace", +] as const; + +export type LogLevel = (typeof LOG_LEVELS)[number]; + +export const VISION_MODES = ["caption", "direct"] as const; +export type VisionMode = (typeof VISION_MODES)[number]; + +export function resolveVisionMode(raw: string | undefined): VisionMode { + const v = (raw ?? "").trim().toLowerCase(); + // Default to caption: it is the only mode that works when the roleplay model + // is text-only, which is the common case. + return v === "direct" ? "direct" : "caption"; +} + +export function resolveLogLevel(raw: string | undefined): LogLevel { + const v = (raw ?? "").trim().toLowerCase(); + return (LOG_LEVELS as readonly string[]).includes(v) + ? (v as LogLevel) + : "info"; +} + +export function resolveRepoRoot(start = process.cwd()): string { + let dir = path.resolve(start); + for (let i = 0; i < 10; i++) { + if (fs.existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + const fromFile = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../..", + ); + if (fs.existsSync(path.join(fromFile, "pnpm-workspace.yaml"))) { + return fromFile; + } + return path.resolve(start); +} + +const repoRootEarly = resolveRepoRoot(); +loadDotenv({ path: path.join(repoRootEarly, ".env") }); +loadDotenv({ path: path.resolve(process.cwd(), ".env") }); + +export interface AppConfig { + host: string; + port: number; + /** legacy bearer for scripts; optional when OAuth enabled */ + token: string; + redisUrl: string; + /** Admin platform LLM (main site connects directly) */ + llmBaseUrl: string; + llmApiKey: string; + llmModel: string; + /** + * HF / tools gateway (user custom LLM egress + web search). + * Main site never dials user custom base_url; only this host. + */ + toolsBaseUrl: string; + toolsApiKey: string; + toolsTimeoutMs: number; + /** Global switch: allow personas to use web_search via tools gateway */ + webSearchEnabled: boolean; + webSearchMaxResults: number; + /** Encrypt user-stored custom LLM API keys at rest */ + llmProviderSecret: string; + /** + * Extra hosts for chatflow http nodes (comma-separated). + * Tools gateway host is always allowed; private IPs still blocked unless tools itself. + */ + chatflowHttpAllowlist: string[]; + chatflowMaxSteps: number; + chatflowMaxNodes: number; + defaultPersonaSlug: string; + allowUnapproved: boolean; + shortHistoryLimit: number; + memoryExtractEveryN: number; + workerEnabled: boolean; + /** Max concurrent iLink long-poll loops in this process */ + maxBotsPerWorker: number; + leaseTtlSec: number; + leaseRenewSec: number; + /** + * Multi-node: overloaded workers shed leases so idle nodes can claim. + * Default on. Set REBALANCE_ENABLED=false to keep sticky leases. + */ + rebalanceEnabled: boolean; + /** Min seconds between rebalance shed attempts on this process */ + rebalanceIntervalSec: number; + /** Allow this many bots above fair share before shedding */ + rebalanceSlack: number; + /** Max leases to release per rebalance tick */ + rebalanceMaxPerTick: number; + /** + * Seconds an admin load-weight override survives after its node stops + * heartbeating, before the fleet deletes it automatically. Must outlast a + * restart / OTA apply, or every deploy would reset the tuning. + */ + workerWeightTtlSec: number; + /** Concurrent LLM/reply jobs */ + replyConcurrency: number; + inboxMaxLen: number; + logLevel: LogLevel; + /** Requests slower than this are logged at warn even when they succeed */ + logSlowRequestMs: number; + peerRatePerMinute: number; + repoRoot: string; + publicBaseUrl: string; + sessionCookieName: string; + adminIds: Set; + cookieSecure: boolean; + /** + * Allowed browser Origins for credentialed CORS on /api/v1. + * Defaults to PUBLIC_BASE_URL origin. Empty list = same-origin only (no ACAO). + */ + corsOrigins: Set; + /** Split AI reply into multiple WeChat bubbles */ + splitReply: boolean; + maxReplyChunks: number; + maxChunkChars: number; + /** Ask model to return {"messages":[...]} JSON bubbles */ + multiBubbleJson: boolean; + /** + * Second-pass AI filter: reformat primary reply into multi-bubble JSON before send. + * Extra LLM cost/latency. Off by default — primary model emits JSON via MULTI_BUBBLE_JSON. + * Enable with REPLY_FILTER_ENABLED=true only if format quality needs a dedicated pass. + */ + replyFilterEnabled: boolean; + /** Human-like delay between WeChat bubbles */ + replyDelayMsPerChar: number; + replyDelayMinMs: number; + replyDelayMaxMs: number; + replyDelayFirstMinMs: number; + replyDelayFirstMaxMs: number; + replyDelayThinkExtraMs: number; + /** Max uploaded sticker size in bytes (blob stored in Redis) */ + stickerMaxBytes: number; + /** Body cap for the few upload routes (global default is 1MB) */ + uploadBodyLimit: number; + /** Inject sticker catalog + allow sendImage */ + stickerSendEnabled: boolean; + maxStickersPerReply: number; + /** + * Master switch for inbound image understanding. Default OFF. + * When off, images are never downloaded and get a canned "can't see it" line. + */ + visionEnabled: boolean; + /** + * How an image reaches the conversation. + * + * `caption` (default): a dedicated vision endpoint describes the image and + * only that TEXT enters the roleplay turn — so the persona's own model needs + * no multimodal support at all. This is the mode that works with a text-only + * platform model such as deepseek. + * + * `direct`: the image is handed to the roleplay model as content parts. + * Requires that model to be vision-capable; it errors outright otherwise. + */ + visionMode: VisionMode; + /** + * Vision endpoint. Admin-level credentials, dialed directly like the platform + * LLM — a captioner is not user-supplied, so it does not need the tools + * gateway. Empty base/key fall back to the platform LLM's. + */ + visionBaseUrl: string; + visionApiKey: string; + /** Vision model id. Required for vision to do anything at all. */ + visionModel: string; + /** Cap on caption length — a description, not an essay. */ + visionCaptionMaxTokens: number; + /** + * Act on the speech-to-text WeChat/iLink already did for a voice message. + * + * Default ON, and unrelated to VISION_ENABLED: the transcript arrives inside + * the inbound message, so using it costs nothing extra and needs no model. + * Turn off to have voice notes answered with "didn't catch that" instead. + */ + voiceTranscriptEnabled: boolean; + /** Max images forwarded from a single message (each one costs real tokens) */ + visionMaxImages: number; + /** Hard cap per downloaded attachment, before base64 inflation */ + inboundMediaMaxBytes: number; + /** Global hard switch for idle proactive outreach */ + proactiveEnabled: boolean; + proactiveIdleHours: number; + proactiveMinIntervalHours: number; + proactiveMaxPerDay: number; + /** e.g. "0-8"; empty string disables quiet hours */ + proactiveQuietHours: string; + proactiveScanIntervalSec: number; + proactiveMaxPerScan: number; + proactiveLockTtlSec: number; + proactiveAttemptCooldownHours: number; + /** Memory: top-K when over fullInjectMax */ + memoryTopK: number; + /** Memory: inject all when count ≤ this */ + memoryFullInjectMax: number; + /** Memory: hard cap stored facts per peer+persona */ + memoryMaxItems: number; + /** LLM tool: get_current_time */ + timeToolEnabled: boolean; + timeToolTimeZone: string; + /** WeChat user-to-user relay via @LINUX DO username */ + p2pEnabled: boolean; + p2pBindCodeTtlSec: number; + p2pRequestTtlSec: number; + p2pSessionIdleSec: number; + p2pRelayMaxChars: number; + p2pMaxRequestsPerDay: number; + /** Admin broadcast: delay between messages (ms) */ + broadcastIntervalMs: number; + /** Admin broadcast: max text length */ + broadcastMaxText: number; + /** Admin broadcast: retained job history count */ + broadcastHistory: number; + /** Web try-chat (persona preview without WeChat) */ + tryChatEnabled: boolean; + tryChatMaxUserMsgsPerDay: number; + tryChatMaxUserMsgsPerSession: number; + tryChatSessionTtlSec: number; + tryChatMaxHistory: number; + /** Server-side persona fork */ + personaForkEnabled: boolean; + /** Local username+password auth */ + localAuthEnabled: boolean; + passwordMinLength: number; + /** Local register requires invite code */ + inviteRequiredForLocal: boolean; + inviteCodeTtlSec: number; + inviteCodeLength: number; + inviteMaxPendingPerUser: number; + /** Sliding window: max invites generated per user per window */ + inviteQuotaWindowHours: number; + inviteQuotaMax: number; + firstUserIsAdmin: boolean; + /** + * Optional ops labels for multi-node fleet display (not public URLs). + * WORKER_ID itself is read by BotWorkerManager from process.env. + */ + nodeLabel: string; + nodeRegion: string; + /** App version string for worker heartbeat */ + appVersion: string; + /** Consume OTA update jobs from Redis (default true) */ + otaEnabled: boolean; + /** Allow pnpm install during OTA when lock/package.json changes */ + otaAllowInstall: boolean; + /** Staging dir for OTA writes (absolute or relative to repo root) */ + otaStagingDir: string; + /** Admin live activity stream (SSE) */ + dataStreamEnabled: boolean; + /** Redis command sample rate for stream (0–1) */ + dataStreamRedisSample: number; + /** Process-local max stream events per second */ + dataStreamMaxEps: number; +} + +/** Read OTA-written version file (monorepo root `.wa-version`). */ +export function readWaVersionFile(repoRoot: string): string | null { + try { + const p = path.join(repoRoot, ".wa-version"); + if (!fs.existsSync(p)) return null; + const v = fs.readFileSync(p, "utf8").trim().split(/\r?\n/)[0]?.trim() ?? ""; + return v || null; + } catch { + return null; + } +} + +export function readPackageJsonVersion(repoRoot: string): string | null { + try { + const p = path.join(repoRoot, "package.json"); + if (!fs.existsSync(p)) return null; + const j = JSON.parse(fs.readFileSync(p, "utf8")) as { version?: string }; + const v = (j.version || "").trim(); + return v || null; + } catch { + return null; + } +} + +/** + * Resolve runtime version for heartbeat / admin display. + * + * Order (OTA-friendly — env cannot be changed by OTA): + * 1. `.wa-version` at monorepo root (written by OTA apply) + * 2. `APP_VERSION` env (optional ops pin when no OTA stamp) + * 3. root `package.json` version (not workspace package / npm_package_version; + * pnpm --filter @wechat-ai/api would otherwise report apps/api's 0.1.0) + * 4. fallback + */ +export function resolveAppVersion( + env: NodeJS.ProcessEnv, + repoRoot: string, + fallback = "0.2.0", +): string { + const fromFile = readWaVersionFile(repoRoot); + if (fromFile) return fromFile; + const fromEnv = (env.APP_VERSION ?? "").trim(); + if (fromEnv) return fromEnv; + // Prefer monorepo root package.json only — ignore npm_package_version + // (set by pnpm to the filtered workspace package, e.g. apps/api @ 0.1.0). + const fromPkg = readPackageJsonVersion(repoRoot); + if (fromPkg) return fromPkg; + return fallback; +} + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): AppConfig { + const repoRoot = resolveRepoRoot(); + const port = Number(env.WECHAT_AI_PORT ?? "8787"); + const host = env.WECHAT_AI_HOST ?? "127.0.0.1"; + const publicBaseUrl = + env.PUBLIC_BASE_URL ?? + `http://${host === "0.0.0.0" ? "127.0.0.1" : host}:${port}`; + const corsOrigins = new Set(); + try { + corsOrigins.add(new URL(publicBaseUrl).origin); + } catch { + /* ignore bad PUBLIC_BASE_URL */ + } + if (env.CORS_ORIGINS) { + for (const part of env.CORS_ORIGINS.split(",")) { + const o = part.trim(); + if (!o) continue; + try { + corsOrigins.add(new URL(o).origin); + } catch { + // allow bare origin like https://a.com + if (/^https?:\/\//i.test(o)) corsOrigins.add(o.replace(/\/$/, "")); + } + } + } + return { + host, + port, + token: env.WECHAT_AI_TOKEN ?? "dev-insecure-token", + redisUrl: env.REDIS_URL ?? "redis://127.0.0.1:6379", + stickerMaxBytes: Number(env.STICKER_MAX_BYTES ?? String(2 * 1024 * 1024)), + uploadBodyLimit: Math.max( + 12 * 1024 * 1024, + Number(env.STICKER_MAX_BYTES ?? String(2 * 1024 * 1024)) * 2, + ), + stickerSendEnabled: env.STICKER_SEND_ENABLED !== "false", + maxStickersPerReply: Number(env.MAX_STICKERS_PER_REPLY ?? "2"), + visionEnabled: env.VISION_ENABLED === "true", + visionMode: resolveVisionMode(env.VISION_MODE), + visionBaseUrl: (env.VISION_BASE_URL ?? "").trim(), + visionApiKey: (env.VISION_API_KEY ?? "").trim(), + visionModel: (env.VISION_MODEL ?? "").trim(), + visionCaptionMaxTokens: Math.max( + 32, + Number(env.VISION_CAPTION_MAX_TOKENS ?? "300") || 300, + ), + voiceTranscriptEnabled: env.VOICE_TRANSCRIPT_ENABLED !== "false", + visionMaxImages: Math.max(1, Number(env.VISION_MAX_IMAGES ?? "2") || 2), + inboundMediaMaxBytes: Math.max( + 64 * 1024, + Number(env.INBOUND_MEDIA_MAX_BYTES ?? String(4 * 1024 * 1024)) || + 4 * 1024 * 1024, + ), + proactiveEnabled: env.PROACTIVE_ENABLED === "true", + proactiveIdleHours: Number(env.PROACTIVE_IDLE_HOURS ?? "12"), + proactiveMinIntervalHours: Number( + env.PROACTIVE_MIN_INTERVAL_HOURS ?? "24", + ), + proactiveMaxPerDay: Number(env.PROACTIVE_MAX_PER_DAY ?? "1"), + proactiveQuietHours: + env.PROACTIVE_QUIET_HOURS === undefined + ? "0-8" + : String(env.PROACTIVE_QUIET_HOURS).trim(), + proactiveScanIntervalSec: Number( + env.PROACTIVE_SCAN_INTERVAL_SEC ?? "300", + ), + proactiveMaxPerScan: Number(env.PROACTIVE_MAX_PER_SCAN ?? "10"), + proactiveLockTtlSec: Number(env.PROACTIVE_LOCK_TTL_SEC ?? "180"), + proactiveAttemptCooldownHours: Number( + env.PROACTIVE_ATTEMPT_COOLDOWN_HOURS ?? "1", + ), + memoryTopK: Number(env.MEMORY_TOP_K ?? "12"), + memoryFullInjectMax: Number(env.MEMORY_FULL_INJECT_MAX ?? "20"), + memoryMaxItems: Number(env.MEMORY_MAX_ITEMS ?? "100"), + timeToolEnabled: env.TIME_TOOL_ENABLED !== "false", + timeToolTimeZone: (env.TIME_TOOL_TIMEZONE ?? "Asia/Shanghai").trim() || + "Asia/Shanghai", + p2pEnabled: env.P2P_ENABLED !== "false", + p2pBindCodeTtlSec: Number(env.P2P_BIND_CODE_TTL_SEC ?? "600"), + p2pRequestTtlSec: Number(env.P2P_REQUEST_TTL_SEC ?? "300"), + p2pSessionIdleSec: Number(env.P2P_SESSION_IDLE_SEC ?? "1800"), + p2pRelayMaxChars: Number(env.P2P_RELAY_MAX_CHARS ?? "500"), + p2pMaxRequestsPerDay: Number(env.P2P_MAX_REQUESTS_PER_DAY ?? "20"), + broadcastIntervalMs: Number(env.BROADCAST_INTERVAL_MS ?? "200"), + broadcastMaxText: Number(env.BROADCAST_MAX_TEXT ?? "2000"), + broadcastHistory: Number(env.BROADCAST_HISTORY ?? "100"), + tryChatEnabled: env.TRY_CHAT_ENABLED !== "false", + tryChatMaxUserMsgsPerDay: Number( + env.TRY_CHAT_MAX_USER_MSGS_PER_DAY ?? "40", + ), + tryChatMaxUserMsgsPerSession: Number( + env.TRY_CHAT_MAX_USER_MSGS_PER_SESSION ?? "20", + ), + tryChatSessionTtlSec: Number(env.TRY_CHAT_SESSION_TTL_SEC ?? "3600"), + tryChatMaxHistory: Number(env.TRY_CHAT_MAX_HISTORY ?? "40"), + personaForkEnabled: env.PERSONA_FORK_ENABLED !== "false", + localAuthEnabled: env.LOCAL_AUTH_ENABLED !== "false", + passwordMinLength: Number(env.PASSWORD_MIN_LENGTH ?? "8"), + inviteRequiredForLocal: env.INVITE_REQUIRED_FOR_LOCAL !== "false", + inviteCodeTtlSec: Number(env.INVITE_CODE_TTL_SEC ?? String(7 * 24 * 3600)), + inviteCodeLength: Number(env.INVITE_CODE_LENGTH ?? "10"), + inviteMaxPendingPerUser: Number(env.INVITE_MAX_PENDING_PER_USER ?? "20"), + inviteQuotaWindowHours: Number(env.INVITE_QUOTA_WINDOW_HOURS ?? "24"), + inviteQuotaMax: Number(env.INVITE_QUOTA_MAX ?? "3"), + firstUserIsAdmin: env.FIRST_USER_IS_ADMIN !== "false", + llmBaseUrl: env.LLM_BASE_URL ?? "https://api.openai.com/v1", + llmApiKey: env.LLM_API_KEY ?? "", + llmModel: env.LLM_MODEL ?? "gpt-4o-mini", + toolsBaseUrl: (env.TOOLS_BASE_URL ?? "").trim().replace(/\/+$/, ""), + toolsApiKey: (env.TOOLS_API_KEY ?? "").trim(), + toolsTimeoutMs: Number(env.TOOLS_TIMEOUT_MS ?? "60000"), + webSearchEnabled: env.WEB_SEARCH_ENABLED === "true", + webSearchMaxResults: Number(env.WEB_SEARCH_MAX_RESULTS ?? "5"), + llmProviderSecret: (env.LLM_PROVIDER_SECRET ?? "").trim(), + chatflowHttpAllowlist: (env.CHATFLOW_HTTP_ALLOWLIST ?? "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + chatflowMaxSteps: Number(env.CHATFLOW_MAX_STEPS ?? "32"), + chatflowMaxNodes: Number(env.CHATFLOW_MAX_NODES ?? "40"), + defaultPersonaSlug: env.DEFAULT_PERSONA_SLUG ?? "catgirl", + allowUnapproved: env.ALLOW_UNAPPROVED_USERS === "true", + shortHistoryLimit: Number(env.SHORT_HISTORY_LIMIT ?? "20"), + memoryExtractEveryN: Number(env.MEMORY_EXTRACT_EVERY_N ?? "8"), + workerEnabled: env.WORKER_ENABLED !== "false", + maxBotsPerWorker: Number(env.MAX_BOTS_PER_WORKER ?? "500"), + leaseTtlSec: Number(env.LEASE_TTL_SEC ?? "45"), + leaseRenewSec: Number(env.LEASE_RENEW_SEC ?? "15"), + rebalanceEnabled: env.REBALANCE_ENABLED !== "false", + rebalanceIntervalSec: Number(env.REBALANCE_INTERVAL_SEC ?? "60"), + rebalanceSlack: Number(env.REBALANCE_SLACK ?? "2"), + rebalanceMaxPerTick: Number(env.REBALANCE_MAX_PER_TICK ?? "50"), + workerWeightTtlSec: Number(env.WORKER_WEIGHT_TTL_SEC ?? "3600"), + replyConcurrency: Number(env.REPLY_CONCURRENCY ?? "16"), + inboxMaxLen: Number(env.INBOX_MAX_LEN ?? "20000"), + logLevel: resolveLogLevel(env.LOG_LEVEL), + logSlowRequestMs: Math.max( + 50, + Number(env.LOG_SLOW_REQUEST_MS ?? "1000") || 1000, + ), + peerRatePerMinute: Number(env.PEER_RATE_PER_MINUTE ?? "20"), + splitReply: env.SPLIT_REPLY !== "false", + maxReplyChunks: Number(env.MAX_REPLY_CHUNKS ?? "5"), + maxChunkChars: Number(env.MAX_CHUNK_CHARS ?? "72"), + multiBubbleJson: env.MULTI_BUBBLE_JSON !== "false", + replyFilterEnabled: env.REPLY_FILTER_ENABLED === "true", + replyDelayMsPerChar: Number(env.REPLY_DELAY_MS_PER_CHAR ?? "90"), + replyDelayMinMs: Number(env.REPLY_DELAY_MIN_MS ?? "1400"), + replyDelayMaxMs: Number(env.REPLY_DELAY_MAX_MS ?? "5500"), + replyDelayFirstMinMs: Number(env.REPLY_DELAY_FIRST_MIN_MS ?? "900"), + replyDelayFirstMaxMs: Number(env.REPLY_DELAY_FIRST_MAX_MS ?? "2200"), + replyDelayThinkExtraMs: Number(env.REPLY_DELAY_THINK_EXTRA_MS ?? "400"), + repoRoot, + publicBaseUrl, + sessionCookieName: env.SESSION_COOKIE_NAME ?? "wa_session", + adminIds: parseAdminIds(env.LINUXDO_ADMIN_IDS), + cookieSecure: env.COOKIE_SECURE === "true", + corsOrigins, + nodeLabel: (env.NODE_LABEL ?? "").trim(), + nodeRegion: (env.NODE_REGION ?? "").trim(), + appVersion: resolveAppVersion(env, repoRoot), + otaEnabled: env.OTA_ENABLED !== "false", + otaAllowInstall: env.OTA_ALLOW_INSTALL !== "false", + otaStagingDir: (env.OTA_STAGING_DIR ?? ".wa-update-staging").trim() || + ".wa-update-staging", + dataStreamEnabled: env.DATA_STREAM_ENABLED !== "false", + dataStreamRedisSample: Math.min( + 1, + Math.max(0, Number(env.DATA_STREAM_REDIS_SAMPLE ?? "0.08")), + ), + dataStreamMaxEps: Math.max(5, Number(env.DATA_STREAM_MAX_EPS ?? "80")), + }; +} diff --git a/apps/api/src/inbound-media.test.ts b/apps/api/src/inbound-media.test.ts new file mode 100644 index 0000000..4555bb5 --- /dev/null +++ b/apps/api/src/inbound-media.test.ts @@ -0,0 +1,153 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import type { InboundMediaRef } from "@wechat-ai/ilink"; +import { planInboundMedia, unreadableMediaReply } from "./inbound-media.js"; + +function ref( + kind: InboundMediaRef["kind"], + extra: Partial = {}, +): InboundMediaRef { + return { + kind, + index: 0, + encryptQueryParam: "eqp", + aesKey: "a".repeat(32), + ...extra, + }; +} + +const visionOn = { visionEnabled: true, maxImages: 2 }; +const visionOff = { visionEnabled: false, maxImages: 2 }; + +describe("planInboundMedia", () => { + it("downloads an image when vision is on", () => { + const plan = planInboundMedia([ref("image")], visionOn); + assert.equal(plan.length, 1); + assert.equal(plan[0]!.download, true); + }); + + it("never downloads when vision is off", () => { + const plan = planInboundMedia([ref("image")], visionOff); + assert.equal(plan.length, 1); + assert.equal( + plan[0]!.download, + false, + "an image must stay notice-only so the persona says it cannot see it", + ); + }); + + it("never downloads non-image media, even with vision on", () => { + // WeChat voice is SILK/AMR and no chat completions endpoint accepts it; + // video/file bytes are equally unusable. Fetching them is pure bandwidth. + for (const kind of ["voice", "video", "file"] as const) { + const plan = planInboundMedia([ref(kind)], visionOn); + assert.equal(plan.length, 1, kind); + assert.equal(plan[0]!.download, false, kind); + } + }); + + it("drops a voice note that iLink already transcribed", () => { + // extractText folded the transcript into the message text, so listing the + // attachment would claim the model cannot hear what it is about to read. + assert.deepEqual( + planInboundMedia([ref("voice", { transcript: "语音转写" })], visionOn), + [], + ); + }); + + it("keeps a transcribed voice note when transcripts are disabled", () => { + // With VOICE_TRANSCRIPT_ENABLED=false the text never contained the + // transcript, so the ref must survive to produce the "didn't catch that" + // reply — dropping it would leave the message with nothing at all. + const plan = planInboundMedia([ref("voice", { transcript: "语音转写" })], { + ...visionOn, + voiceTranscriptEnabled: false, + }); + assert.equal(plan.length, 1); + assert.equal(plan[0]!.download, false); + }); + + it("keeps a voice note with a blank transcript as notice-only", () => { + const plan = planInboundMedia([ref("voice", { transcript: " " })], visionOn); + assert.equal(plan.length, 1); + assert.equal(plan[0]!.download, false); + }); + + it("caps how many images one message may cost", () => { + const plan = planInboundMedia( + [ref("image"), ref("image"), ref("image"), ref("image")], + { visionEnabled: true, maxImages: 2 }, + ); + assert.equal(plan.length, 4, "every attachment is still reported"); + assert.deepEqual( + plan.map((p) => p.download), + [true, true, false, false], + "images past the cap become notice-only rather than disappearing", + ); + }); + + it("counts only images against the cap", () => { + const plan = planInboundMedia( + [ref("video"), ref("image"), ref("file"), ref("image")], + { visionEnabled: true, maxImages: 2 }, + ); + assert.deepEqual( + plan.map((p) => [p.ref.kind, p.download]), + [ + ["video", false], + ["image", true], + ["file", false], + ["image", true], + ], + ); + }); + + it("clamps a nonsensical cap to at least one", () => { + for (const maxImages of [0, -3, 0.4]) { + const plan = planInboundMedia([ref("image"), ref("image")], { + visionEnabled: true, + maxImages, + }); + assert.equal( + plan.filter((p) => p.download).length, + 1, + `maxImages=${maxImages}`, + ); + } + }); + + it("preserves order and the original refs", () => { + const a = ref("image", { index: 1, encryptQueryParam: "a" }); + const b = ref("file", { index: 2, fileName: "x.pdf" }); + const plan = planInboundMedia([a, b], visionOn); + assert.equal(plan[0]!.ref, a); + assert.equal(plan[1]!.ref, b); + }); + + it("handles an empty list", () => { + assert.deepEqual(planInboundMedia([], visionOn), []); + }); +}); + +describe("unreadableMediaReply", () => { + it("names the kind that actually arrived", () => { + assert.match(unreadableMediaReply([ref("image")]), /看不了图片/); + assert.match(unreadableMediaReply([ref("voice")]), /语音/); + assert.match(unreadableMediaReply([ref("video")]), /视频/); + assert.match(unreadableMediaReply([ref("file")]), /文件/); + }); + + it("keeps the original generic line when there is nothing to name", () => { + assert.equal( + unreadableMediaReply([]), + "目前只支持文字消息喵～请发文字聊天。", + ); + }); + + it("leads with the first attachment for a mixed message", () => { + assert.match( + unreadableMediaReply([ref("video"), ref("image")]), + /视频/, + ); + }); +}); diff --git a/apps/api/src/inbound-media.ts b/apps/api/src/inbound-media.ts new file mode 100644 index 0000000..61d901c --- /dev/null +++ b/apps/api/src/inbound-media.ts @@ -0,0 +1,91 @@ +import type { InboundMediaRef } from "@wechat-ai/ilink"; + +/** + * Decides what to do with the attachments on an inbound WeChat message. + * + * Split out of BotWorkerManager so the gating rules — which attachments are + * worth pulling off the CDN, how many images one message may cost, and which + * ones are already covered by the text — are testable without a Redis handle or + * a live iLink client. + */ + +export interface MediaPlanEntry { + ref: InboundMediaRef; + /** + * True when the bytes should be fetched and offered to the model. False means + * notice-only: the persona is told the attachment exists but that it cannot + * perceive it, which is what stops it inventing contents. + */ + download: boolean; +} + +export interface MediaPlanOptions { + /** Global switch; off means no image is ever downloaded */ + visionEnabled: boolean; + /** Max images from one message that may be sent to the model */ + maxImages: number; + /** + * Whether the message text already carries iLink's voice transcript + * (default true, matching VOICE_TRANSCRIPT_ENABLED). + * + * Must agree with what was passed to extractText: when transcripts are in use + * the voice note is already readable as text, and when they are not it has to + * stay in the plan so the peer gets the "didn't catch that" line. + */ + voiceTranscriptEnabled?: boolean; +} + +export function planInboundMedia( + refs: readonly InboundMediaRef[], + opts: MediaPlanOptions, +): MediaPlanEntry[] { + const maxImages = Math.max(1, Math.floor(opts.maxImages)); + const useTranscript = opts.voiceTranscriptEnabled !== false; + const plan: MediaPlanEntry[] = []; + let images = 0; + + for (const ref of refs) { + // iLink already transcribed this one and extractText folded it into the + // message text. Listing it would tell the model it cannot hear something it + // is about to read. With transcripts disabled there is no such text, so the + // ref stays and becomes a notice-only attachment. + if (useTranscript && ref.kind === "voice" && ref.transcript?.trim()) { + continue; + } + + // Only images can be handed to an OpenAI-compatible model. Fetching voice / + // video / file bytes we cannot use would be pure bandwidth, so they are + // never downloaded — WeChat voice is SILK/AMR, which no chat completions + // endpoint accepts anyway. + const wantImage = + ref.kind === "image" && opts.visionEnabled && images < maxImages; + if (wantImage) images++; + plan.push({ ref, download: wantImage }); + } + + return plan; +} + +/** + * Reply for a message that is nothing but media the bot cannot perceive. + * + * Replaces a blanket "目前只支持文字消息喵～" with a line that names what + * actually arrived, so the user knows whether to retype it or that this kind of + * attachment simply is not supported. + */ +export function unreadableMediaReply( + refs: readonly InboundMediaRef[], +): string { + switch (refs[0]?.kind) { + case "image": + return "我这边还看不了图片呢～你用文字跟我说说好不好？"; + case "voice": + return "这段语音我没听清，方便打字告诉我吗？"; + case "video": + return "视频我还看不了呀，用文字聊好不好～"; + case "file": + return "文件我打不开呢，重要内容可以贴成文字发我～"; + default: + return "目前只支持文字消息喵～请发文字聊天。"; + } +} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts new file mode 100644 index 0000000..6ea8935 --- /dev/null +++ b/apps/api/src/index.ts @@ -0,0 +1,492 @@ +import path from "node:path"; +import { constants as zlibConstants } from "node:zlib"; +import { fileURLToPath } from "node:url"; +import Fastify from "fastify"; +import compress from "@fastify/compress"; +import { ChatService, TryChatService } from "@wechat-ai/core"; +import { openDatabase, seedPersonas, setRedisCommandHook } from "@wechat-ai/db"; +import { LlmClient } from "@wechat-ai/llm"; +import { BotLoginSessionManager } from "./bot-login-sessions.js"; +import { + CC_HTML_APP, + CC_HTML_MARKETING, + CC_OG, + CDN_HTML_APP, + CDN_HTML_MARKETING, + CDN_OG, + ifNoneMatchHits, + setPublicCache, +} from "./cache-headers.js"; +import { initActivityBus } from "./activity-stream.js"; +import { LOG_LEVELS, loadConfig } from "./config.js"; +import { registerRoutes } from "./routes.js"; +import { + buildFastifyOptions, + registerRequestLogging, +} from "./server-options.js"; +import { RuntimeConfigManager } from "./runtime-config.js"; +import { + applyRuntimeConfigToServices, + type RuntimeConfigTargets, +} from "./runtime-config-apply.js"; +import { + loadStaticAssets, + pickEncoded, + upgradeStaticCompression, +} from "./static-pages.js"; +import { BotWorkerManager } from "./worker.js"; +import { loadLinuxDoConfig } from "./oauth-linuxdo.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// The bot worker runs in this same process/event loop. A stray rejection from +// any of its detached loops must not take the HTTP server down with it. +process.on("unhandledRejection", (reason) => { + console.error("[fatal] unhandled rejection (kept alive):", reason); +}); +process.on("uncaughtException", (err) => { + console.error("[fatal] uncaught exception (kept alive):", err); +}); + +async function main(): Promise { + const cfg = loadConfig(); + console.log(`[config] repoRoot=${cfg.repoRoot}`); + console.log(`[config] redis=${cfg.redisUrl}`); + console.log( + `[config] stickers=redis blob (max ${cfg.stickerMaxBytes} bytes)`, + ); + + const db = openDatabase(cfg.redisUrl); + try { + await db.ping(); + console.log("[redis] PONG"); + } catch (err) { + console.error( + "[redis] 无法连接 REDIS_URL，请检查远端 Redis：", + cfg.redisUrl, + err, + ); + process.exit(1); + } + + await seedPersonas(db); + + // Redis-stored admin overrides on top of env. Loaded BEFORE any service is + // constructed so boot already uses the effective values; the fan-out target + // is filled in once the services exist, and the 5s poll starts after that. + let runtimeTargets: RuntimeConfigTargets | null = null; + const settings = new RuntimeConfigManager(db, cfg, (changed, live) => { + if (runtimeTargets) { + applyRuntimeConfigToServices(changed, live, runtimeTargets); + } + }); + await settings.init(); + { + const v = settings.view(); + console.log( + `[settings] runtime overrides=${v.overriddenCount}/${v.items.length}` + + (v.updatedAt ? ` updatedAt=${v.updatedAt} by=${v.updatedBy}` : ""), + ); + for (const w of settings.currentWarnings()) console.warn(`[settings] ${w}`); + } + + const activityBus = initActivityBus({ + db, + source: process.env.WORKER_ID?.trim() || "api", + enabled: cfg.dataStreamEnabled, + maxEps: cfg.dataStreamMaxEps, + redisSample: cfg.dataStreamRedisSample, + }); + // Installed unconditionally: noteRedisCmd() no-ops while the bus is + // disabled, and the admin panel can turn DATA_STREAM_ENABLED on at runtime — + // a boot-time branch here would leave that switch permanently dead. + setRedisCommandHook((info) => activityBus.noteRedisCmd(info)); + if (cfg.dataStreamEnabled) { + void activityBus.start().then(() => { + console.log( + `[stream] activity bus on sample=${cfg.dataStreamRedisSample} maxEps=${cfg.dataStreamMaxEps}`, + ); + }); + } + + // Platform (admin) LLM: direct. User custom APIs + search: TOOLS gateway only. + const llm = LlmClient.forPlatform({ + baseURL: cfg.llmBaseUrl, + apiKey: cfg.llmApiKey || "missing", + model: cfg.llmModel, + toolsBaseUrl: cfg.toolsBaseUrl || undefined, + toolsApiKey: cfg.toolsApiKey || undefined, + }); + if (!cfg.llmApiKey) { + console.warn("[warn] LLM_API_KEY not set (platform / admin LLM)"); + } + if (cfg.webSearchEnabled && !cfg.toolsBaseUrl) { + console.warn( + "[warn] WEB_SEARCH_ENABLED but TOOLS_BASE_URL empty — search will fail until HF tools is configured", + ); + } + if (cfg.toolsBaseUrl) { + console.log(`[config] tools gateway=${cfg.toolsBaseUrl} (user custom LLM + search)`); + } else { + console.log( + "[config] TOOLS_BASE_URL not set — user custom LLM APIs and web search unavailable", + ); + } + + /** + * Vision endpoint for reading inbound images. + * + * Separate from the platform LLM on purpose: the roleplay model is usually + * text-only (deepseek et al), so caption mode sends the image to a + * vision-capable endpoint and passes only its text description onward. + * Base/key default to the platform LLM's, which covers providers that host a + * vision model alongside the chat model. + */ + const visionLlm = cfg.visionEnabled + ? LlmClient.forPlatform({ + baseURL: cfg.visionBaseUrl || cfg.llmBaseUrl, + apiKey: cfg.visionApiKey || cfg.llmApiKey || "missing", + model: cfg.visionModel || cfg.llmModel, + maxTokens: cfg.visionCaptionMaxTokens, + }) + : null; + if (cfg.visionEnabled) { + if (!cfg.visionModel) { + console.warn( + "[warn] VISION_ENABLED=true but VISION_MODEL is empty — images will be reported as unreadable. Set VISION_MODEL to a vision-capable model id.", + ); + } else { + console.log( + `[config] vision mode=${cfg.visionMode} model=${cfg.visionModel} base=${ + cfg.visionBaseUrl || cfg.llmBaseUrl + }`, + ); + } + } + + const publicBase = cfg.publicBaseUrl.replace(/\/$/, ""); + const chat = new ChatService( + db, + llm, + { + shortHistoryLimit: cfg.shortHistoryLimit, + memoryExtractEveryN: cfg.memoryExtractEveryN, + allowUnapproved: cfg.allowUnapproved, + unapprovedReply: + `账号尚未开通对话权限。请前往网页端批准对话权限！\n（此项目为公益免费项目！使用文档：${publicBase}/docs）`, + multiBubbleJson: cfg.multiBubbleJson, + replyFilterEnabled: cfg.replyFilterEnabled, + maxReplyBubbles: cfg.maxReplyChunks, + maxChunkChars: cfg.maxChunkChars, + maxStickersPerReply: cfg.maxStickersPerReply, + stickersEnabled: cfg.stickerSendEnabled, + memoryTopK: cfg.memoryTopK, + memoryFullInjectMax: cfg.memoryFullInjectMax, + memoryMaxItems: cfg.memoryMaxItems, + timeToolEnabled: cfg.timeToolEnabled, + timeToolTimeZone: cfg.timeToolTimeZone, + webSearchEnabled: cfg.webSearchEnabled, + toolsBaseUrl: cfg.toolsBaseUrl || undefined, + toolsApiKey: cfg.toolsApiKey || undefined, + llmProviderSecret: cfg.llmProviderSecret || undefined, + chatflowHttpAllowHosts: cfg.chatflowHttpAllowlist, + chatflowMaxSteps: cfg.chatflowMaxSteps, + chatflowMaxNodes: cfg.chatflowMaxNodes, + visionMode: cfg.visionMode, + visionModel: cfg.visionModel || undefined, + visionCaptionMaxTokens: cfg.visionCaptionMaxTokens, + }, + visionLlm, + ); + + const tryChat = new TryChatService(db, llm, { + sessionTtlSec: cfg.tryChatSessionTtlSec, + maxHistory: cfg.tryChatMaxHistory, + maxUserMsgsPerDay: cfg.tryChatMaxUserMsgsPerDay, + maxUserMsgsPerSession: cfg.tryChatMaxUserMsgsPerSession, + multiBubbleJson: cfg.multiBubbleJson, + replyFilterEnabled: cfg.replyFilterEnabled, + maxReplyBubbles: cfg.maxReplyChunks, + maxChunkChars: cfg.maxChunkChars, + timeToolEnabled: cfg.timeToolEnabled, + timeToolTimeZone: cfg.timeToolTimeZone, + toolsBaseUrl: cfg.toolsBaseUrl || undefined, + toolsApiKey: cfg.toolsApiKey || undefined, + webSearchEnabled: cfg.webSearchEnabled, + chatflowHttpAllowHosts: cfg.chatflowHttpAllowlist, + chatflowMaxSteps: cfg.chatflowMaxSteps, + chatflowMaxNodes: cfg.chatflowMaxNodes, + }); + + const worker = new BotWorkerManager({ + db, + chat, + stickerSendEnabled: cfg.stickerSendEnabled, + maxStickersPerReply: cfg.maxStickersPerReply, + visionEnabled: cfg.visionEnabled, + visionMaxImages: cfg.visionMaxImages, + inboundMediaMaxBytes: cfg.inboundMediaMaxBytes, + voiceTranscriptEnabled: cfg.voiceTranscriptEnabled, + peerRatePerMinute: cfg.peerRatePerMinute, + maxBotsPerWorker: cfg.maxBotsPerWorker, + leaseTtlSec: cfg.leaseTtlSec, + leaseRenewSec: cfg.leaseRenewSec, + rebalanceEnabled: cfg.rebalanceEnabled, + rebalanceIntervalSec: cfg.rebalanceIntervalSec, + rebalanceSlack: cfg.rebalanceSlack, + rebalanceMaxPerTick: cfg.rebalanceMaxPerTick, + workerWeightTtlSec: cfg.workerWeightTtlSec, + replyConcurrency: cfg.replyConcurrency, + inboxMaxLen: cfg.inboxMaxLen, + splitReply: cfg.splitReply, + maxReplyChunks: cfg.maxReplyChunks, + maxChunkChars: cfg.maxChunkChars, + replyDelay: { + msPerChar: cfg.replyDelayMsPerChar, + minMs: cfg.replyDelayMinMs, + maxMs: cfg.replyDelayMaxMs, + firstMinMs: cfg.replyDelayFirstMinMs, + firstMaxMs: cfg.replyDelayFirstMaxMs, + thinkExtraMs: cfg.replyDelayThinkExtraMs, + }, + proactive: { + globalEnabled: cfg.proactiveEnabled, + defaultIdleHours: cfg.proactiveIdleHours, + defaultMinIntervalHours: cfg.proactiveMinIntervalHours, + defaultMaxPerDay: cfg.proactiveMaxPerDay, + defaultQuietHours: cfg.proactiveQuietHours, + scanIntervalSec: cfg.proactiveScanIntervalSec, + maxPerScan: cfg.proactiveMaxPerScan, + lockTtlSec: cfg.proactiveLockTtlSec, + attemptCooldownHours: cfg.proactiveAttemptCooldownHours, + }, + broadcast: { + intervalMs: cfg.broadcastIntervalMs, + pollIntervalMs: 2_000, + lockTtlSec: 60, + }, + p2pEnabled: cfg.p2pEnabled, + p2p: { + bindCodeTtlSec: cfg.p2pBindCodeTtlSec, + requestTtlSec: cfg.p2pRequestTtlSec, + sessionIdleSec: cfg.p2pSessionIdleSec, + relayMaxChars: cfg.p2pRelayMaxChars, + maxRequestsPerDay: cfg.p2pMaxRequestsPerDay, + }, + nodeLabel: cfg.nodeLabel, + nodeRegion: cfg.nodeRegion, + appVersion: cfg.appVersion, + repoRoot: cfg.repoRoot, + otaEnabled: cfg.otaEnabled, + otaAllowInstall: cfg.otaAllowInstall, + otaStagingDir: cfg.otaStagingDir, + log: (msg, extra) => { + if (extra) console.log(msg, extra); + else console.log(msg); + }, + }); + + runtimeTargets = { chat, tryChat, worker, activityBus }; + settings.start(); + + const loginSessions = new BotLoginSessionManager(db, worker); + // Stickers / OTA blob upload as JSON base64 (~4/3 raw); allow up to ~12MB payload + // 12MB is only needed by the upload routes; as a global default it let any + // unauthenticated POST make the process buffer 12MB before a handler ran. + // Those routes set `bodyLimit: cfg.uploadBodyLimit` per route instead. + const app = Fastify(buildFastifyOptions(cfg)); + + const rawLogLevel = (process.env.LOG_LEVEL ?? "").trim(); + if (rawLogLevel && rawLogLevel.toLowerCase() !== cfg.logLevel) { + app.log.warn( + { requested: rawLogLevel, using: cfg.logLevel, valid: LOG_LEVELS }, + "LOG_LEVEL is not a pino level — falling back", + ); + } + + registerRequestLogging(app, cfg); + + await app.register(compress, { + global: true, + threshold: 4096, + encodings: ["br", "gzip", "deflate"], + // Dynamic JSON gets compressed synchronously on the event loop. Default + // brotli quality is far too slow for 30-60KB admin listings; q4 lands + // near gzip speed at better ratio. Static shells bypass this middleware + // entirely (static-pages.ts sets Content-Encoding itself). + brotliOptions: { + params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 }, + }, + }); + + await registerRoutes(app, { + db, + chat, + tryChat, + worker, + loginSessions, + cfg, + activityBus, + settings, + }); + + const publicDir = path.join(__dirname, "../public"); + const staticAssets = loadStaticAssets(publicDir, publicBase); + console.log( + `[static] pages=${[...staticAssets.pages.keys()].join(",") || "(none)"} og=${staticAssets.og ? "yes" : "no"}`, + ); + + const sendCachedPage = ( + route: string, + browserCc: string, + edgeCc: string, + req: import("fastify").FastifyRequest, + reply: import("fastify").FastifyReply, + ) => { + const page = staticAssets.pages.get(route); + if (!page) return null; + // Serve the boot-time brotli/gzip buffer when the client accepts it. + // Setting Content-Encoding also tells @fastify/compress to stand down. + const variant = pickEncoded(page, req.headers["accept-encoding"]); + const etag = variant?.etag ?? page.etag; + setPublicCache(reply, browserCc, edgeCc, { + etag, + cacheTag: "html-shell", + }); + reply.header("Vary", "Accept-Encoding"); + if (ifNoneMatchHits(req.headers["if-none-match"], etag)) { + return reply.code(304).send(); + } + reply.type(page.contentType); + if (variant) { + reply.header("Content-Encoding", variant.encoding); + return reply.send(variant.body); + } + return reply.send(page.body); + }; + + // Landing (feature intro + OG for link previews). App console stays at /app. + app.get("/", async (req, reply) => { + const sent = sendCachedPage( + "/", + CC_HTML_MARKETING, + CDN_HTML_MARKETING, + req, + reply, + ); + if (sent) return sent; + return reply.redirect("/app"); + }); + app.get("/app", async (req, reply) => { + const sent = sendCachedPage( + "/app", + CC_HTML_APP, + CDN_HTML_APP, + req, + reply, + ); + if (sent) return sent; + return reply.code(404).send("app.html missing"); + }); + app.get("/docs", async (req, reply) => { + const sent = sendCachedPage( + "/docs", + CC_HTML_MARKETING, + CDN_HTML_MARKETING, + req, + reply, + ); + if (sent) return sent; + return reply.code(404).send("docs.html missing"); + }); + app.get("/admin", async (req, reply) => { + const sent = sendCachedPage( + "/admin", + CC_HTML_APP, + CDN_HTML_APP, + req, + reply, + ); + if (sent) return sent; + return reply.code(404).send("admin.html missing"); + }); + app.get("/chatflow", async (req, reply) => { + const sent = sendCachedPage( + "/chatflow", + CC_HTML_APP, + CDN_HTML_APP, + req, + reply, + ); + if (sent) return sent; + return reply.code(404).send("chatflow.html missing"); + }); + app.get("/og.jpg", async (req, reply) => { + const og = staticAssets.og; + if (!og) return reply.code(404).send("og image missing"); + setPublicCache(reply, CC_OG, CDN_OG, { + etag: og.etag, + cacheTag: "og-image", + }); + if (ifNoneMatchHits(req.headers["if-none-match"], og.etag)) { + return reply.code(304).send(); + } + return reply.type(og.contentType).send(og.body); + }); + + await app.listen({ host: cfg.host, port: cfg.port }); + // Max-quality shell compression, off the boot path + void upgradeStaticCompression(staticAssets).then( + () => console.log("[static] shells recompressed (brotli q11)"), + (err) => console.warn("[static] recompress failed (serving q5):", err), + ); + const oauth = loadLinuxDoConfig(); + console.log(`Landing http://${cfg.host}:${cfg.port}/`); + console.log(`App UI http://${cfg.host}:${cfg.port}/app`); + console.log(`Docs http://${cfg.host}:${cfg.port}/docs`); + console.log(`Admin UI http://${cfg.host}:${cfg.port}/admin`); + console.log(`Chatflow http://${cfg.host}:${cfg.port}/chatflow`); + console.log( + `[version] ${cfg.appVersion} ota=${cfg.otaEnabled ? "on" : "off"}`, + ); + console.log( + oauth + ? `[oauth] LINUX DO enabled → ${oauth.redirectUri}` + : "[oauth] LINUX DO 未配置（设置 LINUXDO_CLIENT_ID/SECRET/REDIRECT_URI）", + ); + + if (cfg.workerEnabled) { + // Do not block process forever if Redis is slow; start() is still awaited + // but bootstrap is now batched. Log clearly on failure. + try { + await worker.start(); + } catch (err) { + console.error( + "[worker] start failed (API stays up; check Redis / logs):", + err, + ); + } + } else { + console.log("WORKER_ENABLED=false"); + } + + let shuttingDown = false; + const shutdown = async () => { + if (shuttingDown) return; + shuttingDown = true; + settings.stop(); + // Await the fleet deregistration so peers re-claim this node's bots + // immediately instead of waiting out the lease TTL. + await worker.stopAsync().catch(() => undefined); + await app.close().catch(() => undefined); + await db.close().catch(() => undefined); + process.exit(0); + }; + process.on("SIGINT", () => void shutdown()); + process.on("SIGTERM", () => void shutdown()); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/api/src/oauth-linuxdo.ts b/apps/api/src/oauth-linuxdo.ts new file mode 100644 index 0000000..4090c88 --- /dev/null +++ b/apps/api/src/oauth-linuxdo.ts @@ -0,0 +1,236 @@ +import { randomBytes, createHash } from "node:crypto"; + +export interface LinuxDoOAuthConfig { + clientId: string; + clientSecret: string; + redirectUri: string; + authorizeUrl: string; + tokenUrl: string; + userInfoUrl: string; + scope: string; +} + +export interface LinuxDoUserInfo { + id: string | number; + username: string; + name?: string; + avatar_url?: string; + avatar_template?: string; + trust_level?: number; + active?: boolean; + silenced?: boolean; + email?: string; + sub?: string; + login?: string; +} + +export function loadLinuxDoConfig( + env: NodeJS.ProcessEnv = process.env, +): LinuxDoOAuthConfig | null { + const clientId = env.LINUXDO_CLIENT_ID ?? ""; + const clientSecret = env.LINUXDO_CLIENT_SECRET ?? ""; + const redirectUri = env.LINUXDO_REDIRECT_URI ?? ""; + if (!clientId || !clientSecret || !redirectUri) return null; + return { + clientId, + clientSecret, + redirectUri, + authorizeUrl: + env.LINUXDO_AUTHORIZE_URL ?? + "https://connect.linux.do/oauth2/authorize", + tokenUrl: + env.LINUXDO_TOKEN_URL ?? "https://connect.linux.do/oauth2/token", + userInfoUrl: + env.LINUXDO_USERINFO_URL ?? "https://connect.linux.do/api/user", + // OIDC discovery: scopes_supported = openid, profile, email + scope: env.LINUXDO_SCOPE ?? "openid profile", + }; +} + +export function newOAuthState(): string { + return randomBytes(24).toString("hex"); +} + +export function buildAuthorizeUrl( + cfg: LinuxDoOAuthConfig, + state: string, +): string { + const u = new URL(cfg.authorizeUrl); + u.searchParams.set("client_id", cfg.clientId); + u.searchParams.set("redirect_uri", cfg.redirectUri); + u.searchParams.set("response_type", "code"); + u.searchParams.set("scope", cfg.scope); + u.searchParams.set("state", state); + return u.toString(); +} + +export async function exchangeCode( + cfg: LinuxDoOAuthConfig, + code: string, +): Promise<{ access_token: string; token_type?: string }> { + const body = new URLSearchParams({ + grant_type: "authorization_code", + client_id: cfg.clientId, + client_secret: cfg.clientSecret, + code, + redirect_uri: cfg.redirectUri, + }); + const res = await fetch(cfg.tokenUrl, { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Accept: "application/json", + }, + body, + }); + const text = await res.text(); + let data: { + access_token?: string; + token_type?: string; + error?: string; + error_description?: string; + }; + try { + data = JSON.parse(text) as typeof data; + } catch { + throw new Error(`token exchange invalid JSON: HTTP ${res.status} ${text.slice(0, 200)}`); + } + if (!res.ok || !data.access_token) { + throw new Error( + data.error_description || + data.error || + `token exchange HTTP ${res.status}: ${text.slice(0, 200)}`, + ); + } + return { access_token: data.access_token, token_type: data.token_type }; +} + +/** + * Resolve avatar URL from LINUX DO / Discourse / OIDC userinfo. + * Supports avatar_url, picture, and Discourse avatar_template ("…/{size}/…"). + */ +export function resolveAvatarUrl( + raw: Record, + size = 96, +): string | undefined { + const pick = (...keys: string[]): string | undefined => { + for (const k of keys) { + const v = raw[k]; + if (typeof v === "string" && v.trim()) return v.trim(); + } + return undefined; + }; + + let url = + pick("avatar_url", "avatarUrl", "picture", "image", "avatar") || + undefined; + + const template = pick("avatar_template", "avatarTemplate"); + if (!url && template) { + url = template.includes("{size}") + ? template.replace(/\{size\}/g, String(size)) + : template; + } + + if (!url) return undefined; + + // Protocol-relative //cdn... + if (url.startsWith("//")) url = "https:" + url; + // Discourse sometimes returns /user_avatar/... + if (url.startsWith("/")) url = "https://linux.do" + url; + + try { + const u = new URL(url); + if (u.protocol !== "http:" && u.protocol !== "https:") return undefined; + return u.toString(); + } catch { + return undefined; + } +} + +/** + * Normalize LINUX DO / OIDC userinfo into a stable shape. + * Fields may be id|sub, username|login, name, avatar_url|picture|avatar_template, trust_level. + */ +export function normalizeUserInfo(raw: Record): LinuxDoUserInfo { + const id = raw.id ?? raw.sub ?? raw.user_id; + const username = + (raw.username as string) || + (raw.login as string) || + (raw.preferred_username as string) || + (id != null ? String(id) : ""); + if (id == null || !username) { + throw new Error( + `userinfo missing id/username: ${JSON.stringify(raw).slice(0, 300)}`, + ); + } + return { + id: id as string | number, + username, + name: (raw.name as string) || username, + avatar_url: resolveAvatarUrl(raw), + avatar_template: + typeof raw.avatar_template === "string" + ? raw.avatar_template + : typeof raw.avatarTemplate === "string" + ? raw.avatarTemplate + : undefined, + trust_level: Number(raw.trust_level ?? 0), + active: raw.active as boolean | undefined, + silenced: raw.silenced as boolean | undefined, + email: raw.email as string | undefined, + sub: raw.sub as string | undefined, + login: raw.login as string | undefined, + }; +} + +export async function fetchUserInfo( + cfg: LinuxDoOAuthConfig, + accessToken: string, +): Promise { + const res = await fetch(cfg.userInfoUrl, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: "application/json", + }, + }); + const text = await res.text(); + let data: Record; + try { + data = JSON.parse(text) as Record; + } catch { + throw new Error(`userinfo invalid JSON: HTTP ${res.status}`); + } + if (!res.ok) { + throw new Error(`userinfo HTTP ${res.status}: ${text.slice(0, 200)}`); + } + // Some providers nest under { user: {...} } + if (data.user && typeof data.user === "object") { + return normalizeUserInfo(data.user as Record); + } + return normalizeUserInfo(data); +} + +export function parseAdminIds(raw: string | undefined): Set { + if (!raw) return new Set(); + return new Set( + raw + .split(/[,;\s]+/) + .map((s) => s.trim()) + .filter(Boolean), + ); +} + +export function defaultSessionSecret(): string { + return createHash("sha256") + .update(process.env.WECHAT_AI_TOKEN || "dev") + .digest("hex"); +} + +export function isPlaceholderRedisUrl(url: string): boolean { + return ( + !url || + /YOUR_ENDPOINT|YOUR_UPSTASH|password@your|localhost:6379\/0$/i.test(url) || + url.includes("YOUR_") + ); +} diff --git a/apps/api/src/ota-apply.test.ts b/apps/api/src/ota-apply.test.ts new file mode 100644 index 0000000..6cdc80a --- /dev/null +++ b/apps/api/src/ota-apply.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, it } from "node:test"; +import { createHash } from "node:crypto"; +import { scanLocalOtaHashes } from "./ota-apply.js"; + +describe("ota-apply scanLocalOtaHashes", () => { + it("scans allowed files under a fake monorepo", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "wa-ota-")); + try { + fs.writeFileSync(path.join(root, "package.json"), '{"version":"1.0.0"}'); + fs.writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages:\n - packages/*\n"); + fs.mkdirSync(path.join(root, "apps", "api", "src"), { recursive: true }); + fs.writeFileSync( + path.join(root, "apps", "api", "src", "index.ts"), + "export {}\n", + ); + fs.mkdirSync(path.join(root, "apps", "api", "node_modules", "x"), { + recursive: true, + }); + fs.writeFileSync( + path.join(root, "apps", "api", "node_modules", "x", "a.js"), + "nope", + ); + fs.mkdirSync(path.join(root, "packages", "db", "src"), { recursive: true }); + fs.writeFileSync( + path.join(root, "packages", "db", "package.json"), + "{}", + ); + fs.writeFileSync( + path.join(root, "packages", "db", "src", "keys.ts"), + "export {}\n", + ); + fs.writeFileSync(path.join(root, "packages", "db", "README.md"), "no"); + + const map = scanLocalOtaHashes(root); + assert.ok(map.has("package.json")); + assert.ok(map.has("apps/api/src/index.ts")); + assert.ok(map.has("packages/db/src/keys.ts")); + assert.ok(map.has("packages/db/package.json")); + assert.equal(map.has("apps/api/node_modules/x/a.js"), false); + assert.equal(map.has("packages/db/README.md"), false); + + const h = createHash("sha256") + .update(fs.readFileSync(path.join(root, "package.json"))) + .digest("hex"); + assert.equal(map.get("package.json"), h); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/api/src/ota-apply.ts b/apps/api/src/ota-apply.ts new file mode 100644 index 0000000..9e61055 --- /dev/null +++ b/apps/api/src/ota-apply.ts @@ -0,0 +1,435 @@ +/** + * Local OTA apply: hash scan, download blobs, staging, atomic swap, optional install. + * Invoked by BotWorkerManager when a Redis update job is pending. + */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { + clearWorkerUpdateJob, + diffReleaseFiles, + getBlob, + getReleaseMeta, + getWorkerUpdateJob, + isAllowedOtaPath, + normalizeOtaPath, + setWorkerUpdateStatus, + type Db, + type NodeUpdateJob, + type NodeUpdateStatus, + type ReleaseFileEntry, + type ReleaseMeta, +} from "@wechat-ai/db"; + +export interface OtaApplyOptions { + db: Db; + workerId: string; + repoRoot: string; + appVersion: string; + stagingDir: string; + allowInstall: boolean; + log?: (msg: string, extra?: unknown) => void; + /** Called after files applied, before process exit */ + beforeRestart?: () => void | Promise; +} + +function sha256File(abs: string): string | null { + try { + if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) return null; + const h = createHash("sha256"); + h.update(fs.readFileSync(abs)); + return h.digest("hex"); + } catch { + return null; + } +} + +/** Walk whitelist trees under repo root and return path → sha256. */ +export function scanLocalOtaHashes(repoRoot: string): Map { + const out = new Map(); + const root = path.resolve(repoRoot); + + const tryFile = (relPosix: string) => { + if (!isAllowedOtaPath(relPosix)) return; + const abs = path.join(root, ...relPosix.split("/")); + const hash = sha256File(abs); + if (hash) out.set(relPosix, hash); + }; + + for (const f of [ + "package.json", + "pnpm-workspace.yaml", + "pnpm-lock.yaml", + "tsconfig.base.json", + ]) { + tryFile(f); + } + + const walk = (absDir: string, relPrefix: string) => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(absDir, { withFileTypes: true }); + } catch { + return; + } + for (const ent of entries) { + const name = ent.name; + if ( + name === "node_modules" || + name === "data" || + name === ".git" || + name === ".wa-update-staging" || + name === ".wa-backup" + ) { + continue; + } + const rel = relPrefix ? `${relPrefix}/${name}` : name; + const abs = path.join(absDir, name); + if (ent.isDirectory()) { + walk(abs, rel); + } else if (ent.isFile()) { + tryFile(rel.replace(/\\/g, "/")); + } + } + }; + + for (const dir of [ + "apps/api", + "packages/core", + "packages/db", + "packages/ilink", + "packages/llm", + "scripts", + ]) { + const abs = path.join(root, ...dir.split("/")); + if (fs.existsSync(abs)) walk(abs, dir); + } + + return out; +} + +function resolveStagingAbs(repoRoot: string, stagingDir: string): string { + if (path.isAbsolute(stagingDir)) return stagingDir; + return path.join(path.resolve(repoRoot), stagingDir); +} + +function safeWriteFile(abs: string, data: Buffer): void { + fs.mkdirSync(path.dirname(abs), { recursive: true }); + const tmp = abs + ".tmp." + process.pid; + fs.writeFileSync(tmp, data); + fs.renameSync(tmp, abs); +} + +async function runPnpmInstall(repoRoot: string, log?: OtaApplyOptions["log"]): Promise { + await new Promise((resolve, reject) => { + const child = spawn( + "pnpm", + ["install", "--frozen-lockfile"], + { + cwd: repoRoot, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + shell: true, + }, + ); + let err = ""; + child.stderr?.on("data", (c: Buffer) => { + err += c.toString(); + if (err.length > 4000) err = err.slice(-4000); + }); + child.on("error", (e) => reject(e)); + child.on("close", (code) => { + if (code === 0) resolve(); + else { + log?.(`[ota] pnpm install exit ${code}: ${err.slice(0, 500)}`); + reject(new Error(`pnpm_install_failed:${code}`)); + } + }); + }); +} + +async function downloadNeeded( + db: Db, + needed: ReleaseFileEntry[], + onProgress: (done: number, bytesDone: number) => Promise, +): Promise> { + const map = new Map(); + let done = 0; + let bytesDone = 0; + for (const f of needed) { + const buf = await getBlob(db, f.sha256); + if (!buf) throw new Error(`blob_missing:${f.path}`); + if (buf.length !== f.size) { + throw new Error(`blob_size_mismatch:${f.path}`); + } + map.set(f.path, buf); + done++; + bytesDone += buf.length; + await onProgress(done, bytesDone); + } + return map; +} + +/** + * Apply one OTA job if present. Returns: + * - `null` if no job + * - `"applied"` if files written and caller should restart + * - `"noop"` if already on version / skipped + * - `"failed"` if error recorded + */ +export async function tryApplyOtaUpdate( + opts: OtaApplyOptions, +): Promise<"applied" | "noop" | "failed" | null> { + const job = await getWorkerUpdateJob(opts.db, opts.workerId); + if (!job) return null; + + return applyOtaJob(opts, job); +} + +export async function applyOtaJob( + opts: OtaApplyOptions, + job: NodeUpdateJob, +): Promise<"applied" | "noop" | "failed"> { + const log = opts.log; + const workerId = opts.workerId; + const now = () => new Date().toISOString(); + + const patchStatus = async ( + phase: NodeUpdateStatus["phase"], + extra?: Partial, + ) => { + const st: NodeUpdateStatus = { + workerId, + version: job.version, + phase, + error: null, + startedAt: extra?.startedAt ?? now(), + updatedAt: now(), + progress: extra?.progress, + changedFiles: extra?.changedFiles, + message: extra?.message ?? null, + }; + await setWorkerUpdateStatus(opts.db, st); + }; + + try { + // Idempotent: already running this version + if (!job.force && opts.appVersion === job.version) { + await patchStatus("done", { + startedAt: now(), + message: "already_on_version", + progress: { done: 0, total: 0 }, + }); + await clearWorkerUpdateJob(opts.db, workerId); + log?.(`[ota] skip already on version ${job.version}`); + return "noop"; + } + + const release = + (await getReleaseMeta(opts.db, job.version)) ?? null; + if (!release) { + throw new Error("release_not_found"); + } + + await patchStatus("downloading", { + startedAt: now(), + message: "scanning_local", + progress: { done: 0, total: 0 }, + }); + + const local = scanLocalOtaHashes(opts.repoRoot); + const needed = diffReleaseFiles(release, local); + const bytesTotal = needed.reduce((a, f) => a + f.size, 0); + + log?.( + `[ota] update ${opts.appVersion} → ${job.version}: ${needed.length}/${release.fileCount} file(s) changed (${bytesTotal} bytes)`, + ); + + await patchStatus("downloading", { + startedAt: now(), + message: "downloading", + changedFiles: needed.length, + progress: { + done: 0, + total: needed.length, + bytesDone: 0, + bytesTotal, + }, + }); + + const blobs = + needed.length === 0 + ? new Map() + : await downloadNeeded( + opts.db, + needed, + async (done, bytesDone) => { + await patchStatus("downloading", { + startedAt: now(), + message: "downloading", + changedFiles: needed.length, + progress: { + done, + total: needed.length, + bytesDone, + bytesTotal, + }, + }); + }, + ); + + await patchStatus("applying", { + startedAt: now(), + message: "staging", + changedFiles: needed.length, + progress: { + done: 0, + total: needed.length, + bytesDone: bytesTotal, + bytesTotal, + }, + }); + + const staging = resolveStagingAbs(opts.repoRoot, opts.stagingDir); + // clean staging + fs.rmSync(staging, { recursive: true, force: true }); + fs.mkdirSync(staging, { recursive: true }); + + const root = path.resolve(opts.repoRoot); + const backupRoot = path.join(root, ".wa-backup", job.version); + fs.rmSync(backupRoot, { recursive: true, force: true }); + + let applied = 0; + for (const f of needed) { + const n = normalizeOtaPath(f.path); + if (!n || !isAllowedOtaPath(n)) { + throw new Error(`path_not_allowed:${f.path}`); + } + const data = blobs.get(f.path); + if (!data) throw new Error(`blob_missing:${f.path}`); + const stageAbs = path.join(staging, ...n.split("/")); + safeWriteFile(stageAbs, data); + applied++; + if (applied % 20 === 0 || applied === needed.length) { + await patchStatus("applying", { + startedAt: now(), + message: "staging", + changedFiles: needed.length, + progress: { + done: applied, + total: needed.length, + bytesDone: bytesTotal, + bytesTotal, + }, + }); + } + } + + // Atomic swap: backup then replace + for (const f of needed) { + const n = f.path; + const dest = path.join(root, ...n.split("/")); + const stageAbs = path.join(staging, ...n.split("/")); + if (fs.existsSync(dest) && fs.statSync(dest).isFile()) { + const bak = path.join(backupRoot, ...n.split("/")); + fs.mkdirSync(path.dirname(bak), { recursive: true }); + try { + fs.copyFileSync(dest, bak); + } catch { + /* best effort backup */ + } + } + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(stageAbs, dest); + } + + // Write version stamp (read before APP_VERSION env on next boot) + safeWriteFile( + path.join(root, ".wa-version"), + Buffer.from(`${job.version}\n`, "utf8"), + ); + + // Full-tree files not in local but in release that weren't "needed" are same hash — OK. + // Files only local extra: leave in place (no delete of unknown files for safety). + + if (release.requiresInstall || needsInstallFromNeeded(needed, release)) { + if (!opts.allowInstall) { + throw new Error("install_required_but_ota_allow_install_false"); + } + await patchStatus("installing", { + startedAt: now(), + message: "pnpm_install", + changedFiles: needed.length, + progress: { + done: needed.length, + total: needed.length, + bytesDone: bytesTotal, + bytesTotal, + }, + }); + log?.(`[ota] running pnpm install --frozen-lockfile`); + await runPnpmInstall(root, log); + } + + await patchStatus("restarting", { + startedAt: now(), + message: "restarting", + changedFiles: needed.length, + progress: { + done: needed.length, + total: needed.length, + bytesDone: bytesTotal, + bytesTotal, + }, + }); + + // Clear job so reboot doesn't re-apply immediately + await clearWorkerUpdateJob(opts.db, workerId); + + try { + fs.rmSync(staging, { recursive: true, force: true }); + } catch { + /* */ + } + + log?.(`[ota] applied ${job.version}, restarting process`); + await opts.beforeRestart?.(); + return "applied"; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log?.(`[ota] failed: ${message}`); + try { + await setWorkerUpdateStatus(opts.db, { + workerId, + version: job.version, + phase: "failed", + error: message.slice(0, 500), + startedAt: now(), + updatedAt: now(), + message: "failed", + }); + } catch { + /* */ + } + try { + await clearWorkerUpdateJob(opts.db, workerId); + } catch { + /* allow retry via new enqueue */ + } + return "failed"; + } +} + +function needsInstallFromNeeded( + needed: ReleaseFileEntry[], + release: ReleaseMeta, +): boolean { + if (release.requiresInstall) { + // only if install-related files actually changed + return needed.some((f) => + /package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$/.test(f.path), + ); + } + return false; +} diff --git a/apps/api/src/proactive-scheduler.ts b/apps/api/src/proactive-scheduler.ts new file mode 100644 index 0000000..1b3bdef --- /dev/null +++ b/apps/api/src/proactive-scheduler.ts @@ -0,0 +1,354 @@ +import { + type Db, + getBotAccount, + getContextToken, + getProactiveDayCount, + incrProactiveDayCount, + listProactivePeerIds, + markPeerProactive, + releaseProactiveLock, + tryAcquireProactiveLock, + writeAudit, + type Peer, + K, +} from "@wechat-ai/db"; +import { + isProactiveEligible, + mergeBotProactiveConfig, + type ChatService, + type ReplyPart, +} from "@wechat-ai/core"; +import type { ILinkClient } from "@wechat-ai/ilink"; + +export interface ProactiveSchedulerOptions { + db: Db; + chat: ChatService; + /** Global hard switch */ + globalEnabled: boolean; + defaultIdleHours: number; + defaultMinIntervalHours: number; + defaultMaxPerDay: number; + defaultQuietHours: string; + scanIntervalSec: number; + maxPerScan: number; + lockTtlSec: number; + attemptCooldownHours: number; + log?: (msg: string, extra?: unknown) => void; + /** + * Bots this process currently long-polls (has live ILinkClient). + */ + getLocalBotIds: () => string[]; + getClient: (botId: string) => ILinkClient | undefined; + /** + * Serialize send with inbound reply chain for the same peer. + * Handler should perform send + mark success. + */ + runOnPeerChain: ( + botId: string, + peerId: string, + fn: () => Promise, + ) => Promise; + sendParts: ( + client: ILinkClient, + peerId: string, + contextToken: string, + parts: ReplyPart[], + ownerUserId: string, + ) => Promise; +} + +/** + * Periodic scan: idle peers with proactive enabled get an LLM-generated nudge. + */ +export class ProactiveScheduler { + private stopped = true; + private timer: ReturnType | null = null; + private running = false; + /** + * Bumped on every stop(). An in-flight tick re-arms the chain from its + * `finally`, which stop() cannot cancel — without this, a disable/enable + * toggle during a tick leaves the old chain running untracked alongside the + * new one and the scan rate doubles per toggle. + */ + private gen = 0; + + constructor(private opts: ProactiveSchedulerOptions) {} + + /** + * Apply admin-editable settings in place (runtime settings reload). + * + * `globalEnabled` is latched by start(): the scheduler early-returns and + * leaves `stopped = true`, so flipping it on has to re-enter start() rather + * than just mutate the flag. + */ + applyRuntimeOptions(patch: Partial): void { + const wasEnabled = this.opts.globalEnabled; + Object.assign(this.opts, patch); + if (patch.globalEnabled === undefined) return; + if (!wasEnabled && this.opts.globalEnabled) { + this.stop(); + this.start(); + } else if (wasEnabled && !this.opts.globalEnabled) { + this.stop(); + } + } + + isRunning(): boolean { + return !this.stopped; + } + + start(): void { + if (!this.opts.globalEnabled) { + this.opts.log?.( + "[proactive] disabled (PROACTIVE_ENABLED=false); scheduler not started", + ); + return; + } + this.stopped = false; + this.opts.log?.( + `[proactive] scheduler start interval=${this.opts.scanIntervalSec}s ` + + `maxPerScan=${this.opts.maxPerScan}`, + ); + this.scheduleNext(2_000, this.gen); + } + + stop(): void { + this.stopped = true; + this.gen++; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + private scheduleNext(ms: number, gen: number): void { + if (this.stopped || gen !== this.gen) return; + this.timer = setTimeout(() => { + if (gen !== this.gen) return; + void this.tick() + .catch((err) => { + this.opts.log?.( + `[proactive] tick error: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }) + .finally(() => { + // A tick started before a stop() must not resurrect the chain. + this.scheduleNext(this.opts.scanIntervalSec * 1000, gen); + }); + }, ms); + } + + /** Exposed for tests / manual trigger */ + async tick(): Promise<{ sent: number; skipped: number; considered: number }> { + if (this.stopped || !this.opts.globalEnabled) { + return { sent: 0, skipped: 0, considered: 0 }; + } + if (this.running) { + return { sent: 0, skipped: 0, considered: 0 }; + } + this.running = true; + let sent = 0; + let skipped = 0; + let considered = 0; + try { + const botIds = this.opts.getLocalBotIds(); + if (!botIds.length) return { sent, skipped, considered }; + + const budget = Math.max(1, this.opts.maxPerScan); + + for (const botId of botIds) { + if (this.stopped || sent >= budget) break; + const client = this.opts.getClient(botId); + if (!client) continue; + + const bot = await getBotAccount(this.opts.db, botId); + if (!bot || bot.status !== "active") continue; + + const cfg = mergeBotProactiveConfig(bot, { + idleHours: this.opts.defaultIdleHours, + minIntervalHours: this.opts.defaultMinIntervalHours, + maxPerDay: this.opts.defaultMaxPerDay, + quietHours: this.opts.defaultQuietHours, + }); + if (!cfg.enabled) continue; + + const peerIds = await listProactivePeerIds(this.opts.db, botId); + if (!peerIds.length) continue; + + for (const peerId of peerIds) { + if (this.stopped || sent >= budget) break; + considered++; + + const peer = await this.opts.db.getJson( + K.peer(botId, peerId), + ); + if (!peer) continue; + + const contextToken = await getContextToken( + this.opts.db, + botId, + peerId, + ); + const dayCount = await getProactiveDayCount( + this.opts.db, + botId, + peerId, + ); + + const elig = isProactiveEligible({ + botStatus: bot.status, + botProactiveEnabled: cfg.enabled ? 1 : 0, + peerApproved: peer.approved, + peerProactiveEnabled: peer.proactive_enabled, + hasContextToken: Boolean(contextToken), + lastActivityAt: peer.last_activity_at, + peerCreatedAt: peer.created_at, + lastProactiveAt: peer.last_proactive_at, + lastProactiveAttemptAt: peer.last_proactive_attempt_at, + dayCount, + idleHours: cfg.idleHours, + minIntervalHours: cfg.minIntervalHours, + maxPerDay: cfg.maxPerDay, + quietHours: cfg.quietHours, + attemptCooldownHours: this.opts.attemptCooldownHours, + }); + + if (!elig.ok) { + skipped++; + continue; + } + + const locked = await tryAcquireProactiveLock( + this.opts.db, + botId, + peerId, + this.opts.lockTtlSec, + ); + if (!locked) { + this.opts.log?.( + `[proactive] bot=${botId} peer=${peerId} action=lock_miss`, + ); + skipped++; + continue; + } + + try { + await this.opts.runOnPeerChain(botId, peerId, async () => { + // Re-check context in case it was cleared + const tok = + contextToken || + (await getContextToken(this.opts.db, botId, peerId)); + if (!tok) { + await markPeerProactive(this.opts.db, botId, peerId, { + sent: false, + }); + this.opts.log?.( + `[proactive] bot=${botId} peer=${peerId} action=no_ctx`, + ); + skipped++; + return; + } + + const result = await this.opts.chat.handleProactive({ + botAccountId: botId, + peerId, + contextToken: tok, + idleHours: elig.idleHoursActual ?? cfg.idleHours, + }); + + if (result.kind === "skip" || result.kind === "reject") { + await markPeerProactive(this.opts.db, botId, peerId, { + sent: false, + }); + this.opts.log?.( + `[proactive] bot=${botId} peer=${peerId} action=skip ` + + `reason=${result.skipReason ?? result.kind} ` + + `idle=${(elig.idleHoursActual ?? 0).toFixed(1)}h`, + ); + skipped++; + return; + } + + if (result.kind !== "reply") { + await markPeerProactive(this.opts.db, botId, peerId, { + sent: false, + }); + skipped++; + return; + } + + let parts: ReplyPart[] = + result.parts && result.parts.length > 0 + ? result.parts + : result.bubbles && result.bubbles.length > 0 + ? result.bubbles.map((t) => ({ + kind: "text" as const, + text: t, + })) + : result.text + ? [{ kind: "text" as const, text: result.text }] + : []; + + if (!parts.length) { + await markPeerProactive(this.opts.db, botId, peerId, { + sent: false, + }); + skipped++; + return; + } + + const ownerUserId = bot.owner_user_id || ""; + await this.opts.sendParts( + client, + peerId, + tok, + parts, + ownerUserId, + ); + + await markPeerProactive(this.opts.db, botId, peerId, { + sent: true, + }); + await incrProactiveDayCount(this.opts.db, botId, peerId); + await writeAudit(this.opts.db, "proactive_sent", "system", { + botId, + peerId, + idleHours: elig.idleHoursActual, + personaId: result.personaId, + }); + sent++; + this.opts.log?.( + `[proactive] bot=${botId} peer=${peerId} action=send ` + + `idle=${(elig.idleHoursActual ?? 0).toFixed(1)}h ` + + `parts=${parts.length}`, + ); + }); + } catch (err) { + await markPeerProactive(this.opts.db, botId, peerId, { + sent: false, + }).catch(() => undefined); + this.opts.log?.( + `[proactive] bot=${botId} peer=${peerId} action=error: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } finally { + await releaseProactiveLock(this.opts.db, botId, peerId).catch( + () => undefined, + ); + } + } + } + } finally { + this.running = false; + } + if (sent || considered) { + this.opts.log?.( + `[proactive] tick done considered=${considered} sent=${sent} skipped=${skipped}`, + ); + } + return { sent, skipped, considered }; + } +} diff --git a/apps/api/src/qrcode.test.ts b/apps/api/src/qrcode.test.ts new file mode 100644 index 0000000..5d26ae3 --- /dev/null +++ b/apps/api/src/qrcode.test.ts @@ -0,0 +1,655 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + MAX_VERSION, + QrError, + addEccAndInterleave, + alignmentPatternPositions, + byteCapacity, + encodeQr, + gfMultiply, + maskPredicate, + numDataCodewords, + numEccBlocks, + numRawDataModules, + pickVersion, + qrSvg, + renderQrSvg, + rsComputeDivisor, + rsComputeRemainder, + type EcLevel, + type QrCode, +} from "./qrcode.js"; + +// ── Independent helpers, written against the spec rather than reusing the +// encoder's internals, so a bug in one side does not hide behind the other. ── + +/** Bit-polynomial remainder; used to check the BCH-protected fields. */ +function polyRemainder(value: number, gen: number, genBits: number): number { + let v = value; + for (;;) { + const bits = 32 - Math.clz32(v); + if (bits < genBits) return v; + v ^= gen << (bits - genBits); + } +} + +function gfPow(base: number, exp: number): number { + let r = 1; + for (let i = 0; i < exp; i++) r = gfMultiply(r, base); + return r; +} + +/** Evaluate a codeword polynomial (first byte = highest degree) at x. */ +function gfEvaluate(codeword: readonly number[], x: number): number { + let acc = 0; + for (const c of codeword) acc = gfMultiply(acc, x) ^ c; + return acc; +} + +/** + * A Reed–Solomon codeword built with generator prod(x - a^i), i number> = { + L: (v) => numRawDataModulesCodewords(v) - numDataCodewords(v, "L"), + M: (v) => numRawDataModulesCodewords(v) - numDataCodewords(v, "M"), + Q: (v) => numRawDataModulesCodewords(v) - numDataCodewords(v, "Q"), + H: (v) => numRawDataModulesCodewords(v) - numDataCodewords(v, "H"), +}; + +function numRawDataModulesCodewords(v: number): number { + return Math.floor(numRawDataModules(v) / 8); +} + +/** Function-module map derived from the spec, not from the encoder. */ +function functionMap(version: number, size: number): boolean[][] { + const fn = Array.from({ length: size }, () => + new Array(size).fill(false), + ); + const box = (x0: number, y0: number, x1: number, y1: number): void => { + for (let y = y0; y <= y1; y++) { + for (let x = x0; x <= x1; x++) { + if (x >= 0 && x < size && y >= 0 && y < size) fn[y]![x] = true; + } + } + }; + // Finder + separator + the format strips that hug them + box(0, 0, 8, 8); + box(size - 8, 0, size - 1, 8); + box(0, size - 8, 8, size - 1); + // Timing patterns + box(0, 6, size - 1, 6); + box(6, 0, 6, size - 1); + // Alignment patterns, minus the three finder corners + const aligns = alignmentPatternPositions(version); + const last = aligns.length - 1; + for (let i = 0; i <= last; i++) { + for (let j = 0; j <= last; j++) { + const corner = + (i === 0 && j === 0) || + (i === 0 && j === last) || + (i === last && j === 0); + if (corner) continue; + box(aligns[j]! - 2, aligns[i]! - 2, aligns[j]! + 2, aligns[i]! + 2); + } + } + // Version information blocks + if (version >= 7) { + box(0, size - 11, 5, size - 9); + box(size - 11, 0, size - 9, 5); + } + return fn; +} + +interface Decoded { + text: string; + ec: EcLevel; + mask: number; + blocks: number[][]; +} + +/** + * Read a symbol back: format info → unmask → zigzag → de-interleave → segment. + * Deliberately re-derives every step instead of calling encoder helpers. + */ +function decodeQr(qr: QrCode): Decoded { + const { size, version } = qr; + + // Format info copy 1: bits 0..5 down column 8, then the corner, then along row 8 + const c1: boolean[] = []; + for (let i = 0; i <= 5; i++) c1.push(qr.modules[i]![8]!); + c1.push(qr.modules[7]![8]!); + c1.push(qr.modules[8]![8]!); + c1.push(qr.modules[8]![7]!); + for (let i = 9; i < 15; i++) c1.push(qr.modules[8]![14 - i]!); + + // Format info copy 2: along row 8 from the right, then up column 8 + const c2: boolean[] = []; + for (let i = 0; i < 8; i++) c2.push(qr.modules[8]![size - 1 - i]!); + for (let i = 8; i < 15; i++) c2.push(qr.modules[size - 15 + i]![8]!); + + assert.deepEqual(c1, c2, "the two format-info copies must agree"); + + let raw = 0; + for (let i = 14; i >= 0; i--) raw = (raw << 1) | (c1[i] ? 1 : 0); + assert.equal( + polyRemainder(raw ^ 0x5412, 0x537, 11), + 0, + "format info must satisfy its BCH(15,5) check", + ); + const unmasked = (raw ^ 0x5412) >>> 10; + const ecBits = (unmasked >>> 3) & 0b11; + const mask = unmasked & 0b111; + const ec = (["M", "L", "H", "Q"] as const)[ecBits]!; + + if (version >= 7) { + let vRaw = 0; + for (let i = 17; i >= 0; i--) { + const a = size - 11 + (i % 3); + const b = Math.floor(i / 3); + vRaw = (vRaw << 1) | (qr.modules[b]![a] ? 1 : 0); + } + assert.equal( + polyRemainder(vRaw, 0x1f25, 13), + 0, + "version info must satisfy its BCH(18,6) check", + ); + assert.equal(vRaw >>> 12, version, "version info must encode the version"); + } + + // Unmask the data region + const fn = functionMap(version, size); + const grid = qr.modules.map((row) => row.slice()); + for (let y = 0; y < size; y++) { + for (let x = 0; x < size; x++) { + if (!fn[y]![x] && maskPredicate(mask, x, y)) grid[y]![x] = !grid[y]![x]; + } + } + + // Zigzag read + const total = numRawDataModulesCodewords(version); + const bits: number[] = []; + for (let right = size - 1; right >= 1; right -= 2) { + if (right === 6) right = 5; + for (let vert = 0; vert < size; vert++) { + for (let j = 0; j < 2; j++) { + const x = right - j; + const upward = ((right + 1) & 2) === 0; + const y = upward ? size - 1 - vert : vert; + if (!fn[y]![x] && bits.length < total * 8) { + bits.push(grid[y]![x] ? 1 : 0); + } + } + } + } + assert.equal(bits.length, total * 8, "zigzag must cover every data module"); + + const stream: number[] = []; + for (let i = 0; i < bits.length; i += 8) { + let b = 0; + for (let k = 0; k < 8; k++) b = (b << 1) | bits[i + k]!; + stream.push(b); + } + + // De-interleave + const eccTotal = ECC_PER_BLOCK[ec](version); + const dataTotal = numDataCodewords(version, ec); + const numBlocks = countBlocks(version, ec); + const blockEccLen = eccTotal / numBlocks; + assert.ok( + Number.isInteger(blockEccLen), + "ECC codewords must divide evenly across blocks", + ); + const numShortBlocks = numBlocks - (total % numBlocks); + const shortBlockLen = Math.floor(total / numBlocks); + + const blocks: number[][] = Array.from({ length: numBlocks }, () => []); + let p = 0; + for (let i = 0; i < shortBlockLen + 1; i++) { + for (let j = 0; j < numBlocks; j++) { + if (i === shortBlockLen - blockEccLen && j < numShortBlocks) continue; + blocks[j]!.push(stream[p++]!); + } + } + assert.equal(p, stream.length, "de-interleave must consume the whole stream"); + + const data: number[] = []; + for (let j = 0; j < numBlocks; j++) { + const dataLen = shortBlockLen - blockEccLen + (j < numShortBlocks ? 0 : 1); + data.push(...blocks[j]!.slice(0, dataLen)); + } + assert.equal(data.length, dataTotal, "recovered data codeword count"); + + // Parse the byte-mode segment + const dbits: number[] = []; + for (const b of data) { + for (let k = 7; k >= 0; k--) dbits.push((b >>> k) & 1); + } + let at = 0; + const take = (n: number): number => { + let v = 0; + for (let i = 0; i < n; i++) v = (v << 1) | dbits[at++]!; + return v; + }; + assert.equal(take(4), 0b0100, "mode indicator must be byte mode"); + const len = take(version <= 9 ? 8 : 16); + const out = Buffer.alloc(len); + for (let i = 0; i < len; i++) out[i] = take(8); + + return { text: out.toString("utf8"), ec, mask, blocks }; +} + +/** + * The block count genuinely cannot be inferred from the totals — many + * (blocks, eccPerBlock) pairs give the same product — so this reader takes the + * table value. The *split* is still independently validated: if it were wrong, + * blocks would be sliced at the wrong boundaries and the Reed–Solomon syndrome + * assertions below could not vanish. + */ +function countBlocks(version: number, ec: EcLevel): number { + return numEccBlocks(version, ec); +} + +// ── Tests ── + +describe("GF(256) arithmetic", () => { + it("has 1 as the multiplicative identity and 0 as annihilator", () => { + for (let x = 0; x < 256; x++) { + assert.equal(gfMultiply(x, 1), x); + assert.equal(gfMultiply(x, 0), 0); + } + }); + + it("is commutative", () => { + for (let x = 0; x < 256; x += 7) { + for (let y = 0; y < 256; y += 11) { + assert.equal(gfMultiply(x, y), gfMultiply(y, x)); + } + } + }); + + it("is associative and distributive over XOR", () => { + for (const [a, b, c] of [ + [2, 3, 5], + [0x53, 0xca, 0x1f], + [255, 128, 7], + ]) { + assert.equal( + gfMultiply(gfMultiply(a!, b!), c!), + gfMultiply(a!, gfMultiply(b!, c!)), + ); + assert.equal( + gfMultiply(a!, b! ^ c!), + gfMultiply(a!, b!) ^ gfMultiply(a!, c!), + ); + } + }); + + it("2 is primitive: its powers cycle with order 255", () => { + const seen = new Set(); + let v = 1; + for (let i = 0; i < 255; i++) { + assert.equal(seen.has(v), false, `repeat at exponent ${i}`); + seen.add(v); + v = gfMultiply(v, 2); + } + assert.equal(v, 1, "a^255 must wrap to 1"); + assert.equal(seen.size, 255); + }); +}); + +describe("Reed-Solomon", () => { + it("generator polynomial vanishes at a^0..a^(t-1)", () => { + for (const t of [7, 10, 13, 17, 22, 26, 30]) { + const divisor = rsComputeDivisor(t); + // g(x) = x^t + divisor[0]x^(t-1) + ... ; prepend the implicit leading 1. + const g = [1, ...divisor]; + for (let i = 0; i < t; i++) { + assert.equal( + gfEvaluate(g, gfPow(2, i)), + 0, + `g(a^${i}) should be 0 for t=${t}`, + ); + } + } + }); + + it("produces codewords with vanishing syndromes", () => { + for (const t of [7, 10, 18, 28]) { + const divisor = rsComputeDivisor(t); + const data = Array.from({ length: 20 }, (_, i) => (i * 37 + 11) & 0xff); + const ecc = rsComputeRemainder(data, divisor); + assert.equal(ecc.length, t); + assertValidRsCodeword([...data, ...ecc], t); + } + }); + + it("rejects an out-of-range degree", () => { + assert.throws(() => rsComputeDivisor(0), QrError); + assert.throws(() => rsComputeDivisor(256), QrError); + }); +}); + +describe("capacity tables", () => { + // Cross-check against the spec's published byte-mode maxima. These numbers + // are independent of the tables in qrcode.ts, so a transcription slip in + // ECC_CODEWORDS_PER_BLOCK / NUM_ECC_BLOCKS shows up here. + it("matches the published version 1 capacities", () => { + assert.equal(byteCapacity(1, "L"), 17); + assert.equal(byteCapacity(1, "M"), 14); + assert.equal(byteCapacity(1, "Q"), 11); + assert.equal(byteCapacity(1, "H"), 7); + }); + + it("matches the published version 40 capacities", () => { + assert.equal(byteCapacity(40, "L"), 2953); + assert.equal(byteCapacity(40, "M"), 2331); + assert.equal(byteCapacity(40, "Q"), 1663); + assert.equal(byteCapacity(40, "H"), 1273); + }); + + it("matches published mid-range capacities", () => { + assert.equal(byteCapacity(2, "M"), 26); + assert.equal(byteCapacity(3, "M"), 42); + // v7 is the first version with a 16-bit-free header but multiple blocks: + // 156 data codewords at L, 124 at M. + assert.equal(byteCapacity(7, "L"), 154); + assert.equal(byteCapacity(7, "M"), 122); + assert.equal(byteCapacity(10, "M"), 213); + assert.equal(byteCapacity(27, "H"), 625); + }); + + it("raw codewords equal data + ECC for every version and level", () => { + for (let v = 1; v <= MAX_VERSION; v++) { + for (const ec of ["L", "M", "Q", "H"] as EcLevel[]) { + const data = numDataCodewords(v, ec); + assert.ok(data > 0, `v${v} ${ec} must have data capacity`); + assert.ok( + data < numRawDataModulesCodewords(v), + `v${v} ${ec} must leave room for ECC`, + ); + } + } + }); + + it("capacity grows monotonically with version", () => { + for (const ec of ["L", "M", "Q", "H"] as EcLevel[]) { + for (let v = 2; v <= MAX_VERSION; v++) { + assert.ok( + byteCapacity(v, ec) > byteCapacity(v - 1, ec), + `v${v} ${ec} should exceed v${v - 1}`, + ); + } + } + }); + + it("stronger EC never has more capacity", () => { + for (let v = 1; v <= MAX_VERSION; v++) { + assert.ok(byteCapacity(v, "L") >= byteCapacity(v, "M")); + assert.ok(byteCapacity(v, "M") >= byteCapacity(v, "Q")); + assert.ok(byteCapacity(v, "Q") >= byteCapacity(v, "H")); + } + }); +}); + +describe("alignment patterns", () => { + it("version 1 has none", () => { + assert.deepEqual(alignmentPatternPositions(1), []); + }); + + it("known positions", () => { + assert.deepEqual(alignmentPatternPositions(2), [6, 18]); + assert.deepEqual(alignmentPatternPositions(7), [6, 22, 38]); + assert.deepEqual(alignmentPatternPositions(32), [6, 34, 60, 86, 112, 138]); + }); + + it("count and bounds hold for every version", () => { + for (let v = 2; v <= MAX_VERSION; v++) { + const pos = alignmentPatternPositions(v); + const size = v * 4 + 17; + assert.equal(pos.length, Math.floor(v / 7) + 2, `count for v${v}`); + assert.equal(pos[0], 6); + assert.equal(pos[pos.length - 1], size - 7); + for (let i = 1; i < pos.length; i++) { + assert.ok(pos[i]! > pos[i - 1]!, `ascending for v${v}`); + } + } + }); +}); + +describe("symbol structure", () => { + const qr = encodeQr("https://liteapp.weixin.qq.com/q/7GiQu1?qrcode=abc"); + + it("is square with the right side length", () => { + assert.equal(qr.size, qr.version * 4 + 17); + assert.equal(qr.modules.length, qr.size); + for (const row of qr.modules) assert.equal(row.length, qr.size); + }); + + it("draws all three finder patterns", () => { + const centres: Array<[number, number]> = [ + [3, 3], + [qr.size - 4, 3], + [3, qr.size - 4], + ]; + for (const [cx, cy] of centres) { + for (let dy = -3; dy <= 3; dy++) { + for (let dx = -3; dx <= 3; dx++) { + const dist = Math.max(Math.abs(dx), Math.abs(dy)); + assert.equal( + qr.modules[cy + dy]![cx + dx], + dist !== 2, + `finder at (${cx},${cy}) offset (${dx},${dy})`, + ); + } + } + } + }); + + it("keeps the separator ring light", () => { + for (let i = 0; i <= 7; i++) { + assert.equal(qr.modules[7]![i], false, `top-left separator row at ${i}`); + assert.equal(qr.modules[i]![7], false, `top-left separator col at ${i}`); + } + }); + + it("draws alternating timing patterns", () => { + for (let i = 8; i < qr.size - 8; i++) { + assert.equal(qr.modules[6]![i], i % 2 === 0, `h timing at ${i}`); + assert.equal(qr.modules[i]![6], i % 2 === 0, `v timing at ${i}`); + } + }); + + it("sets the always-dark module", () => { + assert.equal(qr.modules[qr.size - 8]![8], true); + }); + + it("picks a mask in range", () => { + assert.ok(qr.mask >= 0 && qr.mask <= 7); + }); +}); + +describe("round trip through an independent decoder", () => { + const cases: Array<{ name: string; text: string; ec?: EcLevel }> = [ + { name: "short ascii", text: "hi" }, + { name: "single char", text: "a" }, + { + name: "the real iLink scan link", + text: + "https://liteapp.weixin.qq.com/q/7GiQu1?qrcode=" + + "AQAAAO8xZ2s2S2hZbFZ4dGpuTWs5OFRvUXc9PQ%3D%3D&bot_type=3", + }, + { name: "utf-8 chinese", text: "扫码登录微信机器人：小铃" }, + { name: "mixed", text: "登录 https://a.example/x?y=1&z=2 #frag" }, + { name: "url with padding chars", text: "=".repeat(40) }, + { name: "exactly one codeword short", text: "x".repeat(13) }, + { name: "medium", text: "y".repeat(200) }, + { name: "large", text: "z".repeat(1200) }, + { name: "ec L", text: "level L payload", ec: "L" }, + { name: "ec Q", text: "level Q payload", ec: "Q" }, + { name: "ec H", text: "level H payload", ec: "H" }, + ]; + + for (const c of cases) { + it(`recovers ${c.name}`, () => { + const qr = encodeQr(c.text, c.ec ? { ec: c.ec } : {}); + const decoded = decodeQr(qr); + assert.equal(decoded.text, c.text); + assert.equal(decoded.ec, c.ec ?? "M"); + assert.equal(decoded.mask, qr.mask); + }); + } + + it("every ECC block is a valid Reed-Solomon codeword", () => { + for (const [text, ec] of [ + ["hello", "M"], + ["x".repeat(300), "M"], + ["x".repeat(300), "L"], + ["x".repeat(300), "H"], + ] as Array<[string, EcLevel]>) { + const qr = encodeQr(text, { ec }); + const { blocks } = decodeQr(qr); + const t = ECC_PER_BLOCK[ec](qr.version) / blocks.length; + for (const block of blocks) assertValidRsCodeword(block, t); + } + }); + + it("survives every version that a forced minVersion can reach", () => { + // Walk a sample of versions so multi-block, short/long-block, and + // version-info (>= 7) code paths all get exercised. + for (const v of [1, 2, 6, 7, 10, 14, 20, 27, 32, 40]) { + const qr = encodeQr("payload-" + v, { minVersion: v }); + assert.equal(qr.version, v); + assert.equal(decodeQr(qr).text, "payload-" + v); + } + }); + + it("bumps the version when the payload does not fit", () => { + const small = encodeQr("x".repeat(14)); + assert.equal(small.version, 1); + const bigger = encodeQr("x".repeat(15)); + assert.equal(bigger.version, 2); + assert.equal(decodeQr(bigger).text, "x".repeat(15)); + }); +}); + +describe("version selection", () => { + it("picks the smallest version that fits", () => { + assert.equal(pickVersion(14, "M"), 1); + assert.equal(pickVersion(15, "M"), 2); + assert.equal(pickVersion(2953, "L"), 40); + assert.equal(pickVersion(2954, "L"), null); + }); + + it("honours a minimum version", () => { + assert.equal(pickVersion(1, "M", 9), 9); + }); + + it("throws on oversized payloads", () => { + assert.throws(() => encodeQr("x".repeat(2954), { ec: "L" }), QrError); + assert.throws(() => encodeQr("x".repeat(2332), { ec: "M" }), QrError); + }); + + it("counts UTF-8 bytes, not characters", () => { + // 14 Chinese characters = 42 bytes, well past v1-M's 14-byte capacity. + const qr = encodeQr("一二三四五六七八九十壹贰叁肆"); + assert.ok(qr.version >= 3, `expected >= v3, got v${qr.version}`); + assert.equal(decodeQr(qr).text, "一二三四五六七八九十壹贰叁肆"); + }); + + it("rejects a bad EC level", () => { + assert.throws( + () => encodeQr("x", { ec: "Z" as unknown as EcLevel }), + QrError, + ); + }); +}); + +describe("interleaving invariants", () => { + it("produces exactly the raw codeword count for every version/level", () => { + for (const v of [1, 3, 5, 7, 13, 21, 33, 40]) { + for (const ec of ["L", "M", "Q", "H"] as EcLevel[]) { + const data = new Uint8Array(numDataCodewords(v, ec)).fill(0x42); + const out = addEccAndInterleave(data, v, ec); + assert.equal(out.length, numRawDataModulesCodewords(v), `v${v} ${ec}`); + } + } + }); + + it("rejects a wrong-sized data block", () => { + assert.throws(() => addEccAndInterleave(new Uint8Array(3), 5, "M"), QrError); + }); +}); + +describe("SVG rendering", () => { + const qr = encodeQr("https://example.test/login?ticket=abc123"); + + it("wraps the symbol in a quiet zone", () => { + const svg = renderQrSvg(qr, { border: 4 }); + const dim = qr.size + 8; + assert.ok(svg.includes(`viewBox="0 0 ${dim} ${dim}"`), svg.slice(0, 120)); + }); + + it("honours a custom border", () => { + const svg = renderQrSvg(qr, { border: 0 }); + assert.ok(svg.includes(`viewBox="0 0 ${qr.size} ${qr.size}"`)); + }); + + it("emits a light background and a single dark path", () => { + const svg = renderQrSvg(qr); + assert.equal((svg.match(/ { + const svg = renderQrSvg(qr); + const moves = (svg.match(/M\d+ \d+h/g) ?? []).length; + let darkModules = 0; + for (const row of qr.modules) { + for (const m of row) if (m) darkModules++; + } + assert.ok( + moves < darkModules, + `expected run merging: ${moves} runs vs ${darkModules} modules`, + ); + }); + + it("escapes the title", () => { + const svg = renderQrSvg(qr, { title: '登录 & "b"' }); + assert.ok(svg.includes("<a>")); + assert.ok(svg.includes("&")); + assert.ok(svg.includes(""")); + assert.ok(!svg.includes("")); + }); + + it("sets explicit pixel dimensions on the svg element only", () => { + // The background always carries width/height, so inspect the opening + // tag rather than the whole document. + const openTag = (svg: string): string => svg.slice(0, svg.indexOf(">") + 1); + assert.ok( + openTag(renderQrSvg(qr, { pixelSize: 200 })).includes('width="200"'), + ); + assert.ok(!openTag(renderQrSvg(qr)).includes("width=")); + }); + + it("qrSvg is encode + render in one call", () => { + assert.equal(qrSvg("abc"), renderQrSvg(encodeQr("abc"))); + }); + + it("never leaks the payload into the markup", () => { + // The whole point of local rendering: the ticket must not appear anywhere + // a proxy or log could read it as text. + const ticket = "SECRETTICKET123"; + const svg = qrSvg(`https://x.test/q?qrcode=${ticket}`); + assert.ok(!svg.includes(ticket)); + }); +}); diff --git a/apps/api/src/qrcode.ts b/apps/api/src/qrcode.ts new file mode 100644 index 0000000..85e28cc --- /dev/null +++ b/apps/api/src/qrcode.ts @@ -0,0 +1,685 @@ +/** + * Dependency-free QR Code encoder (byte mode) → SVG. + * + * Exists so the iLink login QR is rendered locally instead of by a third party. + * The scan link carries a login ticket, and the previous implementation handed + * it to `api.qrserver.com` as a query parameter — an outbound copy of a + * credential to a service that has no business seeing it. + * + * Hand-rolled rather than pulled from npm on purpose: this ships through the + * OTA channel, which packs `.ts` sources and cannot carry node_modules, so a + * dependency here would force `requiresInstall` on every release. + * + * Structure follows ISO/IEC 18004. The version/ECC tables and the raw-module + * formula are cross-checked in qrcode.test.ts against the spec's published + * byte-mode capacities (v1: 17/14/11/7, v40: 2953/2331/1663/1273), and the + * generated ECC blocks are checked against the Reed–Solomon syndrome property + * rather than against my own generator polynomial. + */ + +export type EcLevel = "L" | "M" | "Q" | "H"; + +/** Format-info value per EC level (spec table, not the L = { L: 1, M: 0, Q: 3, H: 2 }; + +const EC_LEVELS: EcLevel[] = ["L", "M", "Q", "H"]; + +/** ECC codewords per block, indexed [ecLevel][version]; index 0 unused. */ +const ECC_CODEWORDS_PER_BLOCK: Record = { + L: [ + -1, 7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, + 28, 28, 28, 28, 30, 30, 26, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, + ], + M: [ + -1, 10, 16, 26, 18, 24, 16, 18, 22, 22, 26, 30, 22, 22, 24, 24, 28, 28, 26, + 26, 26, 26, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, + ], + Q: [ + -1, 13, 22, 18, 26, 18, 24, 18, 22, 20, 24, 28, 26, 24, 20, 30, 24, 28, 28, + 26, 30, 28, 30, 30, 30, 30, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, + ], + H: [ + -1, 17, 28, 22, 16, 22, 28, 26, 26, 24, 28, 24, 28, 22, 24, 24, 30, 28, 28, + 26, 28, 30, 24, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, + ], +}; + +/** Number of ECC blocks, indexed [ecLevel][version]; index 0 unused. */ +const NUM_ECC_BLOCKS: Record = { + L: [ + -1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 6, 6, 6, 6, 7, 8, 8, 9, 9, 10, + 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 24, 25, + ], + M: [ + -1, 1, 1, 1, 2, 2, 4, 4, 4, 5, 5, 5, 8, 9, 9, 10, 10, 11, 13, 14, 16, 17, + 17, 18, 20, 21, 23, 25, 26, 28, 29, 31, 33, 35, 37, 38, 40, 43, 45, 47, 49, + ], + Q: [ + -1, 1, 1, 2, 2, 4, 4, 6, 6, 8, 8, 8, 10, 12, 16, 12, 17, 16, 18, 21, 20, 23, + 23, 25, 27, 29, 34, 34, 35, 38, 40, 43, 45, 48, 51, 53, 56, 59, 62, 65, 68, + ], + H: [ + -1, 1, 1, 2, 4, 4, 4, 5, 6, 8, 8, 11, 11, 16, 16, 18, 16, 19, 21, 25, 25, + 25, 34, 30, 32, 35, 37, 40, 42, 45, 48, 51, 54, 57, 60, 63, 66, 70, 74, 77, + 81, + ], +}; + +export const MIN_VERSION = 1; +export const MAX_VERSION = 40; + +const PENALTY_N1 = 3; +const PENALTY_N2 = 3; +const PENALTY_N3 = 40; +const PENALTY_N4 = 10; + +export class QrError extends Error { + constructor(message: string) { + super(message); + this.name = "QrError"; + } +} + +// ── GF(256) with the QR primitive polynomial x^8+x^4+x^3+x^2+1 (0x11D) ── + +/** Carry-less multiply then reduce; no lookup tables to get out of sync. */ +export function gfMultiply(x: number, y: number): number { + let z = 0; + for (let i = 7; i >= 0; i--) { + z = (z << 1) ^ ((z >>> 7) * 0x11d); + z ^= ((y >>> i) & 1) * x; + } + return z & 0xff; +} + +/** Coefficients of the RS generator polynomial, minus the leading 1. */ +export function rsComputeDivisor(degree: number): Uint8Array { + if (degree < 1 || degree > 255) { + throw new QrError(`RS degree out of range: ${degree}`); + } + const result = new Uint8Array(degree); + result[degree - 1] = 1; + let root = 1; + for (let i = 0; i < degree; i++) { + for (let j = 0; j < degree; j++) { + result[j] = gfMultiply(result[j]!, root); + if (j + 1 < degree) result[j]! ^= result[j + 1]!; + } + root = gfMultiply(root, 0x02); + } + return result; +} + +export function rsComputeRemainder( + data: Uint8Array | readonly number[], + divisor: Uint8Array, +): Uint8Array { + const result = new Uint8Array(divisor.length); + for (const b of data) { + const factor = b ^ result[0]!; + result.copyWithin(0, 1); + result[result.length - 1] = 0; + for (let i = 0; i < result.length; i++) { + result[i]! ^= gfMultiply(divisor[i]!, factor); + } + } + return result; +} + +// ── Version geometry ── + +/** Modules available for data+ECC, in bits, before block splitting. */ +export function numRawDataModules(version: number): number { + assertVersion(version); + let result = (16 * version + 128) * version + 64; + if (version >= 2) { + const numAlign = Math.floor(version / 7) + 2; + result -= (25 * numAlign - 10) * numAlign - 55; + if (version >= 7) result -= 36; + } + return result; +} + +export function numDataCodewords(version: number, ec: EcLevel): number { + return ( + Math.floor(numRawDataModules(version) / 8) - + ECC_CODEWORDS_PER_BLOCK[ec][version]! * NUM_ECC_BLOCKS[ec][version]! + ); +} + +/** + * Number of ECC blocks the codewords are split across. + * + * Exposed because the block count is not derivable from the totals — several + * (blocks, eccPerBlock) pairs multiply to the same ECC total, so anything + * reading a symbol back needs the table value. + */ +export function numEccBlocks(version: number, ec: EcLevel): number { + assertVersion(version); + return NUM_ECC_BLOCKS[ec][version]!; +} + +/** Character-count field width for byte mode. */ +function charCountBits(version: number): number { + return version <= 9 ? 8 : 16; +} + +/** Max payload bytes for byte mode at this version + EC level. */ +export function byteCapacity(version: number, ec: EcLevel): number { + const bits = numDataCodewords(version, ec) * 8 - 4 - charCountBits(version); + return Math.max(0, Math.floor(bits / 8)); +} + +function assertVersion(version: number): void { + if ( + !Number.isInteger(version) || + version < MIN_VERSION || + version > MAX_VERSION + ) { + throw new QrError(`version out of range: ${version}`); + } +} + +export function alignmentPatternPositions(version: number): number[] { + assertVersion(version); + if (version === 1) return []; + const size = version * 4 + 17; + const numAlign = Math.floor(version / 7) + 2; + const step = + version === 32 + ? 26 + : Math.ceil((version * 4 + 4) / (numAlign * 2 - 2)) * 2; + const result = [6]; + for (let pos = size - 7; result.length < numAlign; pos -= step) { + result.splice(1, 0, pos); + } + return result; +} + +/** Smallest version that fits `byteLen` payload bytes, or null if none does. */ +export function pickVersion( + byteLen: number, + ec: EcLevel, + minVersion = MIN_VERSION, +): number | null { + for ( + let v = Math.max(MIN_VERSION, minVersion); + v <= MAX_VERSION; + v++ + ) { + if (byteLen <= byteCapacity(v, ec)) return v; + } + return null; +} + +// ── Bit buffer ── + +class BitBuffer { + readonly bits: number[] = []; + + append(value: number, len: number): void { + for (let i = len - 1; i >= 0; i--) { + this.bits.push((value >>> i) & 1); + } + } +} + +// ── Encoder ── + +export interface QrOptions { + /** Error correction level; M is the usual choice for scanning off a screen. */ + ec?: EcLevel; + /** Force at least this version (never lowers the auto-picked one). */ + minVersion?: number; +} + +export interface QrCode { + version: number; + ec: EcLevel; + mask: number; + size: number; + /** Row-major; true = dark */ + modules: boolean[][]; +} + +export function encodeQr(text: string, opts: QrOptions = {}): QrCode { + const ec = opts.ec ?? "M"; + if (!EC_LEVELS.includes(ec)) throw new QrError(`bad EC level: ${ec}`); + + const payload = Buffer.from(text, "utf8"); + const version = pickVersion(payload.length, ec, opts.minVersion ?? 1); + if (version === null) { + throw new QrError( + `data too long: ${payload.length} bytes exceeds ${byteCapacity( + MAX_VERSION, + ec, + )} for EC ${ec}`, + ); + } + + const dataCodewords = buildDataCodewords(payload, version, ec); + const finalCodewords = addEccAndInterleave(dataCodewords, version, ec); + return drawSymbol(finalCodewords, version, ec); +} + +/** Mode + count + payload + terminator + pad, to exactly dataCodewords bytes. */ +function buildDataCodewords( + payload: Buffer, + version: number, + ec: EcLevel, +): Uint8Array { + const capacityBits = numDataCodewords(version, ec) * 8; + const bb = new BitBuffer(); + bb.append(0b0100, 4); // byte mode + bb.append(payload.length, charCountBits(version)); + for (const b of payload) bb.append(b, 8); + + if (bb.bits.length > capacityBits) { + throw new QrError("internal: payload overflowed the chosen version"); + } + // Terminator: up to 4 zero bits, only as many as fit. + bb.append(0, Math.min(4, capacityBits - bb.bits.length)); + // Pad to a byte boundary, then alternate the spec's pad bytes. + bb.append(0, (8 - (bb.bits.length % 8)) % 8); + + const out = new Uint8Array(capacityBits / 8); + for (let i = 0; i < bb.bits.length; i++) { + if (bb.bits[i]) out[i >>> 3]! |= 0x80 >>> (i & 7); + } + for (let i = bb.bits.length / 8, pad = 0xec; i < out.length; i++) { + out[i] = pad; + pad = pad === 0xec ? 0x11 : 0xec; + } + return out; +} + +/** Split into blocks, append RS ECC per block, then interleave per the spec. */ +export function addEccAndInterleave( + data: Uint8Array, + version: number, + ec: EcLevel, +): Uint8Array { + const numBlocks = NUM_ECC_BLOCKS[ec][version]!; + const blockEccLen = ECC_CODEWORDS_PER_BLOCK[ec][version]!; + const rawCodewords = Math.floor(numRawDataModules(version) / 8); + if (data.length !== numDataCodewords(version, ec)) { + throw new QrError("internal: data codeword count mismatch"); + } + + const numShortBlocks = numBlocks - (rawCodewords % numBlocks); + const shortBlockLen = Math.floor(rawCodewords / numBlocks); + const divisor = rsComputeDivisor(blockEccLen); + + const blocks: number[][] = []; + for (let i = 0, k = 0; i < numBlocks; i++) { + const datLen = + shortBlockLen - blockEccLen + (i < numShortBlocks ? 0 : 1); + const dat = Array.from(data.subarray(k, k + datLen)); + k += datLen; + const ecc = rsComputeRemainder(dat, divisor); + // Short blocks get a placeholder so column indexing lines up below; it is + // skipped during interleaving and never reaches the symbol. + if (i < numShortBlocks) dat.push(0); + blocks.push([...dat, ...ecc]); + } + + const result: number[] = []; + for (let i = 0; i < blocks[0]!.length; i++) { + blocks.forEach((block, j) => { + if (i !== shortBlockLen - blockEccLen || j >= numShortBlocks) { + result.push(block[i]!); + } + }); + } + if (result.length !== rawCodewords) { + throw new QrError("internal: interleave produced the wrong length"); + } + return Uint8Array.from(result); +} + +// ── Symbol drawing ── + +function drawSymbol( + codewords: Uint8Array, + version: number, + ec: EcLevel, +): QrCode { + const size = version * 4 + 17; + const modules: boolean[][] = Array.from({ length: size }, () => + new Array(size).fill(false), + ); + const isFunction: boolean[][] = Array.from({ length: size }, () => + new Array(size).fill(false), + ); + + const set = (x: number, y: number, dark: boolean): void => { + modules[y]![x] = dark; + isFunction[y]![x] = true; + }; + + // Timing patterns + for (let i = 0; i < size; i++) { + set(6, i, i % 2 === 0); + set(i, 6, i % 2 === 0); + } + // Finder patterns + separators, anchored at the three corners + drawFinder(set, size, 3, 3); + drawFinder(set, size, size - 4, 3); + drawFinder(set, size, 3, size - 4); + + // Alignment patterns, skipping the three finder corners + const aligns = alignmentPatternPositions(version); + const last = aligns.length - 1; + for (let i = 0; i <= last; i++) { + for (let j = 0; j <= last; j++) { + const corner = + (i === 0 && j === 0) || (i === 0 && j === last) || (i === last && j === 0); + if (corner) continue; + for (let dy = -2; dy <= 2; dy++) { + for (let dx = -2; dx <= 2; dx++) { + set( + aligns[j]! + dx, + aligns[i]! + dy, + Math.max(Math.abs(dx), Math.abs(dy)) !== 1, + ); + } + } + } + } + + drawVersionInfo(set, size, version); + // Reserve the format areas now (mask 0); rewritten once the mask is chosen. + drawFormatBits(set, size, ec, 0); + + drawCodewords(modules, isFunction, size, codewords); + + let bestMask = 0; + let bestPenalty = Number.POSITIVE_INFINITY; + for (let mask = 0; mask < 8; mask++) { + applyMask(modules, isFunction, size, mask); + drawFormatBits(set, size, ec, mask); + const penalty = penaltyScore(modules, size); + if (penalty < bestPenalty) { + bestPenalty = penalty; + bestMask = mask; + } + applyMask(modules, isFunction, size, mask); // XOR is its own inverse + } + applyMask(modules, isFunction, size, bestMask); + drawFormatBits(set, size, ec, bestMask); + + return { version, ec, mask: bestMask, size, modules }; +} + +type SetFn = (x: number, y: number, dark: boolean) => void; + +/** 7x7 finder centred at (cx, cy) plus its light separator ring. */ +function drawFinder(set: SetFn, size: number, cx: number, cy: number): void { + for (let dy = -4; dy <= 4; dy++) { + for (let dx = -4; dx <= 4; dx++) { + const x = cx + dx; + const y = cy + dy; + if (x < 0 || x >= size || y < 0 || y >= size) continue; + const dist = Math.max(Math.abs(dx), Math.abs(dy)); + set(x, y, dist !== 2 && dist !== 4); + } + } +} + +function drawFormatBits( + set: SetFn, + size: number, + ec: EcLevel, + mask: number, +): void { + const data = (EC_FORMAT_BITS[ec] << 3) | mask; + let rem = data; + for (let i = 0; i < 10; i++) rem = (rem << 1) ^ ((rem >>> 9) * 0x537); + const bits = (((data << 10) | rem) ^ 0x5412) & 0x7fff; + const bit = (i: number): boolean => ((bits >>> i) & 1) !== 0; + + // Copy 1 — around the top-left finder + for (let i = 0; i <= 5; i++) set(8, i, bit(i)); + set(8, 7, bit(6)); + set(8, 8, bit(7)); + set(7, 8, bit(8)); + for (let i = 9; i < 15; i++) set(14 - i, 8, bit(i)); + + // Copy 2 — split across the other two finders + for (let i = 0; i < 8; i++) set(size - 1 - i, 8, bit(i)); + for (let i = 8; i < 15; i++) set(8, size - 15 + i, bit(i)); + set(8, size - 8, true); // always-dark module +} + +function drawVersionInfo(set: SetFn, size: number, version: number): void { + if (version < 7) return; + let rem = version; + for (let i = 0; i < 12; i++) rem = (rem << 1) ^ ((rem >>> 11) * 0x1f25); + const bits = ((version << 12) | rem) & 0x3ffff; + for (let i = 0; i < 18; i++) { + const dark = ((bits >>> i) & 1) !== 0; + const a = size - 11 + (i % 3); + const b = Math.floor(i / 3); + set(a, b, dark); + set(b, a, dark); + } +} + +/** Zigzag fill of the two-column strips, skipping function modules. */ +function drawCodewords( + modules: boolean[][], + isFunction: boolean[][], + size: number, + codewords: Uint8Array, +): void { + let i = 0; + const totalBits = codewords.length * 8; + for (let right = size - 1; right >= 1; right -= 2) { + // Column 6 is the vertical timing pattern — the strip shifts left past it. + if (right === 6) right = 5; + for (let vert = 0; vert < size; vert++) { + for (let j = 0; j < 2; j++) { + const x = right - j; + const upward = ((right + 1) & 2) === 0; + const y = upward ? size - 1 - vert : vert; + if (!isFunction[y]![x] && i < totalBits) { + modules[y]![x] = ((codewords[i >>> 3]! >>> (7 - (i & 7))) & 1) !== 0; + i++; + } + } + } + } + if (i !== totalBits) { + throw new QrError( + `internal: placed ${i} of ${totalBits} data bits`, + ); + } +} + +export function maskPredicate(mask: number, x: number, y: number): boolean { + switch (mask) { + case 0: + return (x + y) % 2 === 0; + case 1: + return y % 2 === 0; + case 2: + return x % 3 === 0; + case 3: + return (x + y) % 3 === 0; + case 4: + return (Math.floor(x / 3) + Math.floor(y / 2)) % 2 === 0; + case 5: + return ((x * y) % 2) + ((x * y) % 3) === 0; + case 6: + return (((x * y) % 2) + ((x * y) % 3)) % 2 === 0; + case 7: + return (((x + y) % 2) + ((x * y) % 3)) % 2 === 0; + default: + throw new QrError(`bad mask: ${mask}`); + } +} + +function applyMask( + modules: boolean[][], + isFunction: boolean[][], + size: number, + mask: number, +): void { + for (let y = 0; y < size; y++) { + for (let x = 0; x < size; x++) { + if (!isFunction[y]![x] && maskPredicate(mask, x, y)) { + modules[y]![x] = !modules[y]![x]; + } + } + } +} + +const FINDER_RUN = [true, false, true, true, true, false, true]; + +/** Spec penalty rules N1–N4; drives mask selection only, never correctness. */ +export function penaltyScore(modules: boolean[][], size: number): number { + let penalty = 0; + + const lines: boolean[][] = []; + for (let y = 0; y < size; y++) lines.push(modules[y]!.slice()); + for (let x = 0; x < size; x++) { + lines.push(Array.from({ length: size }, (_, y) => modules[y]![x]!)); + } + + for (const line of lines) { + // N1: runs of 5+ + let runLen = 1; + for (let i = 1; i <= line.length; i++) { + if (i < line.length && line[i] === line[i - 1]) { + runLen++; + continue; + } + if (runLen >= 5) penalty += PENALTY_N1 + (runLen - 5); + runLen = 1; + } + // N3: finder-like 1:1:3:1:1 with 4 light modules on either side + for (let i = 0; i + 7 <= line.length; i++) { + let match = true; + for (let k = 0; k < 7; k++) { + if (line[i + k] !== FINDER_RUN[k]) { + match = false; + break; + } + } + if (!match) continue; + const beforeClear = + i >= 4 && line.slice(i - 4, i).every((v) => !v); + const afterClear = + i + 11 <= line.length && line.slice(i + 7, i + 11).every((v) => !v); + if (beforeClear || afterClear) penalty += PENALTY_N3; + } + } + + // N2: 2x2 blocks of one colour + for (let y = 0; y + 1 < size; y++) { + for (let x = 0; x + 1 < size; x++) { + const c = modules[y]![x]; + if ( + c === modules[y]![x + 1] && + c === modules[y + 1]![x] && + c === modules[y + 1]![x + 1] + ) { + penalty += PENALTY_N2; + } + } + } + + // N4: deviation of dark proportion from 50%, in 5% steps + let dark = 0; + for (let y = 0; y < size; y++) { + for (let x = 0; x < size; x++) if (modules[y]![x]) dark++; + } + const total = size * size; + const k = Math.floor(Math.abs((dark * 100) / total - 50) / 5); + penalty += k * PENALTY_N4; + + return penalty; +} + +// ── SVG rendering ── + +export interface SvgOptions { + /** Quiet zone in modules; the spec requires 4 and scanners rely on it. */ + border?: number; + /** Rendered pixel size of the whole square (viewBox stays in modules). */ + pixelSize?: number; + dark?: string; + light?: string; + title?: string; +} + +/** + * Render to SVG. Horizontal runs are merged into one path so the markup stays + * a few KB instead of one element per module. + */ +export function renderQrSvg(qr: QrCode, opts: SvgOptions = {}): string { + const border = Math.max(0, Math.floor(opts.border ?? 4)); + const dark = opts.dark ?? "#000000"; + const light = opts.light ?? "#ffffff"; + const dim = qr.size + border * 2; + + const segments: string[] = []; + for (let y = 0; y < qr.size; y++) { + let x = 0; + while (x < qr.size) { + if (!qr.modules[y]![x]) { + x++; + continue; + } + let run = 1; + while (x + run < qr.size && qr.modules[y]![x + run]) run++; + segments.push(`M${x + border} ${y + border}h${run}v1h-${run}z`); + x += run; + } + } + + const sizeAttrs = + opts.pixelSize && opts.pixelSize > 0 + ? ` width="${Math.round(opts.pixelSize)}" height="${Math.round( + opts.pixelSize, + )}"` + : ""; + const titleEl = opts.title + ? `${escapeXml(opts.title)}` + : ""; + + return ( + `` + + titleEl + + `` + + `` + + `` + ); +} + +function escapeXml(s: string): string { + return s.replace(/[&<>"']/g, (c) => + c === "&" + ? "&" + : c === "<" + ? "<" + : c === ">" + ? ">" + : c === '"' + ? """ + : "'", + ); +} + +/** One-shot: text → SVG markup. */ +export function qrSvg( + text: string, + opts: QrOptions & SvgOptions = {}, +): string { + return renderQrSvg(encodeQr(text, opts), opts); +} diff --git a/apps/api/src/rate-limit.test.ts b/apps/api/src/rate-limit.test.ts new file mode 100644 index 0000000..1e3a3c6 --- /dev/null +++ b/apps/api/src/rate-limit.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { RateLimiter } from "./rate-limit.js"; + +describe("RateLimiter", () => { + it("allows up to max within window", () => { + const rl = new RateLimiter(3, 60_000); + assert.equal(rl.tryTake("a", 1000), true); + assert.equal(rl.tryTake("a", 1001), true); + assert.equal(rl.tryTake("a", 1002), true); + assert.equal(rl.tryTake("a", 1003), false); + }); + + it("resets after window", () => { + const rl = new RateLimiter(1, 1000); + assert.equal(rl.tryTake("b", 0), true); + assert.equal(rl.tryTake("b", 500), false); + assert.equal(rl.tryTake("b", 1001), true); + }); + + it("still denies across many distinct keys", () => { + const rl = new RateLimiter(2, 60_000, 128); + for (let i = 0; i < 500; i++) rl.tryTake(`k${i}`, 1000); + // The key we keep hitting must never be evicted out from under itself + assert.equal(rl.tryTake("hot", 1000), true); + assert.equal(rl.tryTake("hot", 1001), true); + assert.equal(rl.tryTake("hot", 1002), false); + }); + + it("bounds the key map under unbounded distinct keys", () => { + const rl = new RateLimiter(5, 60_000, 128); + for (let i = 0; i < 100_000; i++) rl.tryTake(`ip-${i}`, 1000); + assert.ok( + rl.size <= 128, + `expected bounded map, got ${rl.size}`, + ); + }); + + it("drops keys whose window went empty", () => { + const rl = new RateLimiter(5, 1000); + rl.tryTake("gone", 0); + assert.equal(rl.size, 1); + // Same key, long after the window — re-inserted with a single hit + assert.equal(rl.tryTake("gone", 10_000), true); + assert.equal(rl.remaining("gone", 10_000), 4); + }); +}); diff --git a/apps/api/src/rate-limit.ts b/apps/api/src/rate-limit.ts new file mode 100644 index 0000000..d8bf6a9 --- /dev/null +++ b/apps/api/src/rate-limit.ts @@ -0,0 +1,81 @@ +/** + * Simple sliding-window rate limiter (in-memory, per process). + * + * Keys are attacker-influenced (client IP on the public /cdn/s route, username + * on login), so the map is bounded. Evicting a bucket is always safe in this + * direction: the key simply gets a fresh window, it can never turn an allow + * into a deny. + */ + +export class RateLimiter { + private hits = new Map(); + private readonly maxKeys: number; + + constructor( + private max: number, + private windowMs: number, + maxKeys = 20_000, + ) { + this.maxKeys = Math.max(64, maxKeys); + } + + /** + * Runtime settings reload. Existing buckets are kept: raising the cap frees + * callers immediately, lowering it applies from the next request onward. + */ + setLimits(max: number, windowMs = this.windowMs): void { + this.max = Math.max(1, max); + this.windowMs = Math.max(1_000, windowMs); + } + + /** Drop timestamps that fell out of the window, in place (no re-allocation). */ + private prune(list: number[], cutoff: number): number[] { + let i = 0; + while (i < list.length && list[i]! <= cutoff) i++; + if (i > 0) list.splice(0, i); + return list; + } + + private evictIfFull(): void { + if (this.hits.size < this.maxKeys) return; + // Drop the oldest ~10% by insertion order + const n = Math.ceil(this.maxKeys * 0.1); + let i = 0; + for (const k of this.hits.keys()) { + this.hits.delete(k); + if (++i >= n) break; + } + } + + /** Returns true if the key is allowed; records a hit when allowed. */ + tryTake(key: string, now = Date.now()): boolean { + const cutoff = now - this.windowMs; + const existing = this.hits.get(key); + const prev = existing ? this.prune(existing, cutoff) : []; + if (prev.length >= this.max) { + // `prev` is the same array instance already in the map + return false; + } + if (!prev.length) { + // Window went empty — forget the key so idle callers stop occupying it, + // then re-insert at the tail so eviction order stays LRU-ish. + this.hits.delete(key); + this.evictIfFull(); + } + prev.push(now); + this.hits.set(key, prev); + return true; + } + + remaining(key: string, now = Date.now()): number { + const cutoff = now - this.windowMs; + const existing = this.hits.get(key); + const prev = existing ? this.prune(existing, cutoff) : []; + return Math.max(0, this.max - prev.length); + } + + /** Tracked key count — for tests / diagnostics. */ + get size(): number { + return this.hits.size; + } +} diff --git a/apps/api/src/request-log.test.ts b/apps/api/src/request-log.test.ts new file mode 100644 index 0000000..7fb3b97 --- /dev/null +++ b/apps/api/src/request-log.test.ts @@ -0,0 +1,178 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { LOG_LEVELS, resolveLogLevel } from "./config.js"; +import { + describeRequest, + isQuietPath, + isStreamingPath, + logPath, + requestLogLevel, +} from "./request-log.js"; + +const base = { + method: "GET", + url: "/api/v1/me/bots", + status: 200, + elapsedMs: 12, + slowMs: 1000, +}; + +describe("resolveLogLevel", () => { + it("accepts every pino level", () => { + for (const level of LOG_LEVELS) { + assert.equal(resolveLogLevel(level), level); + } + }); + + it("normalises case and whitespace", () => { + assert.equal(resolveLogLevel(" DEBUG "), "debug"); + }); + + it("falls back to info rather than letting pino throw at boot", () => { + for (const bad of ["verbose", "", " ", undefined, "10", "critical"]) { + assert.equal(resolveLogLevel(bad), "info", String(bad)); + } + }); +}); + +describe("logPath", () => { + it("keeps a plain path", () => { + assert.equal(logPath("/api/v1/auth/me"), "/api/v1/auth/me"); + }); + + it("drops the query string so OAuth credentials never reach the log", () => { + assert.equal( + logPath("/api/v1/auth/callback?code=SECRET&state=ALSOSECRET"), + "/api/v1/auth/callback", + ); + }); + + it("never returns empty", () => { + assert.equal(logPath(""), "/"); + assert.equal(logPath("?a=1"), "/"); + }); +}); + +describe("requestLogLevel", () => { + it("escalates on server errors", () => { + assert.equal(requestLogLevel(500, 5, 1000), "error"); + assert.equal(requestLogLevel(503, 5, 1000), "error"); + }); + + it("warns on client errors", () => { + assert.equal(requestLogLevel(400, 5, 1000), "warn"); + assert.equal(requestLogLevel(404, 5, 1000), "warn"); + assert.equal(requestLogLevel(429, 5, 1000), "warn"); + }); + + it("warns on a slow success", () => { + assert.equal(requestLogLevel(200, 1001, 1000), "warn"); + assert.equal(requestLogLevel(200, 1000, 1000), "info"); + }); + + it("treats redirects and 304 as ordinary", () => { + assert.equal(requestLogLevel(302, 5, 1000), "info"); + assert.equal(requestLogLevel(304, 5, 1000), "info"); + }); +}); + +describe("streaming paths", () => { + it("recognises the admin activity stream", () => { + assert.equal(isStreamingPath("/api/v1/admin/stream"), true); + assert.equal(isStreamingPath("/api/v1/admin/stream/recent"), false); + assert.equal(isStreamingPath("/api/v1/me/bots"), false); + }); + + it("does not escalate a held-open stream to warn on duration", () => { + // The SSE route hijacks the reply and stays open for as long as the + // dashboard is on screen; elapsed time is the viewer's dwell time. + assert.equal(requestLogLevel(200, 600_000, 1000, true), "info"); + // Real failures still escalate. + assert.equal(requestLogLevel(500, 600_000, 1000, true), "error"); + assert.equal(requestLogLevel(403, 5, 1000, true), "warn"); + }); + + it("logs a long SSE session at info end to end", () => { + const line = describeRequest({ + method: "GET", + url: "/api/v1/admin/stream?types=message", + status: 200, + elapsedMs: 8 * 60_000, + slowMs: 1000, + }); + assert.equal(line?.level, "info"); + assert.equal(line?.path, "/api/v1/admin/stream"); + }); +}); + +describe("isQuietPath", () => { + it("covers both health endpoints and nothing else", () => { + assert.equal(isQuietPath("/health"), true); + assert.equal(isQuietPath("/health/ready"), true); + assert.equal(isQuietPath("/healthz"), false); + assert.equal(isQuietPath("/"), false); + assert.equal(isQuietPath("/api/v1/auth/me"), false); + }); +}); + +describe("describeRequest", () => { + it("emits fields for a normal request", () => { + assert.deepEqual(describeRequest({ ...base, socketIp: "10.0.0.5" }), { + level: "info", + method: "GET", + path: "/api/v1/me/bots", + status: 200, + ms: 12, + ip: "10.0.0.5", + }); + }); + + it("stays quiet for a healthy probe", () => { + assert.equal(describeRequest({ ...base, url: "/health" }), null); + assert.equal(describeRequest({ ...base, url: "/health/ready" }), null); + }); + + it("logs a failing probe — the only time anyone reads them", () => { + const line = describeRequest({ + ...base, + url: "/health/ready", + status: 503, + }); + assert.ok(line); + assert.equal(line.level, "error"); + assert.equal(line.path, "/health/ready"); + }); + + it("prefers the Cloudflare header over the socket address", () => { + const line = describeRequest({ + ...base, + cfConnectingIp: " 203.0.113.9 ", + socketIp: "10.0.0.5", + }); + assert.equal(line?.ip, "203.0.113.9"); + }); + + it("falls back through socket IP to unknown", () => { + assert.equal(describeRequest({ ...base, socketIp: "10.0.0.5" })?.ip, "10.0.0.5"); + assert.equal(describeRequest({ ...base })?.ip, "unknown"); + assert.equal( + describeRequest({ ...base, cfConnectingIp: " ", socketIp: undefined })?.ip, + "unknown", + ); + }); + + it("rounds and floors latency", () => { + assert.equal(describeRequest({ ...base, elapsedMs: 12.6 })?.ms, 13); + assert.equal(describeRequest({ ...base, elapsedMs: -1 })?.ms, 0); + }); + + it("never carries a query string into the fields", () => { + const line = describeRequest({ + ...base, + url: "/api/v1/auth/callback?code=SECRET", + status: 302, + }); + assert.equal(line?.path, "/api/v1/auth/callback"); + assert.ok(!JSON.stringify(line).includes("SECRET")); + }); +}); diff --git a/apps/api/src/request-log.ts b/apps/api/src/request-log.ts new file mode 100644 index 0000000..4226cf3 --- /dev/null +++ b/apps/api/src/request-log.ts @@ -0,0 +1,101 @@ +/** + * Decides what a completed request should log. + * + * Split out of the onResponse hook so the rules — which paths stay quiet, when + * a 200 still deserves a warning, and above all that query strings never reach + * the log — are unit-testable instead of buried in a closure. + */ + +export type RequestLogLevel = "info" | "warn" | "error"; + +export interface RequestLogFields { + level: RequestLogLevel; + method: string; + path: string; + status: number; + ms: number; + ip: string; +} + +export interface RequestLogInput { + method: string; + /** Raw request URL, query string included */ + url: string; + status: number; + elapsedMs: number; + cfConnectingIp?: string | undefined; + socketIp?: string | undefined; + /** Successful requests slower than this log at warn */ + slowMs: number; +} + +/** + * Probe endpoints. Docker's HEALTHCHECK and the Cloudflare LB hit these every + * few seconds, so a successful probe logs nothing — a failing one still does, + * which is the only time anyone wants to read them. + */ +const QUIET_PATHS = new Set(["/health", "/health/ready"]); + +/** + * Long-lived responses, where elapsed time measures how long a client stayed + * connected rather than how slow we were. + * + * The admin activity stream calls `reply.hijack()` and holds the socket open for + * as long as the dashboard is on screen. Fastify still fires onResponse when the + * raw socket finishes, so without this every dashboard visit would close with a + * "slow request" warning that means nothing. + */ +const STREAMING_PATHS = new Set(["/api/v1/admin/stream"]); + +export function isStreamingPath(path: string): boolean { + return STREAMING_PATHS.has(path); +} + +/** + * Strip the query string. + * + * `/api/v1/auth/callback?code=…&state=…` carries single-use OAuth credentials; + * writing the full URL would persist them to whatever collects stdout. + */ +export function logPath(url: string): string { + const path = url.split("?")[0] || "/"; + return path; +} + +export function isQuietPath(path: string): boolean { + return QUIET_PATHS.has(path); +} + +export function requestLogLevel( + status: number, + ms: number, + slowMs: number, + streaming = false, +): RequestLogLevel { + if (status >= 500) return "error"; + if (status >= 400) return "warn"; + // Duration is meaningless for a held-open stream — never escalate on it. + if (streaming) return "info"; + return ms > slowMs ? "warn" : "info"; +} + +/** Fields to log, or null when this request should stay quiet. */ +export function describeRequest( + input: RequestLogInput, +): RequestLogFields | null { + const path = logPath(input.url); + const status = input.status; + if (isQuietPath(path) && status < 400) return null; + + const ms = Math.max(0, Math.round(input.elapsedMs)); + return { + level: requestLogLevel(status, ms, input.slowMs, isStreamingPath(path)), + method: input.method, + path, + status, + ms, + // trustProxy is deliberately off, so only Cloudflare's header is trusted + // here — a client-supplied X-Forwarded-For must not shape our logs either. + ip: input.cfConnectingIp?.trim() || input.socketIp || "unknown", + }; +} diff --git a/apps/api/src/routes.ts b/apps/api/src/routes.ts new file mode 100644 index 0000000..3c7da57 --- /dev/null +++ b/apps/api/src/routes.ts @@ -0,0 +1,4899 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { + addPersonaToLibrary, + addStickerToLibrary, + approvePeer, + approveSticker, + clearMemories, + clearMessages, + clearPrimaryBind, + createBindCode, + blockUser, + deleteMemory, + countPendingStickers, + createAppSession, + createLocalUser, + createPersona, + createInviteCode, + consumeInviteCode, + peekInviteCode, + listPendingInvites, + revokeInviteCode, + getInviteSettings, + setInviteSettings, + getInviteQuotaStatus, + forkPersona, + createSticker, + deleteBotAccount, + deleteUserAccount, + deleteSticker, + destroyAppSession, + destroyAllSessionsForUser, + doctorSnapshot, + ensureStickerContentHash, + getAppSession, + getAssignmentsMany, + getBindByUser, + getBotAccount, + getBotAccountsByIds, + getContextToken, + getPersona, + getPersonaBySlug, + getPersonaLibraryIdSet, + getPersonasByIds, + getPublishedPrompt, + getPublishedPromptsMany, + getSticker, + getStickerBlob, + getStickerBySlug, + getStickerLibraryIdSet, + getUsageDayStats, + dayKey, + dayKeyOffset, + getUser, + getUserByUsername, + getUsersByIds, + isInPersonaLibrary, + isInPersonaLibraryLocal, + isInStickerLibrary, + isValidStickerSlug, + listAuditLogs, + listBlockedUserIds, + listBotAccounts, + hasBotCredentials, + hasBotCredentialsMany, + listBotsByOwner, + countBotsByOwners, + countUserMessages, + listMemories, + listMemoriesMany, + listPeers, + listPeersForBots, + listPersonas, + listRecentMessages, + listPersonasByOwner, + listStickers, + listStickersByOwner, + listUserPersonaLibrary, + listUserStickerLibrary, + countUsers, + listUsers, + isSuperAdmin, + resolveSuperAdminId, + peerStatsByBots, + publishPersonaVersion, + rejectSticker, + removePersonaFromLibrary, + removeStickerFromLibrary, + replaceStickerBlob, + restorePersona, + restoreSticker, + setDefaultPersona, + saveOauthState, + searchPublicPersonas, + searchPublicStickers, + seedPersonas, + setAssignment, + setUserAdmin, + setUserBanned, + isUserBanned, + userPublicFields, + validateLocalUsername, + hashPassword, + verifyPassword, + assertPasswordPolicy, + softDeletePersona, + softDeleteSticker, + takeOauthState, + setBotStatus, + setPeerProactiveEnabled, + unblockUser, + updateBotDisplayName, + updateBotProactiveSettings, + updatePersonaMeta, + updateStickerMeta, + userCanUsePersona, + upsertUser, + writeAudit, + cancelBroadcastJob, + createBroadcastJob, + getBroadcastJob, + listBroadcastJobs, + listBotSendTargets, + previewBroadcast, + forceOfflineWorker, + clearWorkerFence, + listWorkerFences, + listLeasedBots, + publishWorkerWake, + listWorkerWeights, + setWorkerWeight, + clearWorkerWeight, + pruneWorkerWeights, + parseWorkerWeightInput, + DEFAULT_WORKER_WEIGHT, + MIN_WORKER_WEIGHT, + MAX_WORKER_WEIGHT, + listPollableBotIds, + getCurrentRelease, + getReleaseMeta, + listReleaseVersions, + publishRelease, + setCurrentRelease, + buildReleaseMeta, + putBlobChunks, + blobExists, + enqueueWorkerUpdate, + getWorkerUpdateStatus, + getWorkerUpdateStatuses, + releaseSummary, + sha256Buffer, + type BroadcastScope, + type BroadcastTarget, + type Db, + type Persona, + type Sticker, + type User, + type ReleaseFileEntry, + personaHeatScore, + createLlmProvider, + updateLlmProvider, + deleteLlmProvider, + listLlmProvidersByOwner, + getLlmProvider, + toPublicProvider, + getPublishedGraph, +} from "@wechat-ai/db"; +import { + mergeBotProactiveConfig, + TryChatError, + TryChatService, + createDefaultChatflowGraph, + validateChatflowGraph, + ChatflowError, + type ChatService, +} from "@wechat-ai/core"; +import { probeToolsHealth } from "@wechat-ai/llm"; +import type { BotWorkerManager } from "./worker.js"; +import type { BotLoginSessionManager } from "./bot-login-sessions.js"; +import type { AppConfig } from "./config.js"; +import { + buildAuthorizeUrl, + exchangeCode, + fetchUserInfo, + loadLinuxDoConfig, + newOAuthState, +} from "./oauth-linuxdo.js"; +import { + decodeBase64Image, + makeStickerFileName, +} from "./sticker-store.js"; +import { + assertSafeStickerImage, + StickerSecurityError, +} from "./sticker-security.js"; +import { + CC_AUTH_CONFIG, + CC_CDN_STICKER, + CC_NO_STORE, + CC_PRIVATE_NO_STORE, + CC_PRIVATE_QR, + CC_PRIVATE_STICKER, + CDN_AUTH_CONFIG, + CDN_CDN_STICKER, + etagFromHash, + ifNoneMatchHits, + setPublicCache, +} from "./cache-headers.js"; +import { qrSvg } from "./qrcode.js"; +import { RateLimiter } from "./rate-limit.js"; +import { RuntimeSettingsUnavailableError } from "./runtime-config.js"; + + +export interface RouteContext { + db: Db; + chat: ChatService; + tryChat: TryChatService; + worker: BotWorkerManager; + loginSessions: BotLoginSessionManager; + cfg: AppConfig; + /** Admin live activity stream (optional when DATA_STREAM_ENABLED=false) */ + activityBus?: import("./activity-stream.js").ActivityBus | null; + /** Redis-backed runtime config overrides (env supplies the defaults) */ + settings?: import("./runtime-config.js").RuntimeConfigManager | null; +} + +function personaPublicDto( + p: Persona, + extra?: Record, +): Record { + const forkedFrom = + p.forked_from_id + ? { + id: p.forked_from_id, + slug: p.forked_from_slug ?? null, + displayName: p.forked_from_name ?? null, + } + : null; + return { + id: p.id, + slug: p.slug, + displayName: p.display_name, + description: p.description, + tags: p.tags, + visibility: p.visibility, + ownerUserId: p.owner_user_id, + useCount: p.use_count || 0, + assignCount: p.assign_count || 0, + forkCount: p.fork_count || 0, + heatScore: personaHeatScore(p), + forkedFrom, + mode: p.mode === "chatflow" ? "chatflow" : "prompt", + llmProviderId: p.llm_provider_id ?? null, + webSearchEnabled: Boolean(p.web_search_enabled), + updatedAt: p.updated_at, + ...extra, + }; +} + +function parseCookie(header: string | undefined): Record { + const out: Record = {}; + if (!header) return out; + for (const part of header.split(";")) { + const i = part.indexOf("="); + if (i < 0) continue; + const k = part.slice(0, i).trim(); + const v = part.slice(i + 1).trim(); + out[k] = decodeURIComponent(v); + } + return out; +} + +function setSessionCookie( + reply: FastifyReply, + cfg: AppConfig, + sid: string, +): void { + const maxAge = 7 * 24 * 3600; + const secure = cfg.cookieSecure ? "; Secure" : ""; + reply.header( + "Set-Cookie", + `${cfg.sessionCookieName}=${encodeURIComponent(sid)}; Path=/; HttpOnly; SameSite=Lax; Max-Age=${maxAge}${secure}`, + ); +} + +function clearSessionCookie(reply: FastifyReply, cfg: AppConfig): void { + reply.header( + "Set-Cookie", + `${cfg.sessionCookieName}=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0`, + ); +} + +async function requireUser( + req: FastifyRequest, + reply: FastifyReply, + ctx: RouteContext, +): Promise { + const cookies = parseCookie(req.headers.cookie); + const sid = cookies[ctx.cfg.sessionCookieName]; + if (!sid) { + reply.code(401).send({ error: "login_required" }); + return null; + } + const sess = await getAppSession(ctx.db, sid); + if (!sess) { + reply.code(401).send({ error: "session_expired" }); + return null; + } + const user = await getUser(ctx.db, sess.userId); + if (!user) { + reply.code(401).send({ error: "user_not_found" }); + return null; + } + if (isUserBanned(user)) { + await destroyAppSession(ctx.db, sid); + clearSessionCookie(reply, ctx.cfg); + reply.code(403).send({ + error: "user_banned", + message: user.banned_reason || "账号已被封禁", + }); + return null; + } + return user; +} + +function clientIp(req: FastifyRequest): string { + return ( + (req.headers["cf-connecting-ip"] as string | undefined) || + (req.headers["x-forwarded-for"] as string | undefined)?.split(",")[0] + ?.trim() || + req.ip || + "unknown" + ); +} + +function inviteDefaultsFromCfg(cfg: AppConfig) { + return { + quotaWindowHours: cfg.inviteQuotaWindowHours, + quotaMax: cfg.inviteQuotaMax, + codeTtlSec: cfg.inviteCodeTtlSec, + maxPendingPerUser: cfg.inviteMaxPendingPerUser, + codeLength: cfg.inviteCodeLength, + }; +} + +function inviteUrl(cfg: AppConfig, code: string): string { + const base = cfg.publicBaseUrl.replace(/\/$/, ""); + return `${base}/app?invite=${encodeURIComponent(code)}`; +} + +async function requireAdmin( + req: FastifyRequest, + reply: FastifyReply, + ctx: RouteContext, +): Promise { + const user = await requireUser(req, reply, ctx); + if (!user) return null; + if (!user.is_admin) { + reply.code(403).send({ error: "admin_required" }); + return null; + } + return user; +} + +/** First system admin only (earliest created_at among is_admin users). */ +async function requireSuperAdmin( + req: FastifyRequest, + reply: FastifyReply, + ctx: RouteContext, +): Promise { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return null; + if (!(await isSuperAdmin(ctx.db, admin.id))) { + reply.code(403).send({ + error: "super_admin_required", + message: "仅系统首位管理员（超管）可执行此操作", + }); + return null; + } + return admin; +} + +/** + * Send non-public sticker bytes (own / pending / admin review) with ETag + * revalidation. Sticker blobs are content-addressed, so a matching + * If-None-Match lets us answer 304 *without* pulling the blob out of Redis — + * these endpoints back thumbnail grids, so that is the whole page's cost. + */ +async function sendPrivateStickerImage( + ctx: RouteContext, + req: FastifyRequest, + reply: FastifyReply, + sticker: Sticker, +): Promise { + let s = sticker; + if (!s.content_hash) { + s = (await ensureStickerContentHash(ctx.db, s)) ?? s; + } + const etag = s.content_hash ? etagFromHash(s.content_hash) : null; + reply.header("Cache-Control", CC_PRIVATE_STICKER); + if (etag) { + reply.header("ETag", etag); + if (ifNoneMatchHits(req.headers["if-none-match"], etag)) { + return reply.code(304).send(); + } + } + const buf = await getStickerBlob(ctx.db, s.id); + if (!buf) return reply.code(404).send({ error: "blob missing" }); + return reply.type(s.mime || "application/octet-stream").send(buf); +} + +/** Parse comma-separated type filters: message,redis,worker,llm or exact types. */ +function parseStreamTypeFilter(raw?: string): Set | null { + if (!raw?.trim()) return null; + const parts = raw + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + if (!parts.length) return null; + return new Set(parts); +} + +function streamTypeMatches(type: string, filter: Set): boolean { + const t = type.toLowerCase(); + if (filter.has(t)) return true; + // group prefixes: message → message.in / message.out + for (const f of filter) { + if (t === f || t.startsWith(f + ".")) return true; + } + return false; +} + +/** + * Shape event for client: short preview by default; full=1 uses fullText (cap 2k). + * Worker emits both preview (48) and fullText (≤2000). + */ +function shapeStreamEventForClient( + ev: import("./activity-stream.js").StreamEvent, + full: boolean, +): import("./activity-stream.js").StreamEvent { + if (!ev.data || !ev.type.startsWith("message.")) { + if (!ev.data) return ev; + // Strip fullText from non-message if ever present + if ("fullText" in ev.data || "text" in ev.data) { + const data = { ...ev.data }; + delete data.fullText; + delete data.text; + return { ...ev, data }; + } + return ev; + } + const data = { ...ev.data }; + const short = + typeof data.preview === "string" ? data.preview : ""; + const long = + typeof data.fullText === "string" + ? data.fullText + : typeof data.text === "string" + ? data.text + : short; + const len = + typeof data.len === "number" ? data.len : Math.max(short.length, long.length); + const chosen = full ? long : short || long.slice(0, 48); + const cap = full ? 2000 : 48; + let preview = chosen; + let truncated = len > chosen.length || !!data.truncated; + if (preview.length > cap) { + preview = preview.slice(0, cap) + "…"; + truncated = true; + } + data.preview = preview; + data.len = len; + data.truncated = truncated; + delete data.fullText; + delete data.text; + return { ...ev, data }; +} + +export async function registerRoutes( + app: FastifyInstance, + ctx: RouteContext, +): Promise { + // CORS only on /api/v1 — never on HTML shells (keeps CF cache clean) + app.addHook("onRequest", async (req, reply) => { + const path = req.url.split("?")[0] || ""; + if (!path.startsWith("/api/v1")) return; + + const origin = req.headers.origin; + if (origin && ctx.cfg.corsOrigins.has(origin)) { + reply.header("Access-Control-Allow-Origin", origin); + reply.header("Access-Control-Allow-Credentials", "true"); + reply.header("Vary", "Origin"); + } + reply.header( + "Access-Control-Allow-Headers", + "Authorization, Content-Type", + ); + reply.header("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS"); + if (req.method === "OPTIONS") { + return reply.code(204).send(); + } + }); + + // Default: private APIs must not be shared-cached + app.addHook("onSend", async (req, reply, payload) => { + const path = req.url.split("?")[0] || ""; + if (!path.startsWith("/api/v1")) return payload; + if (!reply.getHeader("cache-control") && !reply.getHeader("Cache-Control")) { + reply.header("Cache-Control", CC_PRIVATE_NO_STORE); + } + return payload; + }); + + app.get("/health", async (_req, reply) => { + reply.header("Cache-Control", CC_NO_STORE); + return { ok: true, service: "wechat-ai" }; + }); + + /** Cache tools /health so readiness probes stay cheap (TTL 15s). */ + let toolsHealthCache: { at: number; ok: boolean } | null = null; + let toolsHealthInflight: Promise<{ ok: boolean }> | null = null; + const TOOLS_HEALTH_TTL_MS = 15_000; + const cachedToolsHealth = async ( + baseUrl: string, + ): Promise<{ ok: boolean }> => { + const now = Date.now(); + if (toolsHealthCache && now - toolsHealthCache.at < TOOLS_HEALTH_TTL_MS) { + return { ok: toolsHealthCache.ok }; + } + if (toolsHealthInflight) return toolsHealthInflight; + toolsHealthInflight = probeToolsHealth(baseUrl, 4000) + .then((r) => { + toolsHealthCache = { at: Date.now(), ok: r.ok }; + return { ok: r.ok }; + }) + .catch(() => { + toolsHealthCache = { at: Date.now(), ok: false }; + return { ok: false }; + }) + .finally(() => { + toolsHealthInflight = null; + }); + return toolsHealthInflight; + }; + + /** LB / CF Worker readiness: Redis + process identity (short timeout on client). */ + app.get("/health/ready", async (_req, reply) => { + reply.header("Cache-Control", CC_NO_STORE); + const workerId = ctx.worker.getWorkerId(); + // LB probes hit this constantly — run the three checks concurrently. + // Tools gateway is the only egress for user custom LLM + web search; + // cachedToolsHealth keeps probes off HF. + const workerRunning = ctx.worker.isRunning(); + const [redisOk, tools] = await Promise.all([ + ctx.db.ping().then( + () => true, + () => false, + ), + ctx.cfg.toolsBaseUrl + ? cachedToolsHealth(ctx.cfg.toolsBaseUrl) + : Promise.resolve(null), + ]); + // Required only when the deployment actually depends on tools + const toolsRequired = + Boolean(ctx.cfg.toolsBaseUrl) && ctx.cfg.webSearchEnabled; + const ok = redisOk && (!toolsRequired || tools?.ok === true); + const body = { + ok, + service: "wechat-ai", + redis: redisOk, + workerId, + workerRunning, + workerEnabled: ctx.cfg.workerEnabled, + tools: ctx.cfg.toolsBaseUrl + ? { configured: true, ok: tools?.ok === true, required: toolsRequired } + : { configured: false, ok: false, required: false }, + }; + if (!ok) return reply.code(503).send(body); + return body; + }); + + // Public CDN: approved + public + enabled stickers only (no cookie) + const cdnLimiter = new RateLimiter(120, 60_000); + app.get<{ Params: { id: string }; Querystring: { v?: string } }>( + "/cdn/s/:id", + async (req, reply) => { + const ip = + (req.headers["cf-connecting-ip"] as string | undefined) || + req.ip || + "unknown"; + if (!cdnLimiter.tryTake(`cdn:${ip}`)) { + return reply.code(429).send({ error: "rate limited" }); + } + // Strip accidental extension from id (e.g. sticker_xxx.png) + let id = req.params.id || ""; + id = id.replace(/\.(png|jpe?g|gif|webp|bin)$/i, ""); + if (!id) return reply.code(404).send({ error: "not found" }); + + let s = await getSticker(ctx.db, id); + if ( + !s || + !s.enabled || + s.visibility !== "public" || + s.review_status !== "approved" + ) { + return reply.code(404).send({ error: "not found" }); + } + if (!s.content_hash) { + s = (await ensureStickerContentHash(ctx.db, s)) ?? s; + } + // Answer conditional requests from the content hash alone — no reason to + // pull the blob out of Redis just to throw it away on a 304. + if (s.content_hash) { + const etag = etagFromHash(s.content_hash); + setPublicCache(reply, CC_CDN_STICKER, CDN_CDN_STICKER, { + etag, + cacheTag: `sticker-${s.id}`, + }); + if (ifNoneMatchHits(req.headers["if-none-match"], etag)) { + return reply.code(304).send(); + } + const buf = await getStickerBlob(ctx.db, s.id); + if (!buf) return reply.code(404).send({ error: "not found" }); + return reply.type(s.mime || "application/octet-stream").send(buf); + } + + // No hash (blob missing on migrate) — fall back to length-based ETag + const buf = await getStickerBlob(ctx.db, s.id); + if (!buf) return reply.code(404).send({ error: "not found" }); + const etag = etagFromHash(String(buf.length)); + setPublicCache(reply, CC_CDN_STICKER, CDN_CDN_STICKER, { + etag, + cacheTag: `sticker-${s.id}`, + }); + if (ifNoneMatchHits(req.headers["if-none-match"], etag)) { + return reply.code(304).send(); + } + return reply.type(s.mime || "application/octet-stream").send(buf); + }, + ); + + // ── Auth (LINUX DO OAuth + local password + invites) ── + + const authLoginLimiter = new RateLimiter(20, 60_000); + const authRegisterLimiter = new RateLimiter(10, 60_000); + const authInvitePeekLimiter = new RateLimiter(40, 60_000); + + app.get("/api/v1/auth/config", async (_req, reply) => { + setPublicCache(reply, CC_AUTH_CONFIG, CDN_AUTH_CONFIG); + const oauth = loadLinuxDoConfig(); + return { + oauthEnabled: Boolean(oauth), + provider: "linux.do", + localAuthEnabled: ctx.cfg.localAuthEnabled, + inviteRequiredForLocal: ctx.cfg.inviteRequiredForLocal, + passwordMinLength: ctx.cfg.passwordMinLength, + }; + }); + + app.get("/api/v1/auth/login", async (req, reply) => { + const oauth = loadLinuxDoConfig(); + if (!oauth) { + return reply + .code(503) + .send({ error: "LINUX DO OAuth 未配置（LINUXDO_CLIENT_ID/SECRET/REDIRECT_URI）" }); + } + const state = newOAuthState(); + const q = req.query as { redirect?: string }; + await saveOauthState(ctx.db, state, { redirect: q.redirect || "/app" }); + return reply.redirect(buildAuthorizeUrl(oauth, state)); + }); + + app.get("/api/v1/auth/callback", async (req, reply) => { + const oauth = loadLinuxDoConfig(); + if (!oauth) return reply.code(503).send("oauth not configured"); + const q = req.query as { code?: string; state?: string; error?: string }; + if (q.error) return reply.code(400).send(`oauth error: ${q.error}`); + if (!q.code || !q.state) return reply.code(400).send("missing code/state"); + const st = await takeOauthState(ctx.db, q.state); + if (!st) return reply.code(400).send("invalid or expired state"); + + try { + const token = await exchangeCode(oauth, q.code); + const info = await fetchUserInfo(oauth, token.access_token); + const user = await upsertUser( + ctx.db, + { + id: String(info.id), + username: info.username, + name: info.name || info.username, + avatarUrl: info.avatar_url ?? null, + trustLevel: info.trust_level ?? 0, + authProvider: "linuxdo", + }, + ctx.cfg.adminIds, + { + firstUserIsAdmin: ctx.cfg.firstUserIsAdmin, + }, + ); + if (isUserBanned(user)) { + return reply + .code(403) + .type("text/plain; charset=utf-8") + .send( + `账号已被封禁${user.banned_reason ? `：${user.banned_reason}` : ""}`, + ); + } + const sid = await createAppSession(ctx.db, user.id); + setSessionCookie(reply, ctx.cfg, sid); + await writeAudit(ctx.db, "user_login", user.id, { + username: user.username, + is_admin: user.is_admin, + method: "oauth", + }); + const dest = st.redirect?.startsWith("/") ? st.redirect : "/app"; + return reply.redirect(dest); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return reply.code(502).type("text/plain").send(`OAuth failed: ${msg}`); + } + }); + + app.post<{ + Body: { + username?: string; + password?: string; + inviteCode?: string; + name?: string; + }; + }>("/api/v1/auth/register", async (req, reply) => { + if (!ctx.cfg.localAuthEnabled) { + return reply.code(503).send({ error: "local_auth_disabled" }); + } + const ip = clientIp(req); + if (!authRegisterLimiter.tryTake(`reg:${ip}`)) { + return reply.code(429).send({ error: "rate limited" }); + } + const usernameRaw = String(req.body?.username || ""); + const password = String(req.body?.password || ""); + const inviteCode = String(req.body?.inviteCode || ""); + const name = req.body?.name ? String(req.body.name).trim() : undefined; + + let username: string; + try { + username = validateLocalUsername(usernameRaw); + assertPasswordPolicy(password, ctx.cfg.passwordMinLength); + } catch (e) { + const msg = e instanceof Error ? e.message : "invalid"; + if (msg === "weak_password") { + return reply.code(400).send({ + error: "weak_password", + message: `密码至少 ${ctx.cfg.passwordMinLength} 位`, + }); + } + if (msg === "reserved_username") { + return reply.code(400).send({ error: "reserved_username", message: "用户名不可用" }); + } + return reply.code(400).send({ + error: "invalid_username", + message: "用户名须以字母开头，3–32 位字母/数字/下划线", + }); + } + + // Unauthenticated endpoint — never scan the user table here. SCARD is O(1) + // and only feeds the `is this the very first user` check. + const [settings, totalUsers] = await Promise.all([ + getInviteSettings(ctx.db, inviteDefaultsFromCfg(ctx.cfg)), + countUsers(ctx.db), + ]); + const needInvite = ctx.cfg.inviteRequiredForLocal; + const bootstrap = + totalUsers === 0 && ctx.cfg.firstUserIsAdmin && ctx.cfg.adminIds.size === 0; + + let inviteRec: Awaited> = null; + if (needInvite && !bootstrap) { + inviteRec = await peekInviteCode(ctx.db, inviteCode); + if (!inviteRec) { + return reply.code(400).send({ + error: "invalid_invite", + message: "邀请码无效或已使用", + }); + } + } + + const passwordHash = await hashPassword(password); + let user; + try { + // Create user first (claims username). Consume invite after; roll back if race. + try { + user = await createLocalUser( + ctx.db, + { + username, + passwordHash, + name, + invitedBy: inviteRec?.inviterUserId ?? null, + inviteCodeUsed: inviteRec?.code ?? null, + }, + ctx.cfg.adminIds, + { firstUserIsAdmin: ctx.cfg.firstUserIsAdmin }, + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (msg === "username_taken") { + return reply.code(409).send({ + error: "username_taken", + message: "用户名已被占用", + }); + } + if (msg === "invalid_username" || msg === "reserved_username") { + return reply.code(400).send({ + error: msg, + message: + msg === "reserved_username" + ? "用户名不可用" + : "用户名须以字母开头，3–32 位字母/数字/下划线", + }); + } + throw err; + } + + if (needInvite && !bootstrap) { + const consumed = await consumeInviteCode(ctx.db, inviteCode, user.id); + if (!consumed) { + await deleteUserAccount(ctx.db, user.id); + return reply.code(400).send({ + error: "invalid_invite", + message: "邀请码无效或已使用", + }); + } + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: msg }); + } + + const sid = await createAppSession(ctx.db, user.id); + setSessionCookie(reply, ctx.cfg, sid); + await writeAudit(ctx.db, "user_register", user.id, { + username: user.username, + method: "password", + invitedBy: user.invited_by, + inviteCode: user.invite_code_used, + }); + return { user: userPublicFields(user) }; + }); + + app.post<{ Body: { username?: string; password?: string } }>( + "/api/v1/auth/password-login", + async (req, reply) => { + if (!ctx.cfg.localAuthEnabled) { + return reply.code(503).send({ error: "local_auth_disabled" }); + } + const ip = clientIp(req); + const usernameRaw = String(req.body?.username || "").trim(); + const password = String(req.body?.password || ""); + const unameKey = usernameRaw.toLowerCase() || "-"; + if ( + !authLoginLimiter.tryTake(`login:${ip}`) || + !authLoginLimiter.tryTake(`loginu:${unameKey}`) + ) { + return reply.code(429).send({ error: "rate limited" }); + } + const user = usernameRaw + ? await getUserByUsername(ctx.db, usernameRaw) + : undefined; + const hash = user?.password_hash || null; + // Always verify to reduce timing gap when hash exists; dummy when missing + let ok = false; + if (hash) { + ok = await verifyPassword(password, hash); + } else { + // burn some CPU with a failed verify against a fixed-format dummy + await verifyPassword( + password || "x", + "scrypt$16384$8$1$AAAAAAAAAAAAAAAAAAAAAA$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); + ok = false; + } + if (!ok || !user) { + return reply.code(401).send({ error: "invalid_credentials" }); + } + if (isUserBanned(user)) { + return reply.code(403).send({ + error: "user_banned", + message: user.banned_reason || "账号已被封禁", + }); + } + const sid = await createAppSession(ctx.db, user.id); + setSessionCookie(reply, ctx.cfg, sid); + await writeAudit(ctx.db, "user_login", user.id, { + username: user.username, + method: "password", + }); + return { user: userPublicFields(user) }; + }, + ); + + app.get<{ Params: { code: string } }>( + "/api/v1/auth/invite/:code", + async (req, reply) => { + const ip = clientIp(req); + if (!authInvitePeekLimiter.tryTake(`invpeek:${ip}`)) { + return reply.code(429).send({ error: "rate limited" }); + } + const rec = await peekInviteCode(ctx.db, req.params.code); + if (!rec) { + return { valid: false }; + } + return { + valid: true, + expiresAt: rec.expiresAt, + inviterUsername: rec.inviterUsername, + }; + }, + ); + + app.post("/api/v1/auth/logout", async (req, reply) => { + const cookies = parseCookie(req.headers.cookie); + const sid = cookies[ctx.cfg.sessionCookieName]; + if (sid) await destroyAppSession(ctx.db, sid); + clearSessionCookie(reply, ctx.cfg); + return { ok: true }; + }); + + app.get("/api/v1/auth/me", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const superAdmin = user.is_admin + ? await isSuperAdmin(ctx.db, user.id) + : false; + return { + user: { + ...userPublicFields(user), + isSuperAdmin: superAdmin, + }, + }; + }); + + // ── Me: invites ─────────────────────────────────────── + + app.get("/api/v1/me/invites", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const settings = await getInviteSettings( + ctx.db, + inviteDefaultsFromCfg(ctx.cfg), + ); + const [items, quota] = await Promise.all([ + listPendingInvites(ctx.db, user.id), + getInviteQuotaStatus(ctx.db, user.id, settings), + ]); + return { + items: items.map((i) => ({ + code: i.code, + createdAt: i.createdAt, + expiresAt: i.expiresAt, + inviteUrl: inviteUrl(ctx.cfg, i.code), + })), + quota: { + used: quota.used, + max: Number.isFinite(quota.max) ? quota.max : null, + windowHours: quota.windowHours, + remaining: Number.isFinite(quota.remaining) ? quota.remaining : null, + retryAfterSec: quota.retryAfterSec, + }, + settings: { + maxPendingPerUser: settings.maxPendingPerUser, + codeTtlSec: settings.codeTtlSec, + }, + }; + }); + + app.post("/api/v1/me/invites", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const settings = await getInviteSettings( + ctx.db, + inviteDefaultsFromCfg(ctx.cfg), + ); + try { + const result = await createInviteCode(ctx.db, { + inviterUserId: user.id, + inviterUsername: user.username, + settings, + }); + if (!result.ok) { + if (result.error === "invite_quota") { + return reply.code(429).send({ + error: "invite_quota", + message: "邀请生成次数已达上限，请稍后再试", + used: result.quota?.used, + max: result.quota?.max, + windowHours: result.quota?.windowHours, + retryAfterSec: result.quota?.retryAfterSec, + }); + } + return reply.code(400).send({ + error: "invite_pending_limit", + message: `未使用邀请码过多（最多 ${result.maxPending} 个）`, + maxPending: result.maxPending, + }); + } + await writeAudit(ctx.db, "invite_created", user.id, { + code: result.invite.code, + }); + return { + code: result.invite.code, + createdAt: result.invite.createdAt, + expiresAt: result.invite.expiresAt, + inviteUrl: inviteUrl(ctx.cfg, result.invite.code), + quota: { + used: result.quota.used, + max: Number.isFinite(result.quota.max) ? result.quota.max : null, + windowHours: result.quota.windowHours, + remaining: Number.isFinite(result.quota.remaining) + ? result.quota.remaining + : null, + retryAfterSec: result.quota.retryAfterSec, + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(500).send({ error: message }); + } + }); + + app.delete<{ Params: { code: string } }>( + "/api/v1/me/invites/:code", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const ok = await revokeInviteCode(ctx.db, user.id, req.params.code); + if (!ok) return reply.code(404).send({ error: "not found" }); + await writeAudit(ctx.db, "invite_revoked", user.id, { + code: req.params.code, + }); + return { ok: true }; + }, + ); + + // ── WeChat ↔ LINUX DO bind (for @username P2P) ───── + + app.get("/api/v1/me/wechat-bind", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.p2pEnabled) { + return { enabled: false, bound: false }; + } + const bind = await getBindByUser(ctx.db, user.id); + if (!bind) { + return { + enabled: true, + bound: false, + username: user.username, + }; + } + const ctxToken = await getContextToken(ctx.db, bind.botId, bind.peerId); + return { + enabled: true, + bound: true, + username: user.username, + botId: bind.botId, + peerId: bind.peerId, + boundAt: bind.boundAt, + reachable: Boolean(ctxToken), + }; + }); + + app.post("/api/v1/me/wechat-bind/code", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.p2pEnabled) { + return reply.code(503).send({ error: "p2p_disabled" }); + } + try { + const rec = await createBindCode( + ctx.db, + user.id, + user.username, + ctx.cfg.p2pBindCodeTtlSec, + ); + await writeAudit(ctx.db, "wechat_bind_code", user.id, { + codePrefix: rec.code.slice(0, 2), + }); + return { + code: rec.code, + expiresInSec: ctx.cfg.p2pBindCodeTtlSec, + username: user.username, + instruction: `在微信中给机器人发送：/绑定 ${rec.code}`, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(500).send({ error: message }); + } + }); + + app.delete("/api/v1/me/wechat-bind", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const prev = await clearPrimaryBind(ctx.db, user.id); + await writeAudit(ctx.db, "wechat_bind_clear", user.id, { + hadBind: Boolean(prev), + botId: prev?.botId, + peerId: prev?.peerId, + }); + return { ok: true, cleared: Boolean(prev) }; + }); + + // ── P2P block list ─────────────────────────────────── + + app.get("/api/v1/me/blocks", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.p2pEnabled) { + return { enabled: false, blocks: [] }; + } + const ids = await listBlockedUserIds(ctx.db, user.id); + const map = await getUsersByIds(ctx.db, ids); + const blocks = ids.map((id) => { + const u = map.get(id); + return { + userId: id, + username: u?.username ?? null, + name: u?.name ?? null, + avatarUrl: u?.avatar_url ?? null, + }; + }); + return { enabled: true, blocks }; + }); + + app.post<{ Body: { username?: string; userId?: string } }>( + "/api/v1/me/blocks", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.p2pEnabled) { + return reply.code(503).send({ error: "p2p_disabled" }); + } + const username = (req.body?.username || "").trim(); + let targetId = (req.body?.userId || "").trim(); + let targetUser = targetId ? await getUser(ctx.db, targetId) : undefined; + if (!targetUser && username) { + targetUser = await getUserByUsername(ctx.db, username); + targetId = targetUser?.id || ""; + } + if (!targetUser || !targetId) { + return reply.code(404).send({ error: "user_not_found" }); + } + const r = await blockUser(ctx.db, user.id, targetId); + if (!r.ok && r.reason === "self") { + return reply.code(400).send({ error: "cannot_block_self" }); + } + await writeAudit(ctx.db, "p2p_block", user.id, { + blockedUserId: targetId, + username: targetUser.username, + already: r.ok === false && r.reason === "already", + }); + return { + ok: true, + already: r.ok === false && r.reason === "already", + blocked: { + userId: targetUser.id, + username: targetUser.username, + name: targetUser.name, + }, + }; + }, + ); + + app.delete<{ Params: { userId: string } }>( + "/api/v1/me/blocks/:userId", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const targetId = req.params.userId; + if (!targetId) { + return reply.code(400).send({ error: "userId required" }); + } + const removed = await unblockUser(ctx.db, user.id, targetId); + await writeAudit(ctx.db, "p2p_unblock", user.id, { + blockedUserId: targetId, + removed, + }); + return { ok: true, removed }; + }, + ); + + // ── User bots ──────────────────────────────────────── + + function proactiveDefaults() { + return { + idleHours: ctx.cfg.proactiveIdleHours, + minIntervalHours: ctx.cfg.proactiveMinIntervalHours, + maxPerDay: ctx.cfg.proactiveMaxPerDay, + quietHours: ctx.cfg.proactiveQuietHours, + }; + } + + function mapBotProactive(b: { + proactive_enabled?: number; + proactive_idle_hours?: number; + proactive_min_interval_hours?: number; + proactive_max_per_day?: number; + proactive_quiet_hours?: string | null; + }) { + const merged = mergeBotProactiveConfig(b, proactiveDefaults()); + return { + globalProactiveEnabled: ctx.cfg.proactiveEnabled, + proactiveEnabled: merged.enabled, + proactiveIdleHours: merged.idleHours, + proactiveMinIntervalHours: merged.minIntervalHours, + proactiveMaxPerDay: merged.maxPerDay, + proactiveQuietHours: merged.quietHours, + }; + } + + app.get("/api/v1/me/bots", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const bots = await listBotsByOwner(ctx.db, user.id); + const tokenMap = await hasBotCredentialsMany( + ctx.db, + bots.map((b) => b.id), + ); + const mapped = bots.map((b) => ({ + id: b.id, + displayName: b.display_name, + accountRef: b.account_ref, + status: b.status, + ownerUserId: b.owner_user_id, + hasToken: Boolean(tokenMap[b.id]), + ...mapBotProactive(b), + })); + return { bots: mapped }; + }); + + app.post<{ Body: { displayName?: string } }>( + "/api/v1/me/bots/login/start", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + try { + const session = await ctx.loginSessions.start( + user.id, + req.body?.displayName, + ); + return { session }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(502).send({ error: message }); + } + }, + ); + + /** Re-scan QR to refresh token for an existing bot (keeps peers / memories / assignments). */ + app.post<{ Params: { botId: string } }>( + "/api/v1/me/bots/:botId/relogin/start", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + if (bot.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + try { + // Session owner = acting user (so poll/cancel ACL works for owner or admin) + const session = await ctx.loginSessions.start( + user.id, + bot.display_name, + { rebindBotId: bot.id }, + ); + return { session }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (/not found/i.test(message)) { + return reply.code(404).send({ error: message }); + } + return reply.code(502).send({ error: message }); + } + }, + ); + + app.get<{ Params: { sessionId: string } }>( + "/api/v1/me/bots/login/:sessionId", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const session = await ctx.loginSessions.get(req.params.sessionId); + if (!session || session.ownerUserId !== user.id) { + return reply.code(404).send({ error: "session not found" }); + } + return { session }; + }, + ); + + /** + * Login QR as SVG, rendered on this box. + * + * The scan link embeds a login ticket. It used to be handed to + * api.qrserver.com as a query parameter to get a QR image back, which put a + * credential in a third party's logs. Rendering locally keeps it between this + * origin and the owner's browser. Served as its own resource (rather than + * inlined in the 1.5s status poll) so the browser fetches it once. + */ + app.get<{ Params: { sessionId: string } }>( + "/api/v1/me/bots/login/:sessionId/qr.svg", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const session = await ctx.loginSessions.get(req.params.sessionId); + if (!session || session.ownerUserId !== user.id) { + return reply.code(404).send({ error: "session not found" }); + } + if (!session.openUrl) { + return reply.code(409).send({ error: "qr not ready" }); + } + + let svg: string; + try { + svg = qrSvg(session.openUrl, { ec: "M", border: 4, pixelSize: 200 }); + } catch (err) { + req.log?.warn( + { err, sessionId: session.sessionId }, + "qr render failed", + ); + return reply.code(500).send({ error: "qr render failed" }); + } + + reply.header("Content-Type", "image/svg+xml; charset=utf-8"); + // Ticket-bearing and per-owner: never shared, never stored. The client + // fetches this once per login session, so no-store costs nothing. + reply.header("Cache-Control", CC_PRIVATE_QR); + return reply.send(svg); + }, + ); + + app.post<{ Params: { sessionId: string } }>( + "/api/v1/me/bots/login/:sessionId/cancel", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const ok = await ctx.loginSessions.cancel(req.params.sessionId, user.id); + if (!ok) return reply.code(404).send({ error: "session not found" }); + return { ok: true }; + }, + ); + + app.patch<{ + Params: { botId: string }; + Body: { + displayName?: string; + proactiveEnabled?: boolean; + proactiveIdleHours?: number; + proactiveMinIntervalHours?: number; + proactiveMaxPerDay?: number; + proactiveQuietHours?: string | null; + }; + }>("/api/v1/me/bots/:botId", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + if (bot.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + const body = req.body ?? {}; + const hasName = body.displayName !== undefined; + const hasProactive = + body.proactiveEnabled !== undefined || + body.proactiveIdleHours !== undefined || + body.proactiveMinIntervalHours !== undefined || + body.proactiveMaxPerDay !== undefined || + body.proactiveQuietHours !== undefined; + + if (!hasName && !hasProactive) { + return reply.code(400).send({ + error: "displayName or proactive settings required", + }); + } + + try { + let updated = bot; + if (hasName) { + if (!body.displayName?.trim()) { + return reply.code(400).send({ error: "displayName required" }); + } + updated = await updateBotDisplayName( + ctx.db, + bot.id, + body.displayName, + ); + await writeAudit(ctx.db, "bot_renamed", user.id, { + botId: bot.id, + displayName: updated.display_name, + }); + } + if (hasProactive) { + updated = await updateBotProactiveSettings(ctx.db, bot.id, { + proactiveEnabled: body.proactiveEnabled, + proactiveIdleHours: body.proactiveIdleHours, + proactiveMinIntervalHours: body.proactiveMinIntervalHours, + proactiveMaxPerDay: body.proactiveMaxPerDay, + proactiveQuietHours: body.proactiveQuietHours, + }); + await writeAudit(ctx.db, "bot_proactive_updated", user.id, { + botId: bot.id, + proactiveEnabled: updated.proactive_enabled, + proactiveIdleHours: updated.proactive_idle_hours, + proactiveMinIntervalHours: updated.proactive_min_interval_hours, + proactiveMaxPerDay: updated.proactive_max_per_day, + proactiveQuietHours: updated.proactive_quiet_hours, + }); + } + return { + bot: { + id: updated.id, + displayName: updated.display_name, + accountRef: updated.account_ref, + status: updated.status, + ownerUserId: updated.owner_user_id, + ...mapBotProactive(updated), + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.delete<{ Params: { botId: string } }>( + "/api/v1/me/bots/:botId", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + if (bot.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + // deleteBotAccount also removes Redis creds (wa:bot:{id}:creds) + await deleteBotAccount(ctx.db, bot.id); + ctx.worker.stopBot(bot.id); + await writeAudit(ctx.db, "bot_deleted", user.id, { botId: bot.id }); + return { ok: true }; + }, + ); + + // Peers for my bots + app.get("/api/v1/me/peers", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const botId = (req.query as { botId?: string }).botId; + const myBots = await listBotsByOwner(ctx.db, user.id); + const botIds = new Set(myBots.map((b) => b.id)); + if (botId && !botIds.has(botId) && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + const peers = botId + ? await listPeers(ctx.db, botId) + : await listPeersForBots(ctx.db, [...botIds]); + const asgMap = await getAssignmentsMany( + ctx.db, + peers.map((p) => ({ + botAccountId: p.bot_account_id, + peerId: p.peer_id, + })), + ); + const personaIds = [ + ...new Set( + peers + .map((p) => asgMap.get(`${p.bot_account_id}|${p.peer_id}`)) + .filter((id): id is string => Boolean(id)), + ), + ]; + const personaMap = await getPersonasByIds(ctx.db, personaIds); + const enriched = peers.map((p) => { + const personaId = + asgMap.get(`${p.bot_account_id}|${p.peer_id}`) ?? null; + const persona = personaId ? personaMap.get(personaId) : undefined; + return { + ...p, + personaId, + personaSlug: persona?.slug ?? null, + personaName: persona?.display_name ?? null, + proactiveEnabled: Boolean(p.proactive_enabled), + lastActivityAt: p.last_activity_at ?? null, + lastProactiveAt: p.last_proactive_at ?? null, + }; + }); + return { + peers: enriched, + globalProactiveEnabled: ctx.cfg.proactiveEnabled, + }; + }); + + app.patch<{ + Body: { botAccountId: string; peerId: string; enabled: boolean }; + }>("/api/v1/me/peers/proactive", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const { botAccountId, peerId, enabled } = req.body ?? {}; + if (!botAccountId || !peerId || typeof enabled !== "boolean") { + return reply + .code(400) + .send({ error: "botAccountId, peerId, enabled required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + try { + const peer = await setPeerProactiveEnabled( + ctx.db, + botAccountId, + peerId, + enabled, + ); + await writeAudit(ctx.db, "peer_proactive_toggled", user.id, { + botAccountId, + peerId, + enabled, + }); + return { + peer: { + ...peer, + proactiveEnabled: Boolean(peer.proactive_enabled), + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ + Body: { botAccountId: string; peerId: string; personaId?: string }; + }>("/api/v1/me/peers/approve", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const { botAccountId, peerId, personaId } = req.body ?? {}; + if (!botAccountId || !peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + const peer = await approvePeer(ctx.db, botAccountId, peerId); + if (personaId) await setAssignment(ctx.db, botAccountId, peerId, personaId); + await writeAudit(ctx.db, "peer_approved", user.id, { + botAccountId, + peerId, + }); + return { peer }; + }); + + app.put<{ + Body: { botAccountId: string; peerId: string; personaId: string }; + }>("/api/v1/me/assignments", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const { botAccountId, peerId, personaId } = req.body ?? {}; + if (!botAccountId || !peerId || !personaId) { + return reply.code(400).send({ error: "missing fields" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + if (!(await userCanUsePersona(ctx.db, user.id, personaId))) { + return reply + .code(403) + .send({ error: "persona_not_in_library", message: "请先添加人设，或人设已私有" }); + } + await setAssignment(ctx.db, botAccountId, peerId, personaId); + return { ok: true }; + }); + + /** @deprecated use /me/personas — kept for compatibility, returns library */ + app.get("/api/v1/personas", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const personas = await listUserPersonaLibrary(ctx.db, user.id); + const prompts = await getPublishedPromptsMany(ctx.db, personas); + return { + personas: personas.map((p) => ({ + ...p, + systemPromptPreview: (prompts.get(p.id) ?? "").slice(0, 200), + })), + }; + }); + + // ── Persona Square ─────────────────────────────────── + + app.get("/api/v1/square/personas", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const q = req.query as { + q?: string; + page?: string; + limit?: string; + sort?: string; + }; + const limit = Math.min(Math.max(Number(q.limit ?? "20") || 20, 1), 50); + const page = Math.max(Number(q.page ?? "1") || 1, 1); + const offset = (page - 1) * limit; + const sort = + q.sort === "recent" || + q.sort === "name" || + q.sort === "use" || + q.sort === "heat" + ? q.sort + : "heat"; + const { items, total } = await searchPublicPersonas(ctx.db, { + q: q.q, + limit, + offset, + sort, + }); + const [libIds, prompts] = await Promise.all([ + getPersonaLibraryIdSet(ctx.db, user.id), + getPublishedPromptsMany(ctx.db, items), + ]); + const personas = items.map((p) => + personaPublicDto(p, { + inLibrary: isInPersonaLibraryLocal(p, user.id, libIds), + systemPromptPreview: (prompts.get(p.id) ?? "").slice(0, 160), + }), + ); + return { personas, total, page, limit }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/square/personas/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p || !p.enabled) { + return reply.code(404).send({ error: "not found" }); + } + if (p.visibility === "private" && p.owner_user_id !== user.id) { + return reply.code(403).send({ error: "private" }); + } + const [prompt, graph, inLibrary] = await Promise.all([ + getPublishedPrompt(ctx.db, p.id), + getPublishedGraph(ctx.db, p.id), + isInPersonaLibrary(ctx.db, user.id, p.id), + ]); + return { + persona: personaPublicDto(p, { + systemPrompt: prompt, + graph: graph ?? null, + inLibrary, + }), + }; + }, + ); + + /** GET chatflow graph (owner or public read for try/editor load). */ + app.get<{ Params: { id: string } }>( + "/api/v1/square/personas/:id/graph", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p || !p.enabled) { + return reply.code(404).send({ error: "not found" }); + } + const canRead = + p.owner_user_id === user.id || + user.is_admin || + (p.visibility === "public" && p.enabled); + if (!canRead) { + return reply.code(403).send({ error: "forbidden" }); + } + const stored = await getPublishedGraph(ctx.db, p.id); + const graph = stored ?? createDefaultChatflowGraph(); + return { + personaId: p.id, + mode: p.mode === "chatflow" ? "chatflow" : "prompt", + graph, + isDefault: !stored, + editable: p.owner_user_id === user.id || Boolean(user.is_admin), + }; + }, + ); + + /** PUT chatflow graph (owner only). Also switches mode to chatflow. */ + app.put<{ + Params: { id: string }; + Body: { graph?: unknown; systemPrompt?: string; mode?: "prompt" | "chatflow" }; + }>("/api/v1/square/personas/:id/graph", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + if (p.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + const body = req.body ?? {}; + try { + const graph = body.graph ?? createDefaultChatflowGraph(); + validateChatflowGraph(graph, { + maxNodes: ctx.cfg.chatflowMaxNodes, + }); + const prompt = + body.systemPrompt?.trim() || + (await getPublishedPrompt(ctx.db, p.id)) || + "你是一个有帮助的助手。"; + const persona = await updatePersonaMeta(ctx.db, p.id, { + systemPrompt: prompt, + graphJson: graph, + mode: body.mode === "prompt" ? "prompt" : "chatflow", + }); + await writeAudit(ctx.db, "persona_graph_saved", user.id, { + id: p.id, + }); + return { + persona: personaPublicDto(persona), + graph, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const code = + err instanceof ChatflowError ? 400 : message.includes("not found") ? 404 : 400; + return reply.code(code).send({ error: message }); + } + }); + + app.post<{ + Params: { id: string }; + Body: { displayName?: string }; + }>("/api/v1/square/personas/:id/fork", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.personaForkEnabled) { + return reply.code(503).send({ error: "fork_disabled", message: "人设改编已关闭" }); + } + try { + const { persona, systemPrompt } = await forkPersona(ctx.db, { + sourceId: req.params.id, + ownerUserId: user.id, + displayName: req.body?.displayName, + allowPrivateSource: Boolean(user.is_admin), + }); + await writeAudit(ctx.db, "persona_forked", user.id, { + sourceId: req.params.id, + personaId: persona.id, + }); + return { + persona: personaPublicDto(persona, { systemPrompt }), + systemPrompt, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (message.includes("not found")) { + return reply.code(404).send({ error: message }); + } + if (message.includes("private")) { + return reply.code(403).send({ error: message }); + } + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ + Body: { + displayName: string; + description?: string; + systemPrompt: string; + visibility?: "public" | "private"; + tags?: string[]; + mode?: "prompt" | "chatflow"; + llmProviderId?: string | null; + webSearchEnabled?: boolean; + }; + }>("/api/v1/square/personas", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const body = req.body ?? {}; + if (!body.displayName?.trim() || !body.systemPrompt?.trim()) { + return reply + .code(400) + .send({ error: "displayName and systemPrompt required" }); + } + try { + if (body.llmProviderId) { + const prov = await getLlmProvider(ctx.db, body.llmProviderId); + if (!prov || prov.owner_user_id !== user.id) { + return reply.code(400).send({ error: "invalid llmProviderId" }); + } + } + const persona = await createPersona(ctx.db, { + displayName: body.displayName, + description: body.description, + systemPrompt: body.systemPrompt, + visibility: body.visibility === "private" ? "private" : "public", + tags: body.tags, + ownerUserId: user.id, + mode: body.mode === "chatflow" ? "chatflow" : "prompt", + llmProviderId: body.llmProviderId ?? null, + webSearchEnabled: Boolean(body.webSearchEnabled), + }); + await writeAudit(ctx.db, "persona_published_square", user.id, { + id: persona.id, + visibility: persona.visibility, + }); + return { persona: personaPublicDto(persona) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.put<{ + Params: { id: string }; + Body: { + displayName?: string; + description?: string; + tags?: string[]; + visibility?: "public" | "private"; + systemPrompt?: string; + mode?: "prompt" | "chatflow"; + llmProviderId?: string | null; + webSearchEnabled?: boolean; + }; + }>("/api/v1/square/personas/:id", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + if (p.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + try { + const body = req.body ?? {}; + if (body.llmProviderId) { + const prov = await getLlmProvider(ctx.db, body.llmProviderId); + if (!prov || prov.owner_user_id !== user.id) { + return reply.code(400).send({ error: "invalid llmProviderId" }); + } + } + const persona = await updatePersonaMeta(ctx.db, p.id, { + displayName: body.displayName, + description: body.description, + tags: body.tags, + visibility: body.visibility, + systemPrompt: body.systemPrompt, + mode: body.mode, + llmProviderId: body.llmProviderId, + webSearchEnabled: body.webSearchEnabled, + }); + return { persona: personaPublicDto(persona) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.delete<{ Params: { id: string } }>( + "/api/v1/square/personas/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + if (p.owner_user_id === "system") { + return reply.code(403).send({ error: "cannot delete system persona" }); + } + if (p.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + await softDeletePersona(ctx.db, p.id); + await writeAudit(ctx.db, "persona_soft_deleted", user.id, { id: p.id }); + return { ok: true }; + }, + ); + + // ── User custom LLM providers (egress via HF tools only) ── + app.get("/api/v1/me/llm-providers", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const secret = ctx.cfg.llmProviderSecret; + if (!secret) { + return { + providers: [], + configured: false, + error: "LLM_PROVIDER_SECRET not configured on server", + }; + } + const rows = await listLlmProvidersByOwner(ctx.db, user.id); + return { + configured: true, + providers: rows.map((r) => toPublicProvider(r, secret)), + toolsGateway: Boolean(ctx.cfg.toolsBaseUrl), + webSearchEnabled: ctx.cfg.webSearchEnabled, + }; + }); + + app.post<{ + Body: { + name?: string; + baseUrl?: string; + apiKey?: string; + defaultModel?: string; + }; + }>("/api/v1/me/llm-providers", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const secret = ctx.cfg.llmProviderSecret; + if (!secret) { + return reply + .code(503) + .send({ error: "LLM_PROVIDER_SECRET not configured on server" }); + } + if (!ctx.cfg.toolsBaseUrl || !ctx.cfg.toolsApiKey) { + return reply.code(503).send({ + error: + "TOOLS_BASE_URL / TOOLS_API_KEY required for user custom LLM (HF tools)", + }); + } + const body = req.body ?? {}; + try { + const row = await createLlmProvider(ctx.db, { + ownerUserId: user.id, + name: body.name || "", + baseUrl: body.baseUrl || "", + apiKey: body.apiKey || "", + defaultModel: body.defaultModel || "", + secret, + }); + await writeAudit(ctx.db, "llm_provider_create", user.id, { + id: row.id, + baseUrl: row.base_url, + }); + return { provider: toPublicProvider(row, secret) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.patch<{ + Params: { id: string }; + Body: { + name?: string; + baseUrl?: string; + apiKey?: string; + defaultModel?: string; + enabled?: boolean; + }; + }>("/api/v1/me/llm-providers/:id", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const secret = ctx.cfg.llmProviderSecret; + if (!secret) { + return reply + .code(503) + .send({ error: "LLM_PROVIDER_SECRET not configured" }); + } + try { + const row = await updateLlmProvider(ctx.db, req.params.id, user.id, { + ...req.body, + secret, + }); + return { provider: toPublicProvider(row, secret) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (message === "not_found") { + return reply.code(404).send({ error: "not found" }); + } + return reply.code(400).send({ error: message }); + } + }); + + app.delete<{ Params: { id: string } }>( + "/api/v1/me/llm-providers/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const ok = await deleteLlmProvider(ctx.db, req.params.id, user.id); + if (!ok) return reply.code(404).send({ error: "not found" }); + await writeAudit(ctx.db, "llm_provider_delete", user.id, { + id: req.params.id, + }); + return { ok: true }; + }, + ); + + app.get("/api/v1/me/personas", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const [library, created] = await Promise.all([ + listUserPersonaLibrary(ctx.db, user.id), + listPersonasByOwner(ctx.db, user.id), + ]); + return { + library: library.map((p) => + personaPublicDto(p, { + enabled: p.enabled, + }), + ), + created: created.map((p) => + personaPublicDto(p, { + enabled: p.enabled, + }), + ), + }; + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/me/personas/:id/add", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + try { + const persona = await addPersonaToLibrary( + ctx.db, + user.id, + req.params.id, + ); + await writeAudit(ctx.db, "persona_added_lib", user.id, { + personaId: persona.id, + }); + return { ok: true, persona }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const code = + message.includes("private") || message.includes("not found") + ? 403 + : 400; + return reply.code(code).send({ error: message }); + } + }, + ); + + app.delete<{ Params: { id: string } }>( + "/api/v1/me/personas/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + await removePersonaFromLibrary(ctx.db, user.id, req.params.id); + return { ok: true }; + }, + ); + + // ── Web try-chat (persona preview, no WeChat) ───────── + + function tryChatHttpError( + reply: FastifyReply, + err: unknown, + ): FastifyReply | null { + if (!(err instanceof TryChatError)) return null; + const map: Record = { + disabled: 503, + not_found: 404, + forbidden: 403, + quota_day: 429, + quota_session: 429, + empty: 400, + llm: 502, + no_prompt: 400, + }; + return reply + .code(map[err.code] ?? 400) + .send({ error: err.code, message: err.message }); + } + + app.post<{ + Body: { personaId: string; botName?: string }; + }>("/api/v1/try-chat/sessions", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.tryChatEnabled) { + return reply + .code(503) + .send({ error: "disabled", message: "网页试聊已关闭" }); + } + const personaId = req.body?.personaId?.trim(); + if (!personaId) { + return reply.code(400).send({ error: "personaId required" }); + } + try { + // inLibrary rides along so the client doesn't need a follow-up + // GET /square/personas/:id just to render the "add" button state. + const [result, inLibrary] = await Promise.all([ + ctx.tryChat.startSession({ + userId: user.id, + personaId, + botName: req.body?.botName, + }), + isInPersonaLibrary(ctx.db, user.id, personaId).catch(() => false), + ]); + return { + sessionId: result.sessionId, + persona: { + id: result.persona.id, + slug: result.persona.slug, + displayName: result.persona.display_name, + description: result.persona.description, + inLibrary, + }, + inLibrary, + botName: result.botName, + expiresInSec: result.expiresInSec, + remainingToday: result.remainingToday, + }; + } catch (err) { + const sent = tryChatHttpError(reply, err); + if (sent) return; + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ + Params: { sessionId: string }; + Body: { text: string }; + }>("/api/v1/try-chat/sessions/:sessionId/messages", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + if (!ctx.cfg.tryChatEnabled) { + return reply + .code(503) + .send({ error: "disabled", message: "网页试聊已关闭" }); + } + try { + const result = await ctx.tryChat.sendMessage({ + userId: user.id, + sessionId: req.params.sessionId, + text: req.body?.text ?? "", + username: user.username, + }); + return { + messages: result.parts.map((p) => + p.kind === "sticker" + ? { type: "sticker" as const, slug: p.slug } + : { type: "text" as const, text: p.text }, + ), + displayText: result.displayText, + usage: result.usage, + remainingToday: result.remainingToday, + remainingSession: result.remainingSession, + }; + } catch (err) { + const sent = tryChatHttpError(reply, err); + if (sent) return; + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.delete<{ Params: { sessionId: string } }>( + "/api/v1/try-chat/sessions/:sessionId", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + try { + await ctx.tryChat.endSession(user.id, req.params.sessionId); + return { ok: true }; + } catch (err) { + const sent = tryChatHttpError(reply, err); + if (sent) return; + return { ok: true }; + } + }, + ); + + // ── Sticker Square ─────────────────────────────────── + + function publicStickerDto( + s: Sticker, + extra?: { inLibrary?: boolean }, + ) { + const isCdnEligible = + !!s.enabled && + s.visibility === "public" && + s.review_status === "approved"; + const imageUrl = isCdnEligible + ? `/cdn/s/${s.id}${s.content_hash ? `?v=${encodeURIComponent(s.content_hash)}` : ""}` + : `/api/v1/square/stickers/${s.id}/image`; + return { + id: s.id, + slug: s.slug, + displayName: s.display_name, + description: s.description, + tags: s.tags, + mime: s.mime, + sizeBytes: s.size_bytes, + visibility: s.visibility, + reviewStatus: s.review_status, + rejectReason: s.reject_reason, + ownerUserId: s.owner_user_id, + useCount: s.use_count, + enabled: !!s.enabled, + contentHash: s.content_hash ?? null, + imageUrl, + updatedAt: s.updated_at, + createdAt: s.created_at, + ...extra, + }; + } + + function parseStickerUpload( + body: { + mime?: string; + dataBase64?: string; + }, + maxBytes: number, + ): { data: Buffer; mime: string } { + if (!body.dataBase64) throw new StickerSecurityError("missing image", "empty"); + let data: Buffer; + try { + data = decodeBase64Image(body.dataBase64); + } catch { + throw new StickerSecurityError("invalid base64", "bad_base64"); + } + const { mime } = assertSafeStickerImage(data, body.mime, { maxBytes }); + return { data, mime }; + } + + app.get("/api/v1/square/stickers", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const q = req.query as { + q?: string; + page?: string; + limit?: string; + sort?: string; + }; + const limit = Math.min(Math.max(Number(q.limit ?? "20") || 20, 1), 50); + const page = Math.max(Number(q.page ?? "1") || 1, 1); + const offset = (page - 1) * limit; + const sort = + q.sort === "recent" || q.sort === "name" || q.sort === "use" + ? q.sort + : "use"; + const [{ items, total }, libIds] = await Promise.all([ + searchPublicStickers(ctx.db, { q: q.q, limit, offset, sort }), + getStickerLibraryIdSet(ctx.db, user.id), + ]); + const stickers = items.map((s) => + publicStickerDto(s, { + inLibrary: libIds.has(s.id), + }), + ); + return { stickers, total, page, limit }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/square/stickers/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const [s, inLibrary] = await Promise.all([ + getSticker(ctx.db, req.params.id), + isInStickerLibrary(ctx.db, user.id, req.params.id), + ]); + if (!s || !s.enabled) { + return reply.code(404).send({ error: "not found" }); + } + const isOwner = s.owner_user_id === user.id; + const isPublicApproved = + s.visibility === "public" && s.review_status === "approved"; + if (!isOwner && !isPublicApproved && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + return { sticker: publicStickerDto(s, { inLibrary }) }; + }, + ); + + app.get<{ Params: { id: string } }>( + "/api/v1/square/stickers/:id/image", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const s = await getSticker(ctx.db, req.params.id); + if (!s || !s.enabled) { + return reply.code(404).send({ error: "not found" }); + } + const isOwner = s.owner_user_id === user.id; + const isPublicApproved = + s.visibility === "public" && s.review_status === "approved"; + if (!isOwner && !isPublicApproved && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + return sendPrivateStickerImage(ctx, req, reply, s); + }, + ); + + app.post<{ + Body: { + slug?: string; + displayName: string; + description?: string; + tags?: string[]; + visibility?: "public" | "private"; + mime?: string; + dataBase64: string; + }; + }>("/api/v1/square/stickers", { bodyLimit: ctx.cfg.uploadBodyLimit }, async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const body = req.body ?? {}; + if (!body.displayName?.trim() || !body.dataBase64) { + return reply + .code(400) + .send({ error: "displayName and dataBase64 required" }); + } + try { + const { data, mime } = parseStickerUpload(body, ctx.cfg.stickerMaxBytes); + const sticker = await createSticker(ctx.db, { + slug: body.slug, + displayName: body.displayName, + description: body.description, + tags: Array.isArray(body.tags) ? body.tags : undefined, + visibility: body.visibility === "private" ? "private" : "public", + mime, + sizeBytes: data.length, + ownerUserId: user.id, + autoApprove: false, + data, + }); + await writeAudit(ctx.db, "sticker_submit", user.id, { + id: sticker.id, + visibility: sticker.visibility, + reviewStatus: sticker.review_status, + }); + return { sticker: publicStickerDto(sticker) }; + } catch (err) { + if (err instanceof StickerSecurityError) { + return reply + .code(400) + .send({ error: "unsafe_image", code: err.code, message: err.message }); + } + const message = err instanceof Error ? err.message : String(err); + const code = message.includes("slug") ? 409 : 400; + return reply.code(code).send({ error: message }); + } + }); + + app.put<{ + Params: { id: string }; + Body: { + slug?: string; + displayName?: string; + description?: string; + tags?: string[]; + visibility?: "public" | "private"; + mime?: string; + dataBase64?: string; + }; + }>("/api/v1/square/stickers/:id", { bodyLimit: ctx.cfg.uploadBodyLimit }, async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const cur = await getSticker(ctx.db, req.params.id); + if (!cur) return reply.code(404).send({ error: "not found" }); + if (cur.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + const body = req.body ?? {}; + try { + if (body.dataBase64) { + const { data, mime } = parseStickerUpload(body, ctx.cfg.stickerMaxBytes); + await replaceStickerBlob(ctx.db, cur.id, data, { + mime, + fileName: makeStickerFileName(cur.id, mime), + }); + } + const sticker = await updateStickerMeta(ctx.db, cur.id, { + slug: body.slug, + displayName: body.displayName, + description: body.description, + tags: body.tags, + visibility: body.visibility, + rePending: true, + }); + await writeAudit(ctx.db, "sticker_update_user", user.id, { + id: sticker.id, + }); + return { sticker: publicStickerDto(sticker) }; + } catch (err) { + if (err instanceof StickerSecurityError) { + return reply + .code(400) + .send({ error: "unsafe_image", code: err.code, message: err.message }); + } + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.delete<{ Params: { id: string } }>( + "/api/v1/square/stickers/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const cur = await getSticker(ctx.db, req.params.id); + if (!cur) return reply.code(404).send({ error: "not found" }); + if (cur.owner_user_id === "system") { + return reply.code(403).send({ error: "cannot delete system sticker" }); + } + if (cur.owner_user_id !== user.id && !user.is_admin) { + return reply.code(403).send({ error: "forbidden" }); + } + await softDeleteSticker(ctx.db, cur.id); + await writeAudit(ctx.db, "sticker_soft_delete", user.id, { id: cur.id }); + return { ok: true }; + }, + ); + + app.get("/api/v1/me/stickers", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const [library, created] = await Promise.all([ + listUserStickerLibrary(ctx.db, user.id), + listStickersByOwner(ctx.db, user.id), + ]); + return { + library: library.map((s) => publicStickerDto(s)), + created: created.map((s) => publicStickerDto(s)), + }; + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/me/stickers/:id/add", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + try { + const sticker = await addStickerToLibrary( + ctx.db, + user.id, + req.params.id, + ); + await writeAudit(ctx.db, "sticker_lib_add", user.id, { + stickerId: sticker.id, + }); + return { ok: true, sticker: publicStickerDto(sticker) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const code = + message.includes("private") || message.includes("not available") + ? 403 + : 400; + return reply.code(code).send({ error: message }); + } + }, + ); + + app.delete<{ Params: { id: string } }>( + "/api/v1/me/stickers/:id", + async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + await removeStickerFromLibrary(ctx.db, user.id, req.params.id); + return { ok: true }; + }, + ); + + // ── Admin ──────────────────────────────────────────── + + app.get("/api/v1/admin/dashboard", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + // Everything here is independent — one wave instead of 4 chained waits. + // redisOk doubles as the liveness probe, so it rides along too. + const [snap, today, yesterday, workerIds, workerStats, nodes, botOwners, redisOk] = + await Promise.all([ + doctorSnapshot(ctx.db), + getUsageDayStats(ctx.db, dayKey()), + getUsageDayStats(ctx.db, dayKeyOffset(-1)), + ctx.worker.listActiveBotIdsAsync(), + ctx.worker.getFleetStats(), + ctx.worker.listFleetNodes().catch(() => []), + listLeasedBots(ctx.db).catch(() => ({} as Record)), + ctx.db.ping().then( + () => true, + () => false, + ), + ]); + const botMap = await getBotAccountsByIds(ctx.db, workerIds); + const workerBots = workerIds.map((id) => { + const b = botMap.get(id); + return { + id, + displayName: b?.display_name ?? id, + status: b?.status ?? "unknown", + workerId: botOwners[id] ?? null, + }; + }); + const nodesOnline = nodes.filter((n) => n.online && !n.fenced).length; + return { + snapshot: snap, + usage: { today, yesterday }, + workers: workerIds, + workerBots, + workerId: ctx.worker.getWorkerId(), + workerStats, + nodes, + nodesOnline, + nodesTotal: nodes.length, + redisOk, + safeConfig: { + publicBaseUrl: ctx.cfg.publicBaseUrl, + workerEnabled: ctx.cfg.workerEnabled, + maxBotsPerWorker: ctx.cfg.maxBotsPerWorker, + replyConcurrency: ctx.cfg.replyConcurrency, + inboxMaxLen: ctx.cfg.inboxMaxLen, + llmModel: ctx.cfg.llmModel, + llmBaseUrl: ctx.cfg.llmBaseUrl, + multiBubbleJson: ctx.cfg.multiBubbleJson, + splitReply: ctx.cfg.splitReply, + allowUnapproved: ctx.cfg.allowUnapproved, + }, + }; + }); + + /** Global peer list for ops (filter unapproved by default via query). */ + app.get("/api/v1/admin/peers", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const q = req.query as { status?: string; limit?: string }; + const status = (q.status || "unapproved").toLowerCase(); + const limit = Math.min(Math.max(Number(q.limit) || 100, 1), 300); + let peers = await listPeers(ctx.db); + if (status === "unapproved") peers = peers.filter((p) => !p.approved); + else if (status === "approved") peers = peers.filter((p) => p.approved); + peers = peers.slice(0, limit); + const botIds = [...new Set(peers.map((p) => p.bot_account_id))]; + const [botMap, asgMap] = await Promise.all([ + getBotAccountsByIds(ctx.db, botIds), + getAssignmentsMany( + ctx.db, + peers.map((p) => ({ + botAccountId: p.bot_account_id, + peerId: p.peer_id, + })), + ), + ]); + const ownerIds = [ + ...new Set( + [...botMap.values()] + .map((b) => b.owner_user_id) + .filter(Boolean), + ), + ]; + const owners = await getUsersByIds(ctx.db, ownerIds); + const items = peers.map((p) => { + const bot = botMap.get(p.bot_account_id); + const owner = bot?.owner_user_id + ? owners.get(bot.owner_user_id) + : undefined; + return { + botAccountId: p.bot_account_id, + peerId: p.peer_id, + approved: Boolean(p.approved), + botName: bot?.display_name ?? p.bot_account_id, + ownerUserId: bot?.owner_user_id ?? null, + ownerUsername: owner?.username ?? null, + personaId: + asgMap.get(`${p.bot_account_id}|${p.peer_id}`) ?? null, + createdAt: p.created_at ?? null, + }; + }); + return { total: items.length, peers: items }; + }); + + app.post<{ + Body: { botAccountId: string; peerId: string }; + }>("/api/v1/admin/peers/approve", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const { botAccountId, peerId } = req.body ?? {}; + if (!botAccountId || !peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot) return reply.code(404).send({ error: "bot not found" }); + await approvePeer(ctx.db, botAccountId, peerId); + await writeAudit(ctx.db, "admin_peer_approve", admin.id, { + botAccountId, + peerId, + }); + return { ok: true }; + }); + + /** Approve all currently unapproved peers (ops convenience). */ + app.post("/api/v1/admin/peers/approve-all", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const peers = (await listPeers(ctx.db)).filter((p) => !p.approved); + // Batched, not serialized: 500 pending peers used to be ~1000 chained + // round trips (~40s, past most proxy timeouts). Still goes through + // approvePeer so each row is re-read — writing back the stale rows we + // already hold would clobber last_activity_at written by another node. + let approved = 0; + const BATCH = 100; + for (let off = 0; off < peers.length; off += BATCH) { + const slice = peers.slice(off, off + BATCH); + const res = await Promise.allSettled( + slice.map((p) => approvePeer(ctx.db, p.bot_account_id, p.peer_id)), + ); + approved += res.filter((r) => r.status === "fulfilled").length; + } + await writeAudit(ctx.db, "admin_peer_approve_all", admin.id, { + approved, + }); + return { ok: true, approved }; + }); + + app.get("/api/v1/admin/system", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + // Independent — one wave instead of three chained waits + const [snap, workerIds, workerStats, nodes, redisOk] = await Promise.all([ + doctorSnapshot(ctx.db), + ctx.worker.listActiveBotIdsAsync(), + ctx.worker.getFleetStats(), + ctx.worker.listFleetNodes().catch(() => []), + ctx.db.ping().then( + () => true, + () => false, + ), + ]); + return { + snapshot: snap, + workers: workerIds, + workerId: ctx.worker.getWorkerId(), + workerStats, + nodes, + nodesOnline: nodes.filter((n) => n.online && !n.fenced).length, + nodesTotal: nodes.length, + redisOk, + uptimeSec: Math.floor(process.uptime()), + node: process.version, + safeConfig: { + publicBaseUrl: ctx.cfg.publicBaseUrl, + workerEnabled: ctx.cfg.workerEnabled, + maxBotsPerWorker: ctx.cfg.maxBotsPerWorker, + replyConcurrency: ctx.cfg.replyConcurrency, + inboxMaxLen: ctx.cfg.inboxMaxLen, + llmModel: ctx.cfg.llmModel, + llmBaseUrl: ctx.cfg.llmBaseUrl, + multiBubbleJson: ctx.cfg.multiBubbleJson, + splitReply: ctx.cfg.splitReply, + allowUnapproved: ctx.cfg.allowUnapproved, + maxReplyChunks: ctx.cfg.maxReplyChunks, + maxChunkChars: ctx.cfg.maxChunkChars, + }, + }; + }); + + /** Deployment nodes — super-admin only. */ + app.get("/api/v1/admin/nodes", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + // targetShare divides the bots that actually want polling, so the panel's + // "目标" column matches what the fleet is really distributing. + const pollableTotal = await listPollableBotIds(ctx.db) + .then((ids) => ids.length) + .catch(() => undefined); + const [nodes, fences, weights, currentRelease] = await Promise.all([ + ctx.worker.listFleetNodes({ totalBots: pollableTotal }), + listWorkerFences(ctx.db).catch(() => []), + listWorkerWeights(ctx.db).catch(() => ({})), + getCurrentRelease(ctx.db).catch(() => null), + ]); + const desiredVersion = currentRelease?.version ?? null; + const statusMap = await getWorkerUpdateStatuses( + ctx.db, + nodes.map((n) => n.id), + ).catch(() => new Map()); + const enriched = nodes.map((n) => { + const st = statusMap.get(n.id) ?? null; + const outdated = Boolean( + desiredVersion && + (n.version || "") !== desiredVersion && + !n.fenced, + ); + return { + ...n, + update: { + outdated, + desiredVersion, + status: st?.phase ?? null, + error: st?.error ?? null, + progress: st?.progress ?? null, + message: st?.message ?? null, + targetVersion: st?.version ?? null, + }, + }; + }); + const online = nodes.filter((n) => n.online && !n.fenced); + return { + nodes: enriched, + nodesOnline: online.length, + nodesTotal: nodes.length, + nodesFenced: nodes.filter((n) => n.fenced).length, + nodesWeighted: nodes.filter((n) => n.weightOverride).length, + fences, + weights, + pollableTotal: pollableTotal ?? null, + /** Sum of weights across online nodes — the denominator of each share */ + weightTotal: online.reduce((a, n) => a + n.weight, 0), + weightLimits: { + min: MIN_WORKER_WEIGHT, + max: MAX_WORKER_WEIGHT, + default: DEFAULT_WORKER_WEIGHT, + }, + /** Weights only take effect while the rebalancer may move leases */ + rebalanceEnabled: ctx.cfg.rebalanceEnabled, + rebalanceIntervalSec: ctx.cfg.rebalanceIntervalSec, + /** Seconds a weight survives after its node stops heartbeating */ + weightTtlSec: ctx.cfg.workerWeightTtlSec, + selfWorkerId: ctx.worker.getWorkerId(), + leaseTtlSec: ctx.cfg.leaseTtlSec, + release: releaseSummary(currentRelease), + appVersion: ctx.cfg.appVersion, + otaEnabled: ctx.cfg.otaEnabled, + }; + }); + + /** + * Set a node's load weight (percent of an even share; 100 = default). + * + * Weights are relative between *online* nodes: with A=200 and B=100 the + * fleet aims for a 2:1 split. 0 drains a node without fencing it — it keeps + * heartbeating and can be restored instantly. + */ + app.post<{ + Params: { workerId: string }; + Body: { weight?: number | string; percent?: number | string }; + }>("/api/v1/admin/nodes/:workerId/weight", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + const parsed = parseWorkerWeightInput( + req.body?.weight ?? req.body?.percent, + ); + if (!parsed.ok) { + return reply.code(400).send({ + error: parsed.error, + message: + parsed.error === "weight_required" + ? "weight required" + : parsed.error === "weight_out_of_range" + ? `weight must be between ${MIN_WORKER_WEIGHT} and ${MAX_WORKER_WEIGHT}` + : "weight must be a number", + }); + } + const weight = parsed.value; + const record = await setWorkerWeight(ctx.db, workerId, weight, { + byUserId: admin.id, + byUsername: admin.username ?? null, + }); + await writeAudit(ctx.db, "admin_node_set_weight", admin.id, { + workerId, + weight, + }); + try { + // Nodes drop their cached weight on wake, so this lands next tick + await publishWorkerWake(ctx.db); + } catch { + /* optional */ + } + const note = ctx.cfg.rebalanceEnabled + ? `约 ${ctx.cfg.rebalanceIntervalSec}s 内生效` + : "但 REBALANCE_ENABLED=false，租约不会自动迁移"; + return { + ok: true, + workerId, + weight, + cleared: record === null, + message: + weight === DEFAULT_WORKER_WEIGHT + ? `已恢复默认权重（100%）；${note}` + : weight === 0 + ? `已设为 0%（腾空节点，不再认领 bot）；${note}` + : `已设为 ${weight}%；${note}`, + }; + }); + + /** Remove the override so the node returns to an even share. */ + app.delete<{ Params: { workerId: string } }>( + "/api/v1/admin/nodes/:workerId/weight", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + const cleared = await clearWorkerWeight(ctx.db, workerId); + await writeAudit(ctx.db, "admin_node_clear_weight", admin.id, { + workerId, + cleared, + }); + try { + await publishWorkerWake(ctx.db); + } catch { + /* optional */ + } + return { + ok: true, + workerId, + cleared, + weight: DEFAULT_WORKER_WEIGHT, + message: cleared ? "已恢复默认权重（100%）" : "该节点没有权重覆盖", + }; + }, + ); + + /** Drop weight overrides for nodes that are gone (neither online nor fenced). */ + app.post("/api/v1/admin/nodes/weights/prune", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const removed = await pruneWorkerWeights(ctx.db); + if (removed.length) { + await writeAudit(ctx.db, "admin_node_prune_weights", admin.id, { + removed, + }); + } + return { + ok: true, + removed, + message: removed.length + ? `已清理 ${removed.length} 个失效节点的权重` + : "没有失效的权重记录", + }; + }); + + // ── OTA releases (super-admin) ───────────────────────── + + app.get("/api/v1/admin/releases/current", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const cur = await getCurrentRelease(ctx.db); + return { + release: cur, + summary: releaseSummary(cur), + appVersion: ctx.cfg.appVersion, + otaEnabled: ctx.cfg.otaEnabled, + }; + }); + + app.get("/api/v1/admin/releases", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const q = req.query as { limit?: string }; + const limit = Number(q.limit ?? "20"); + const [versions, current] = await Promise.all([ + listReleaseVersions(ctx.db, limit), + getCurrentRelease(ctx.db), + ]); + return { + versions, + currentVersion: current?.version ?? null, + appVersion: ctx.cfg.appVersion, + }; + }); + + /** + * Publish release: either full JSON (small packs) or multi-step blob upload. + * + * Body modes: + * 1) { mode: "blob", sha256, dataBase64 } — upload one content-addressed file + * 2) { mode: "publish", version, files: [{path,sha256,size}], setCurrent?, createdBy? } + * (blobs must already exist) + * 3) { mode: "pack", version, files: [{path,sha256,size,dataBase64}], setCurrent? } + * — inline file bodies (CLI convenience for moderate packs) + */ + app.post<{ + Body: { + mode?: string; + sha256?: string; + dataBase64?: string; + version?: string; + files?: Array<{ + path: string; + sha256: string; + size: number; + dataBase64?: string; + }>; + setCurrent?: boolean; + }; + }>("/api/v1/admin/releases", { bodyLimit: ctx.cfg.uploadBodyLimit }, async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const body = req.body || {}; + const mode = (body.mode || "pack").trim(); + + try { + if (mode === "blob") { + const sha = (body.sha256 || "").trim().toLowerCase(); + const b64 = body.dataBase64 || ""; + if (!/^[a-f0-9]{64}$/.test(sha) || !b64) { + return reply.code(400).send({ error: "sha256_and_dataBase64_required" }); + } + const data = Buffer.from(b64, "base64"); + if (sha256Buffer(data) !== sha) { + return reply.code(400).send({ error: "sha256_mismatch" }); + } + if (data.length > 8 * 1024 * 1024) { + return reply.code(400).send({ error: "blob_too_large", max: 8 * 1024 * 1024 }); + } + const meta = await putBlobChunks(ctx.db, sha, data); + return { ok: true, blob: meta, existed: await blobExists(ctx.db, sha) }; + } + + if (mode === "publish") { + const files = (body.files || []).map((f) => ({ + path: f.path, + sha256: f.sha256, + size: f.size, + })); + const meta = buildReleaseMeta({ + version: body.version || "", + files, + createdBy: admin.username || admin.id, + }); + await publishRelease(ctx.db, meta, { + setCurrent: body.setCurrent !== false, + }); + await writeAudit(ctx.db, "admin_release_publish", admin.id, { + version: meta.version, + fileCount: meta.fileCount, + totalBytes: meta.totalBytes, + }); + return { ok: true, release: meta, summary: releaseSummary(meta) }; + } + + // mode === pack: files with inline dataBase64 + if (mode === "pack") { + const rawFiles = body.files || []; + if (!rawFiles.length) { + return reply.code(400).send({ error: "files_required" }); + } + const entries: ReleaseFileEntry[] = []; + for (const f of rawFiles) { + if (!f.dataBase64) { + return reply + .code(400) + .send({ error: "dataBase64_required", path: f.path }); + } + const data = Buffer.from(f.dataBase64, "base64"); + if (data.length > 8 * 1024 * 1024) { + return reply + .code(400) + .send({ error: "file_too_large", path: f.path }); + } + const hash = sha256Buffer(data); + if (f.sha256 && f.sha256.toLowerCase() !== hash) { + return reply + .code(400) + .send({ error: "sha256_mismatch", path: f.path }); + } + await putBlobChunks(ctx.db, hash, data); + entries.push({ + path: f.path, + sha256: hash, + size: data.length, + }); + } + const meta = buildReleaseMeta({ + version: body.version || "", + files: entries, + createdBy: admin.username || admin.id, + }); + await publishRelease(ctx.db, meta, { + setCurrent: body.setCurrent !== false, + }); + await writeAudit(ctx.db, "admin_release_publish", admin.id, { + version: meta.version, + fileCount: meta.fileCount, + totalBytes: meta.totalBytes, + mode: "pack", + }); + return { ok: true, release: meta, summary: releaseSummary(meta) }; + } + + return reply.code(400).send({ error: "unknown_mode", mode }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ Body: { version?: string } }>( + "/api/v1/admin/releases/current", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const version = (req.body?.version || "").trim(); + if (!version) { + return reply.code(400).send({ error: "version required" }); + } + try { + const meta = await setCurrentRelease(ctx.db, version); + await writeAudit(ctx.db, "admin_release_set_current", admin.id, { + version, + }); + return { ok: true, release: meta, summary: releaseSummary(meta) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }, + ); + + app.post<{ + Params: { workerId: string }; + Body: { version?: string; force?: boolean; confirm?: string }; + }>("/api/v1/admin/nodes/:workerId/update", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.cfg.otaEnabled) { + return reply.code(400).send({ error: "OTA_ENABLED=false" }); + } + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + const confirm = (req.body?.confirm || "").trim(); + if (confirm && confirm !== workerId) { + return reply.code(400).send({ + error: "confirm_mismatch", + message: "confirm must equal workerId", + }); + } + let version = (req.body?.version || "").trim(); + if (!version) { + const cur = await getCurrentRelease(ctx.db); + if (!cur) { + return reply + .code(400) + .send({ error: "no_current_release", message: "先发布 release 包" }); + } + version = cur.version; + } else { + const meta = await getReleaseMeta(ctx.db, version); + if (!meta) { + return reply.code(400).send({ error: "release_not_found" }); + } + } + try { + const status = await enqueueWorkerUpdate(ctx.db, workerId, { + version, + requestedAt: new Date().toISOString(), + requestedBy: admin.id, + requestedByUsername: admin.username ?? null, + force: Boolean(req.body?.force), + }); + await writeAudit(ctx.db, "admin_node_update", admin.id, { + workerId, + version, + force: Boolean(req.body?.force), + }); + try { + await publishWorkerWake(ctx.db); + } catch { + /* */ + } + return { + ok: true, + workerId, + version, + status, + message: `已下发更新任务 → ${version}，节点将在下一心跳周期内应用并重启`, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ + Body: { version?: string; force?: boolean }; + }>("/api/v1/admin/nodes/update-outdated", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.cfg.otaEnabled) { + return reply.code(400).send({ error: "OTA_ENABLED=false" }); + } + let version = (req.body?.version || "").trim(); + if (!version) { + const cur = await getCurrentRelease(ctx.db); + if (!cur) { + return reply.code(400).send({ error: "no_current_release" }); + } + version = cur.version; + } + const nodes = await ctx.worker.listFleetNodes(); + const targets = nodes.filter( + (n) => + n.online && + !n.fenced && + (req.body?.force || !n.version || n.version !== version), + ); + const results: Array<{ workerId: string; ok: boolean; error?: string }> = + []; + for (const n of targets) { + try { + await enqueueWorkerUpdate(ctx.db, n.id, { + version, + requestedAt: new Date().toISOString(), + requestedBy: admin.id, + requestedByUsername: admin.username ?? null, + force: Boolean(req.body?.force), + }); + results.push({ workerId: n.id, ok: true }); + } catch (err) { + results.push({ + workerId: n.id, + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + } + } + await writeAudit(ctx.db, "admin_nodes_update_outdated", admin.id, { + version, + count: results.filter((r) => r.ok).length, + targets: results.map((r) => r.workerId), + }); + try { + await publishWorkerWake(ctx.db); + } catch { + /* */ + } + return { + ok: true, + version, + queued: results.filter((r) => r.ok).length, + results, + }; + }); + + app.get<{ Params: { workerId: string } }>( + "/api/v1/admin/nodes/:workerId/update-status", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + const status = await getWorkerUpdateStatus(ctx.db, workerId); + return { workerId, status }; + }, + ); + + /** + * Force a deployment node offline (super-admin only). + */ + app.post<{ + Params: { workerId: string }; + Body: { reason?: string; confirm?: string }; + }>("/api/v1/admin/nodes/:workerId/force-offline", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + // Require confirm match to avoid misclick (UI sends confirm=workerId) + const confirm = (req.body?.confirm || "").trim(); + if (confirm && confirm !== workerId) { + return reply.code(400).send({ + error: "confirm_mismatch", + message: "confirm must equal workerId", + }); + } + if (workerId === ctx.worker.getWorkerId()) { + // Allow fencing self (drain this node via another admin path), but warn + // — request may hang if this process is the only one; still OK for multi-node. + } + try { + const { released, fence } = await forceOfflineWorker(ctx.db, workerId, { + reason: req.body?.reason, + byUserId: admin.id, + byUsername: admin.username ?? null, + }); + await writeAudit(ctx.db, "admin_node_force_offline", admin.id, { + workerId, + released, + reason: fence.reason, + }); + return { + ok: true, + workerId, + released, + fence, + message: + released > 0 + ? `已强制下线，释放 ${released} 个 bot 租约；目标进程将停止认领` + : "已强制下线（无租约或已释放）；目标进程将停止认领", + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + /** Clear force-offline fence so the node may rejoin the fleet. */ + app.post<{ Params: { workerId: string } }>( + "/api/v1/admin/nodes/:workerId/clear-fence", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const workerId = decodeURIComponent(req.params.workerId || "").trim(); + if (!workerId) { + return reply.code(400).send({ error: "workerId required" }); + } + const cleared = await clearWorkerFence(ctx.db, workerId); + await writeAudit(ctx.db, "admin_node_clear_fence", admin.id, { + workerId, + cleared, + }); + try { + await publishWorkerWake(ctx.db); + } catch { + /* optional */ + } + return { + ok: true, + workerId, + cleared, + message: cleared + ? "已解除下线封锁，节点将在下一心跳重新加入" + : "该节点没有封锁记录", + }; + }, + ); + + /** Start/restart workers for every active bot that has Redis credentials. */ + app.post("/api/v1/admin/workers/restart-all", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.cfg.workerEnabled) { + return reply.code(400).send({ error: "WORKER_ENABLED=false" }); + } + const bots = await listBotAccounts(ctx.db); + const tokenMap = await hasBotCredentialsMany( + ctx.db, + bots.map((b) => b.id), + ); + let started = 0; + let skipped = 0; + for (const b of bots) { + if (b.status !== "active") { + skipped++; + continue; + } + if (!tokenMap[b.id]) { + skipped++; + continue; + } + ctx.worker.restartBot(b.id); + started++; + } + await writeAudit(ctx.db, "admin_workers_restart_all", admin.id, { + started, + skipped, + }); + return { + ok: true, + started, + skipped, + workers: await ctx.worker.listActiveBotIdsAsync(), + }; + }); + + app.post("/api/v1/admin/workers/stop-all", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const before = await ctx.worker.stopAllBots(); + await writeAudit(ctx.db, "admin_workers_stop_all", admin.id, { + count: before.length, + botIds: before, + }); + return { ok: true, stopped: before.length }; + }); + + /** Idempotent official persona seed (catgirl / girlfriend if missing). */ + app.post("/api/v1/admin/system/seed-personas", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const before = (await listPersonas(ctx.db)).length; + await seedPersonas(ctx.db); + const afterList = await listPersonas(ctx.db); + const after = afterList.length; + await writeAudit(ctx.db, "admin_seed_personas", admin.id, { + before, + after, + added: Math.max(0, after - before), + }); + return { + ok: true, + before, + after, + added: Math.max(0, after - before), + defaultPersona: + afterList.find((p) => p.is_default)?.slug ?? + afterList.find((p) => p.enabled)?.slug ?? + null, + }; + }); + + app.get("/api/v1/admin/memories", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const q = req.query as { + botAccountId?: string; + peerId?: string; + personaId?: string; + }; + if (!q.botAccountId || !q.peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, q.botAccountId); + if (!bot) return reply.code(404).send({ error: "bot not found" }); + if (q.personaId) { + return { + memories: await listMemories( + ctx.db, + q.botAccountId, + q.peerId, + q.personaId, + ), + }; + } + // All personas for this peer — one pipelined wave, not one LRANGE each + const personas = await listPersonas(ctx.db); + const memMap = await listMemoriesMany( + ctx.db, + q.botAccountId, + q.peerId, + personas.map((p) => p.id), + ); + const groups: { + personaId: string; + personaName: string; + memories: { id: string; content: string }[]; + }[] = []; + let total = 0; + for (const p of personas) { + const mems = memMap.get(p.id); + if (mems?.length) { + groups.push({ + personaId: p.id, + personaName: p.display_name, + memories: mems.map((m) => ({ id: m.id, content: m.content })), + }); + total += mems.length; + } + } + return { total, groups }; + }); + + app.post<{ + Body: { botAccountId: string; peerId: string; personaId?: string }; + }>("/api/v1/admin/memories/reset", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const { botAccountId, peerId, personaId } = req.body ?? {}; + if (!botAccountId || !peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot) return reply.code(404).send({ error: "bot not found" }); + await clearMemories(ctx.db, botAccountId, peerId, personaId); + await writeAudit(ctx.db, "admin_memory_reset", admin.id, { + botAccountId, + peerId, + personaId: personaId || null, + }); + return { ok: true }; + }); + + app.post<{ + Body: { botAccountId: string; peerId: string }; + }>("/api/v1/admin/messages/clear", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const { botAccountId, peerId } = req.body ?? {}; + if (!botAccountId || !peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot) return reply.code(404).send({ error: "bot not found" }); + await clearMessages(ctx.db, botAccountId, peerId); + await writeAudit(ctx.db, "admin_messages_clear", admin.id, { + botAccountId, + peerId, + }); + return { ok: true }; + }); + + app.get("/api/v1/admin/users", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const users = await listUsers(ctx.db); + const [botCounts, superAdminId] = await Promise.all([ + countBotsByOwners( + ctx.db, + users.map((u) => u.id), + ), + resolveSuperAdminId(ctx.db), + ]); + const mapped = users.map((u) => ({ + id: u.id, + username: u.username, + name: u.name, + isAdmin: Boolean(u.is_admin), + isSuperAdmin: Boolean(superAdminId && u.id === superAdminId), + trustLevel: u.trust_level, + avatarUrl: u.avatar_url, + botCount: botCounts[u.id] ?? 0, + createdAt: u.created_at, + authProvider: u.auth_provider === "local" ? "local" : "linuxdo", + isBanned: Boolean(u.is_banned), + bannedAt: u.banned_at ?? null, + bannedReason: u.banned_reason ?? null, + invitedBy: u.invited_by ?? null, + })); + return { users: mapped }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/admin/users/:id", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const u = await getUser(ctx.db, req.params.id); + if (!u) return reply.code(404).send({ error: "not found" }); + const bots = await listBotsByOwner(ctx.db, u.id); + const [active, tokenMap] = await Promise.all([ + ctx.worker.listActiveBotIdsAsync().then((ids) => new Set(ids)), + hasBotCredentialsMany( + ctx.db, + bots.map((b) => b.id), + ), + ]); + return { + user: { + ...userPublicFields(u), + invitedBy: u.invited_by ?? null, + inviteCodeUsed: u.invite_code_used ?? null, + }, + bots: bots.map((b) => ({ + id: b.id, + displayName: b.display_name, + status: b.status, + hasToken: Boolean(tokenMap[b.id]), + workerActive: active.has(b.id), + })), + }; + }, + ); + + app.patch<{ + Params: { id: string }; + Body: { isAdmin?: boolean }; + }>("/api/v1/admin/users/:id", async (req, reply) => { + // Grant/revoke admin is super-admin only. + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const target = await getUser(ctx.db, req.params.id); + if (!target) return reply.code(404).send({ error: "not found" }); + if (typeof req.body?.isAdmin !== "boolean") { + return reply.code(400).send({ error: "isAdmin boolean required" }); + } + const next = req.body.isAdmin; + if (!next && target.id === admin.id) { + return reply.code(400).send({ error: "cannot revoke your own admin" }); + } + // Super-admin (earliest-created admin) cannot be demoted by anyone. + if (!next && target.is_admin && (await isSuperAdmin(ctx.db, target.id))) { + return reply.code(400).send({ + error: "cannot_revoke_super_admin", + message: "系统首位管理员（超管）不可撤销", + }); + } + if (!next && target.is_admin) { + const all = await listUsers(ctx.db); + const adminCount = all.filter((u) => u.is_admin).length; + if (adminCount <= 1) { + return reply + .code(400) + .send({ error: "cannot revoke the last admin" }); + } + } + const updated = await setUserAdmin(ctx.db, target.id, next); + await writeAudit( + ctx.db, + next ? "user_admin_grant" : "user_admin_revoke", + admin.id, + { userId: target.id, username: target.username }, + ); + return { + user: userPublicFields(updated), + }; + }); + + app.post<{ + Params: { id: string }; + Body: { reason?: string; cascadeBots?: boolean }; + }>("/api/v1/admin/users/:id/ban", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const target = await getUser(ctx.db, req.params.id); + if (!target) return reply.code(404).send({ error: "not found" }); + if (target.id === admin.id) { + return reply.code(400).send({ error: "cannot ban yourself" }); + } + // Super-admin cannot be banned by anyone (including other admins). + if (target.is_admin && (await isSuperAdmin(ctx.db, target.id))) { + return reply.code(400).send({ + error: "cannot_ban_super_admin", + message: "系统首位管理员（超管）不可封禁", + }); + } + if (target.is_admin) { + const all = await listUsers(ctx.db); + const adminCount = all.filter((u) => u.is_admin && !u.is_banned).length; + if (adminCount <= 1) { + return reply.code(400).send({ error: "cannot ban the last admin" }); + } + } + const updated = await setUserBanned(ctx.db, target.id, true, { + reason: req.body?.reason, + actorId: admin.id, + }); + await destroyAllSessionsForUser(ctx.db, target.id); + // Default cascade: deactivate all bots + const cascade = req.body?.cascadeBots !== false; + if (cascade) { + const bots = await listBotsByOwner(ctx.db, target.id); + for (const b of bots) { + if (b.status === "active") { + await setBotStatus(ctx.db, b.id, "inactive"); + ctx.worker.stopBot(b.id); + } + } + } + await writeAudit(ctx.db, "user_banned", admin.id, { + userId: target.id, + username: target.username, + reason: req.body?.reason ?? null, + cascadeBots: cascade, + }); + return { user: userPublicFields(updated) }; + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/users/:id/unban", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const target = await getUser(ctx.db, req.params.id); + if (!target) return reply.code(404).send({ error: "not found" }); + const updated = await setUserBanned(ctx.db, target.id, false); + await writeAudit(ctx.db, "user_unbanned", admin.id, { + userId: target.id, + username: target.username, + }); + return { user: userPublicFields(updated) }; + }, + ); + + app.delete<{ + Params: { id: string }; + Querystring: { confirm?: string }; + }>("/api/v1/admin/users/:id", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const target = await getUser(ctx.db, req.params.id); + if (!target) return reply.code(404).send({ error: "not found" }); + if (target.id === admin.id) { + return reply.code(400).send({ error: "cannot delete yourself" }); + } + // Super-admin = earliest-created admin; cannot be deleted by anyone. + if (target.is_admin && (await isSuperAdmin(ctx.db, target.id))) { + return reply.code(400).send({ + error: "cannot_delete_super_admin", + message: "系统首位管理员（超管）不可删除", + }); + } + if (target.is_admin) { + const all = await listUsers(ctx.db); + const adminCount = all.filter((u) => u.is_admin).length; + if (adminCount <= 1) { + return reply.code(400).send({ error: "cannot delete the last admin" }); + } + } + const confirm = (req.query?.confirm || "").trim(); + if (!confirm || confirm.toLowerCase() !== target.username.toLowerCase()) { + return reply.code(400).send({ + error: "confirm_required", + message: "请在 query confirm= 中传入目标用户名以确认删除", + }); + } + // Stop workers for owned bots before delete + const bots = await listBotsByOwner(ctx.db, target.id); + for (const b of bots) { + ctx.worker.stopBot(b.id); + } + const ok = await deleteUserAccount(ctx.db, target.id); + if (!ok) return reply.code(404).send({ error: "not found" }); + await writeAudit(ctx.db, "user_deleted", admin.id, { + userId: target.id, + username: target.username, + }); + return { ok: true }; + }); + + // Invite settings (admin) + app.get("/api/v1/admin/settings/invites", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const settings = await getInviteSettings( + ctx.db, + inviteDefaultsFromCfg(ctx.cfg), + ); + return { settings }; + }); + + app.patch<{ + Body: { + quotaWindowHours?: number; + quotaMax?: number; + codeTtlSec?: number; + maxPendingPerUser?: number; + codeLength?: number; + }; + }>("/api/v1/admin/settings/invites", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const body = req.body || {}; + const settings = await setInviteSettings( + ctx.db, + { + quotaWindowHours: body.quotaWindowHours, + quotaMax: body.quotaMax, + codeTtlSec: body.codeTtlSec, + maxPendingPerUser: body.maxPendingPerUser, + codeLength: body.codeLength, + }, + inviteDefaultsFromCfg(ctx.cfg), + ); + await writeAudit(ctx.db, "invite_settings_updated", admin.id, { + settings, + }); + return { settings }; + }); + + // ── Runtime config (super-admin-only env overrides, stored in Redis) ── + // Read included: the payload names every tuning knob and which ones are + // overridden, and this surface can disable the worker fleet-wide. + app.get("/api/v1/admin/settings/runtime", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.settings) { + return reply + .code(503) + .send({ error: "runtime settings manager not initialized" }); + } + return { + ...ctx.settings.view(), + warnings: ctx.settings.currentWarnings(), + canEdit: true, + }; + }); + + app.patch<{ + Body: { + patch?: Record; + reset?: string[]; + }; + }>("/api/v1/admin/settings/runtime", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.settings) { + return reply + .code(503) + .send({ error: "runtime settings manager not initialized" }); + } + const body = req.body || {}; + let result; + try { + result = await ctx.settings.patch({ + patch: body.patch, + reset: Array.isArray(body.reset) ? body.reset : undefined, + actor: admin.username || admin.id, + }); + } catch (err) { + // Never write a merge derived from a failed read — that would wipe every + // other override fleet-wide. Ask the admin to retry instead. + if (err instanceof RuntimeSettingsUnavailableError) { + return reply.code(503).send({ error: err.message }); + } + throw err; + } + if (result.changed.length) { + await writeAudit(ctx.db, "runtime_settings_updated", admin.id, { + changed: result.changed, + restartRequired: result.restartRequired, + }); + } + return { + ...result.view, + warnings: result.warnings, + changed: result.changed, + restartRequired: result.restartRequired, + canEdit: true, + }; + }); + + app.post("/api/v1/admin/settings/runtime/reset", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.settings) { + return reply + .code(503) + .send({ error: "runtime settings manager not initialized" }); + } + const result = await ctx.settings.patch({ + resetAll: true, + actor: admin.username || admin.id, + }); + await writeAudit(ctx.db, "runtime_settings_reset", admin.id, { + changed: result.changed, + }); + return { + ...result.view, + warnings: result.warnings, + changed: result.changed, + restartRequired: result.restartRequired, + canEdit: true, + }; + }); + + app.get("/api/v1/admin/bots", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + // listActiveBotIdsAsync is itself 2 chained RTTs — don't chain it after + // the bot listing as well. + const [bots, activeIds] = await Promise.all([ + listBotAccounts(ctx.db), + ctx.worker.listActiveBotIdsAsync(), + ]); + const active = new Set(activeIds); + const botIds = bots.map((b) => b.id); + const [owners, tokenMap, peerStats] = await Promise.all([ + getUsersByIds( + ctx.db, + bots.map((b) => b.owner_user_id).filter(Boolean), + ), + hasBotCredentialsMany(ctx.db, botIds), + peerStatsByBots(ctx.db, botIds), + ]); + const mapped = bots.map((b) => { + const owner = b.owner_user_id + ? owners.get(b.owner_user_id) + : undefined; + const stats = peerStats[b.id] ?? { + peerCount: 0, + unapprovedPeerCount: 0, + }; + return { + id: b.id, + displayName: b.display_name, + ownerUserId: b.owner_user_id, + ownerUsername: owner?.username ?? null, + ownerName: owner?.name ?? null, + status: b.status, + accountRef: b.account_ref, + workerActive: active.has(b.id), + hasToken: Boolean(tokenMap[b.id]), + peerCount: stats.peerCount, + unapprovedPeerCount: stats.unapprovedPeerCount, + updatedAt: b.updated_at, + }; + }); + return { bots: mapped }; + }); + + app.get<{ Params: { botId: string } }>( + "/api/v1/admin/bots/:botId", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + const owner = bot.owner_user_id + ? await getUser(ctx.db, bot.owner_user_id) + : undefined; + const peers = await listPeers(ctx.db, bot.id); + const [active, hasToken, asgMap] = await Promise.all([ + ctx.worker.listActiveBotIdsAsync().then((ids) => new Set(ids)), + hasBotCredentials(ctx.db, bot.id), + getAssignmentsMany( + ctx.db, + peers.map((p) => ({ + botAccountId: p.bot_account_id, + peerId: p.peer_id, + })), + ), + ]); + // Message counts / last message still per-peer but parallelized + const peerItems = await Promise.all( + peers.map(async (p) => { + const [msgCount, recent] = await Promise.all([ + countUserMessages(ctx.db, p.bot_account_id, p.peer_id), + listRecentMessages(ctx.db, p.bot_account_id, p.peer_id, 1), + ]); + const last = recent[recent.length - 1]; + return { + peerId: p.peer_id, + approved: Boolean(p.approved), + personaId: + asgMap.get(`${p.bot_account_id}|${p.peer_id}`) ?? null, + createdAt: p.created_at ?? null, + messageCount: msgCount, + lastMessageAt: last?.created_at ?? null, + lastRole: last?.role ?? null, + }; + }), + ); + return { + bot: { + id: bot.id, + displayName: bot.display_name, + ownerUserId: bot.owner_user_id, + ownerUsername: owner?.username ?? null, + status: bot.status, + accountRef: bot.account_ref, + baseUrl: bot.base_url, + workerActive: active.has(bot.id), + hasToken, + updatesCursor: bot.updates_cursor ? "set" : "empty", + createdAt: bot.created_at, + updatedAt: bot.updated_at, + }, + peers: peerItems, + }; + }, + ); + + app.patch<{ + Params: { botId: string }; + Body: { displayName?: string; status?: "active" | "inactive" }; + }>("/api/v1/admin/bots/:botId", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + const displayName = req.body?.displayName; + const status = req.body?.status; + if (!displayName?.trim() && status !== "active" && status !== "inactive") { + return reply + .code(400) + .send({ error: "displayName or status required" }); + } + try { + let updated = bot; + if (displayName?.trim()) { + updated = await updateBotDisplayName(ctx.db, bot.id, displayName); + await writeAudit(ctx.db, "admin_bot_renamed", admin.id, { + botId: bot.id, + displayName: updated.display_name, + }); + } + if (status === "active" || status === "inactive") { + updated = await setBotStatus(ctx.db, bot.id, status); + if (status === "inactive") { + ctx.worker.stopBot(bot.id); + } else if (await hasBotCredentials(ctx.db, bot.id)) { + // Always mark pollable; worker process claims if enabled + ctx.worker.restartBot(bot.id); + } + await writeAudit(ctx.db, "admin_bot_status", admin.id, { + botId: bot.id, + status, + }); + } + return { + bot: { + id: updated.id, + displayName: updated.display_name, + status: updated.status, + ownerUserId: updated.owner_user_id, + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ Params: { botId: string } }>( + "/api/v1/admin/bots/:botId/stop-worker", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + ctx.worker.stopBot(bot.id); + await writeAudit(ctx.db, "admin_bot_stop_worker", admin.id, { + botId: bot.id, + }); + return { ok: true }; + }, + ); + + app.post<{ Params: { botId: string } }>( + "/api/v1/admin/bots/:botId/start-worker", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + if (bot.status !== "active") { + return reply.code(400).send({ error: "bot is not active" }); + } + if (!(await hasBotCredentials(ctx.db, bot.id))) { + return reply + .code(400) + .send({ error: "missing token — user must re-scan login" }); + } + ctx.worker.restartBot(bot.id); + await writeAudit(ctx.db, "admin_bot_start_worker", admin.id, { + botId: bot.id, + }); + return { ok: true, workerActive: true }; + }, + ); + + app.delete<{ Params: { botId: string } }>( + "/api/v1/admin/bots/:botId", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + ctx.worker.stopBot(bot.id); + await deleteBotAccount(ctx.db, bot.id); + await writeAudit(ctx.db, "admin_bot_deleted", admin.id, { + botId: bot.id, + ownerUserId: bot.owner_user_id, + }); + return { ok: true }; + }, + ); + + // ── Admin broadcast (async text push) ────────────────── + + function publicBroadcastJob(job: Awaited>) { + if (!job) return null; + return { + id: job.id, + createdBy: job.createdBy, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + status: job.status, + text: job.text, + scope: job.scope, + botIds: job.botIds, + targetCount: job.targets?.length ?? 0, + stats: job.stats, + error: job.error ?? null, + startedAt: job.startedAt ?? null, + finishedAt: job.finishedAt ?? null, + failures: job.failures ?? [], + cursor: job.cursor ?? 0, + recipientCount: job.recipients?.length ?? 0, + }; + } + + app.get("/api/v1/admin/broadcast", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const limit = Number((req.query as { limit?: string }).limit ?? "30"); + const jobs = await listBroadcastJobs(ctx.db, Math.min(100, Math.max(1, limit))); + return { jobs: jobs.map((j) => publicBroadcastJob(j)) }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/admin/broadcast/:id", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const job = await getBroadcastJob(ctx.db, req.params.id); + if (!job) return reply.code(404).send({ error: "not found" }); + return { job: publicBroadcastJob(job) }; + }, + ); + + app.post<{ + Body: { + text?: string; + scope?: BroadcastScope; + botIds?: string[]; + targets?: BroadcastTarget[]; + /** If true, only expand & return counts — do not create */ + preview?: boolean; + }; + }>("/api/v1/admin/broadcast", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + + const text = (req.body?.text ?? "").trim(); + const maxText = Math.max(1, ctx.cfg.broadcastMaxText || 2000); + if (!text) { + return reply.code(400).send({ error: "text required" }); + } + if (text.length > maxText) { + return reply + .code(400) + .send({ error: `text too long (max ${maxText})` }); + } + + const scope = req.body?.scope; + if (scope !== "all_bots" && scope !== "bots" && scope !== "targets") { + return reply + .code(400) + .send({ error: "scope must be all_bots | bots | targets" }); + } + + const botIds = Array.isArray(req.body?.botIds) + ? req.body!.botIds.map(String) + : []; + const targets = Array.isArray(req.body?.targets) + ? req.body!.targets + .filter( + (t) => + t && + typeof t === "object" && + typeof (t as BroadcastTarget).botId === "string" && + typeof (t as BroadcastTarget).peerId === "string", + ) + .map((t) => ({ + botId: String((t as BroadcastTarget).botId).trim(), + peerId: String((t as BroadcastTarget).peerId).trim(), + })) + : []; + + if (scope === "bots" && !botIds.length) { + return reply.code(400).send({ error: "botIds required for scope=bots" }); + } + if (scope === "targets" && !targets.length) { + return reply + .code(400) + .send({ error: "targets required for scope=targets" }); + } + + if (req.body?.preview) { + const preview = await previewBroadcast(ctx.db, { + scope, + botIds, + targets, + }); + return { + preview: { + deliverable: preview.deliverable, + skippedNoToken: preview.skippedNoToken, + missingBots: preview.missingBots, + textLength: text.length, + }, + }; + } + + try { + const job = await createBroadcastJob(ctx.db, { + createdBy: admin.id, + text, + scope, + botIds, + targets, + historyLimit: ctx.cfg.broadcastHistory, + }); + await writeAudit(ctx.db, "admin_broadcast_create", admin.id, { + jobId: job.id, + scope: job.scope, + total: job.stats.total, + textLen: text.length, + botIds: job.botIds, + }); + // Nudge worker to start immediately (same process or next poll) + try { + ctx.worker.wakeBroadcast(); + } catch { + /* worker may be disabled */ + } + return reply.code(201).send({ job: publicBroadcastJob(job) }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/broadcast/:id/cancel", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + try { + const job = await cancelBroadcastJob(ctx.db, req.params.id); + if (!job) return reply.code(404).send({ error: "not found" }); + await writeAudit(ctx.db, "admin_broadcast_cancel", admin.id, { + jobId: job.id, + stats: job.stats, + }); + return { job: publicBroadcastJob(job) }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }, + ); + + app.get<{ Params: { botId: string } }>( + "/api/v1/admin/bots/:botId/send-targets", + async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + const bot = await getBotAccount(ctx.db, req.params.botId); + if (!bot) return reply.code(404).send({ error: "not found" }); + const targets = await listBotSendTargets(ctx.db, bot.id); + return { + botId: bot.id, + displayName: bot.display_name, + targets, + deliverable: targets.filter((t) => t.hasContextToken).length, + total: targets.length, + }; + }, + ); + + app.get("/api/v1/admin/audit", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const limit = Number((req.query as { limit?: string }).limit ?? "50"); + return { logs: await listAuditLogs(ctx.db, Math.min(limit, 200)) }; + }); + + /** + * Recent activity stream backlog (super-admin). Merges Redis LIST + local ring. + * Query: limit, types=message,redis,worker,llm, full=1 + */ + app.get("/api/v1/admin/stream/recent", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.cfg.dataStreamEnabled || !ctx.activityBus) { + return reply.code(503).send({ error: "data_stream_disabled" }); + } + const q = req.query as { + limit?: string; + types?: string; + full?: string; + }; + const limit = Math.min(Math.max(Number(q.limit) || 100, 1), 300); + const full = q.full === "1" || q.full === "true"; + const typeFilter = parseStreamTypeFilter(q.types); + let events = await ctx.activityBus.recentMerged(limit); + if (typeFilter) { + events = events.filter((e) => streamTypeMatches(e.type, typeFilter)); + } + events = events.map((e) => shapeStreamEventForClient(e, full)); + return { + events, + enabled: true, + source: ctx.activityBus.getSource(), + }; + }); + + /** + * Live SSE activity stream (super-admin). + * Query: types, full=1, heartbeat=15 + */ + app.get("/api/v1/admin/stream", async (req, reply) => { + const admin = await requireSuperAdmin(req, reply, ctx); + if (!admin) return; + if (!ctx.cfg.dataStreamEnabled || !ctx.activityBus) { + return reply.code(503).send({ error: "data_stream_disabled" }); + } + + const q = req.query as { + types?: string; + full?: string; + heartbeat?: string; + limit?: string; + }; + const full = q.full === "1" || q.full === "true"; + const typeFilter = parseStreamTypeFilter(q.types); + const heartbeatSec = Math.min( + Math.max(Number(q.heartbeat) || 15, 5), + 60, + ); + const backlogLimit = Math.min(Math.max(Number(q.limit) || 80, 0), 200); + const bus = ctx.activityBus; + + // Disable compression buffering for this response + reply.hijack(); + const raw = reply.raw; + raw.writeHead(200, { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }); + + let closed = false; + const write = (chunk: string) => { + if (closed) return; + try { + raw.write(chunk); + } catch { + closed = true; + } + }; + + const sendEvent = (ev: import("./activity-stream.js").StreamEvent) => { + if (typeFilter && !streamTypeMatches(ev.type, typeFilter)) return; + const shaped = shapeStreamEventForClient(ev, full); + write(`id: ${shaped.id}\n`); + write(`event: activity\n`); + write(`data: ${JSON.stringify(shaped)}\n\n`); + }; + + write(`event: meta\ndata: ${JSON.stringify({ + ok: true, + source: bus.getSource(), + full, + types: q.types || "all", + ts: new Date().toISOString(), + })}\n\n`); + + if (backlogLimit > 0) { + try { + let backlog = await bus.recentMerged(backlogLimit); + if (typeFilter) { + backlog = backlog.filter((e) => streamTypeMatches(e.type, typeFilter)); + } + // Send oldest first so UI can append chronologically + for (const ev of backlog.slice().reverse()) { + sendEvent(ev); + } + } catch { + /* */ + } + } + + const unsub = bus.subscribe((ev) => { + sendEvent(ev); + }); + + const heartbeat = setInterval(() => { + write(`: ping ${Date.now()}\n\n`); + }, heartbeatSec * 1000); + + const onClose = () => { + if (closed) return; + closed = true; + clearInterval(heartbeat); + unsub(); + try { + raw.end(); + } catch { + /* */ + } + }; + + req.raw.on("close", onClose); + req.raw.on("error", onClose); + }); + + app.get("/api/v1/admin/usage", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const q = req.query as { day?: string; days?: string }; + if (q.days) { + const n = Math.min(Math.max(Number(q.days) || 7, 1), 30); + // Each day is an independent multi-RTT read — fan them out. 30 days + // used to be ~30x the latency of one. + const days = await Promise.all( + Array.from({ length: n }, (_unused, i) => + getUsageDayStats(ctx.db, dayKeyOffset(-i)), + ), + ); + return { days }; + } + const day = q.day || dayKey(); + return { usage: await getUsageDayStats(ctx.db, day) }; + }); + + app.get("/api/v1/admin/personas", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const q = ((req.query as { q?: string }).q ?? "").trim().toLowerCase(); + const includeDisabled = + (req.query as { includeDisabled?: string }).includeDisabled === "1" || + (req.query as { includeDisabled?: string }).includeDisabled === "true"; + const all = await listPersonas(ctx.db); + let personas = includeDisabled ? all : all.filter((p) => p.enabled); + if (q) { + personas = personas.filter((p) => { + const hay = [p.display_name, p.description, p.slug, ...(p.tags || [])] + .join(" ") + .toLowerCase(); + return hay.includes(q); + }); + } + return { + total: personas.length, + personas: personas.map((p) => + personaPublicDto(p, { + enabled: p.enabled !== 0, + isDefault: Boolean(p.is_default), + }), + ), + }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/admin/personas/:id", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + const prompt = await getPublishedPrompt(ctx.db, p.id); + return { + persona: personaPublicDto(p, { + enabled: p.enabled !== 0, + isDefault: Boolean(p.is_default), + systemPrompt: prompt, + }), + }; + }, + ); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/personas/:id/set-default", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const persona = await setDefaultPersona(ctx.db, req.params.id); + await writeAudit(ctx.db, "admin_persona_set_default", admin.id, { + id: persona.id, + }); + return { + ok: true, + persona: { + id: persona.id, + displayName: persona.display_name, + isDefault: true, + }, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }, + ); + + app.put<{ + Params: { id: string }; + Body: { + displayName?: string; + description?: string; + tags?: string[]; + visibility?: "public" | "private"; + systemPrompt?: string; + }; + }>("/api/v1/admin/personas/:id", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + try { + const persona = await updatePersonaMeta(ctx.db, p.id, req.body ?? {}); + await writeAudit(ctx.db, "admin_persona_updated", admin.id, { + id: p.id, + }); + return { persona }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(400).send({ error: message }); + } + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/personas/:id/takedown", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const p = await getPersona(ctx.db, req.params.id); + if (!p) return reply.code(404).send({ error: "not found" }); + if (p.owner_user_id === "system") { + return reply.code(403).send({ error: "cannot takedown system persona" }); + } + await softDeletePersona(ctx.db, p.id); + await writeAudit(ctx.db, "persona_takedown", admin.id, { id: p.id }); + return { ok: true }; + }, + ); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/personas/:id/restore", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const persona = await restorePersona(ctx.db, req.params.id); + await writeAudit(ctx.db, "persona_restore", admin.id, { + id: persona.id, + }); + return { ok: true, persona }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return reply.code(404).send({ error: message }); + } + }, + ); + + app.post<{ + Body: { + slug: string; + displayName: string; + description?: string; + systemPrompt: string; + isDefault?: boolean; + contentPolicy?: string; + tags?: string[]; + }; + }>("/api/v1/admin/personas", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const body = req.body; + if (!body?.slug || !body.displayName || !body.systemPrompt) { + return reply.code(400).send({ error: "missing fields" }); + } + if (await getPersonaBySlug(ctx.db, body.slug)) { + return reply.code(409).send({ error: "slug exists" }); + } + const persona = await createPersona(ctx.db, { + slug: body.slug, + displayName: body.displayName, + description: body.description, + systemPrompt: body.systemPrompt, + isDefault: body.isDefault, + contentPolicy: body.contentPolicy ?? "standard", + ownerUserId: "system", + visibility: "public", + tags: Array.isArray(body.tags) ? body.tags : undefined, + }); + await writeAudit(ctx.db, "persona_created", admin.id, { id: persona.id }); + return { persona }; + }); + + app.post<{ + Params: { id: string }; + Body: { systemPrompt: string }; + }>("/api/v1/admin/personas/:id/publish", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + if (!req.body?.systemPrompt) { + return reply.code(400).send({ error: "systemPrompt required" }); + } + try { + const version = await publishPersonaVersion( + ctx.db, + req.params.id, + req.body.systemPrompt, + ); + await writeAudit(ctx.db, "admin_persona_publish", admin.id, { + id: req.params.id, + }); + return { version }; + } catch { + return reply.code(404).send({ error: "not found" }); + } + }); + + // ── Admin stickers (review + official upload) ──────── + + app.get("/api/v1/admin/stickers", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const q = req.query as { + q?: string; + enabled?: string; + status?: string; + }; + const reviewStatus = + q.status === "pending" || + q.status === "approved" || + q.status === "rejected" + ? q.status + : "all"; + let [stickers, pendingCount] = await Promise.all([ + listStickers(ctx.db, { q: q.q, reviewStatus }), + countPendingStickers(ctx.db), + ]); + if (q.enabled === "1" || q.enabled === "true") { + stickers = stickers.filter((s) => s.enabled); + } else if (q.enabled === "0" || q.enabled === "false") { + stickers = stickers.filter((s) => !s.enabled); + } + return { stickers, pendingCount }; + }); + + app.get<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const sticker = await getSticker(ctx.db, req.params.id); + if (!sticker) return reply.code(404).send({ error: "not found" }); + return { sticker }; + }, + ); + + app.get<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id/image", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const sticker = await getSticker(ctx.db, req.params.id); + if (!sticker) return reply.code(404).send({ error: "not found" }); + return sendPrivateStickerImage(ctx, req, reply, sticker); + }, + ); + + app.post<{ + Body: { + slug?: string; + displayName: string; + description?: string; + tags?: string[]; + mime?: string; + dataBase64: string; + enabled?: boolean; + autoApprove?: boolean; + visibility?: "public" | "private"; + }; + }>("/api/v1/admin/stickers", { bodyLimit: ctx.cfg.uploadBodyLimit }, async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const body = req.body; + if (!body?.displayName || !body.dataBase64) { + return reply.code(400).send({ error: "missing fields" }); + } + if (body.slug && !isValidStickerSlug(body.slug.trim().toLowerCase())) { + return reply + .code(400) + .send({ error: "invalid slug", hint: "use a-z0-9_- , 2-64 chars" }); + } + if (body.slug && (await getStickerBySlug(ctx.db, body.slug))) { + return reply.code(409).send({ error: "slug exists" }); + } + + try { + const { data, mime } = parseStickerUpload(body, ctx.cfg.stickerMaxBytes); + const sticker = await createSticker(ctx.db, { + slug: body.slug, + displayName: body.displayName, + description: body.description, + tags: Array.isArray(body.tags) ? body.tags : undefined, + mime, + sizeBytes: data.length, + enabled: body.enabled !== false, + ownerUserId: "system", + visibility: body.visibility === "private" ? "private" : "public", + autoApprove: body.autoApprove !== false, + data, + }); + await writeAudit(ctx.db, "sticker_create", admin.id, { + id: sticker.id, + slug: sticker.slug, + }); + return { sticker }; + } catch (err) { + if (err instanceof StickerSecurityError) { + return reply + .code(400) + .send({ error: "unsafe_image", code: err.code, message: err.message }); + } + const msg = (err as Error).message; + if (msg === "slug exists") { + return reply.code(409).send({ error: "slug exists" }); + } + if (msg === "invalid slug") { + return reply.code(400).send({ error: "invalid slug" }); + } + throw err; + } + }); + + app.put<{ + Params: { id: string }; + Body: { + slug?: string; + displayName?: string; + description?: string; + tags?: string[]; + enabled?: boolean; + mime?: string; + dataBase64?: string; + visibility?: "public" | "private"; + }; + }>("/api/v1/admin/stickers/:id", { bodyLimit: ctx.cfg.uploadBodyLimit }, async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const cur = await getSticker(ctx.db, req.params.id); + if (!cur) return reply.code(404).send({ error: "not found" }); + + const body = req.body ?? {}; + + try { + if (body.dataBase64) { + const { data, mime } = parseStickerUpload(body, ctx.cfg.stickerMaxBytes); + await replaceStickerBlob(ctx.db, cur.id, data, { + mime, + fileName: makeStickerFileName(cur.id, mime), + }); + } + + // Admin meta edits do not force re-pending for system stickers + const sticker = await updateStickerMeta(ctx.db, cur.id, { + slug: body.slug, + displayName: body.displayName, + description: body.description, + tags: body.tags, + enabled: body.enabled, + visibility: body.visibility, + rePending: cur.owner_user_id !== "system", + }); + await writeAudit(ctx.db, "sticker_update", admin.id, { + id: sticker.id, + slug: sticker.slug, + }); + return { sticker }; + } catch (err) { + if (err instanceof StickerSecurityError) { + return reply + .code(400) + .send({ error: "unsafe_image", code: err.code, message: err.message }); + } + const msg = (err as Error).message; + if (msg === "slug exists") { + return reply.code(409).send({ error: "slug exists" }); + } + if (msg === "invalid slug") { + return reply.code(400).send({ error: "invalid slug" }); + } + if (msg === "not found") { + return reply.code(404).send({ error: "not found" }); + } + throw err; + } + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id/approve", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const sticker = await approveSticker(ctx.db, req.params.id, admin.id); + await writeAudit(ctx.db, "sticker_approve", admin.id, { + id: sticker.id, + }); + return { sticker }; + } catch { + return reply.code(404).send({ error: "not found" }); + } + }, + ); + + app.post<{ + Params: { id: string }; + Body: { reason?: string }; + }>("/api/v1/admin/stickers/:id/reject", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const sticker = await rejectSticker( + ctx.db, + req.params.id, + admin.id, + req.body?.reason, + ); + await writeAudit(ctx.db, "sticker_reject", admin.id, { + id: sticker.id, + reason: req.body?.reason, + }); + return { sticker }; + } catch { + return reply.code(404).send({ error: "not found" }); + } + }); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id/takedown", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const sticker = await softDeleteSticker(ctx.db, req.params.id); + await writeAudit(ctx.db, "sticker_takedown", admin.id, { + id: sticker.id, + }); + return { sticker }; + } catch { + return reply.code(404).send({ error: "not found" }); + } + }, + ); + + app.post<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id/restore", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + try { + const sticker = await restoreSticker(ctx.db, req.params.id); + await writeAudit(ctx.db, "sticker_restore", admin.id, { + id: sticker.id, + }); + return { sticker }; + } catch { + return reply.code(404).send({ error: "not found" }); + } + }, + ); + + app.delete<{ Params: { id: string } }>( + "/api/v1/admin/stickers/:id", + async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const removed = await deleteSticker(ctx.db, req.params.id); + if (!removed) return reply.code(404).send({ error: "not found" }); + await writeAudit(ctx.db, "sticker_delete", admin.id, { + id: removed.id, + slug: removed.slug, + }); + return { ok: true }; + }, + ); + + app.get("/api/v1/me/memories", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const q = req.query as { + botAccountId?: string; + peerId?: string; + personaId?: string; + }; + if (!q.botAccountId || !q.peerId) { + return reply.code(400).send({ error: "botAccountId and peerId required" }); + } + const bot = await getBotAccount(ctx.db, q.botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + if (q.personaId) { + return { + memories: await listMemories( + ctx.db, + q.botAccountId, + q.peerId, + q.personaId, + ), + }; + } + const personas = await listPersonas(ctx.db); + // One pipelined wave instead of one LRANGE per persona on the platform + const memMap = await listMemoriesMany( + ctx.db, + q.botAccountId, + q.peerId, + personas.map((p) => p.id), + ); + const groups: { + personaId: string; + personaName: string; + memories: { id: string; content: string }[]; + }[] = []; + let total = 0; + for (const p of personas) { + const mems = memMap.get(p.id); + if (mems?.length) { + groups.push({ + personaId: p.id, + personaName: p.display_name, + memories: mems.map((m) => ({ id: m.id, content: m.content })), + }); + total += mems.length; + } + } + return { total, groups }; + }); + + app.post<{ + Body: { botAccountId: string; peerId: string; personaId?: string }; + }>("/api/v1/me/memories/reset", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const { botAccountId, peerId, personaId } = req.body ?? {}; + if (!botAccountId || !peerId) { + return reply.code(400).send({ error: "missing fields" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + await ctx.chat.resetMemory(botAccountId, peerId, personaId); + return { ok: true }; + }); + + app.delete<{ + Body: { + botAccountId: string; + peerId: string; + personaId: string; + memoryId: string; + }; + }>("/api/v1/me/memories", async (req, reply) => { + const user = await requireUser(req, reply, ctx); + if (!user) return; + const { botAccountId, peerId, personaId, memoryId } = req.body ?? {}; + if (!botAccountId || !peerId || !personaId || !memoryId) { + return reply.code(400).send({ error: "missing fields" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot || (bot.owner_user_id !== user.id && !user.is_admin)) { + return reply.code(403).send({ error: "forbidden" }); + } + const ok = await deleteMemory( + ctx.db, + botAccountId, + peerId, + personaId, + memoryId, + ); + if (!ok) return reply.code(404).send({ error: "memory not found" }); + await writeAudit(ctx.db, "memory_delete", user.id, { + botAccountId, + peerId, + personaId, + memoryId, + }); + return { ok: true }; + }); + + app.delete<{ + Body: { + botAccountId: string; + peerId: string; + personaId: string; + memoryId: string; + }; + }>("/api/v1/admin/memories", async (req, reply) => { + const admin = await requireAdmin(req, reply, ctx); + if (!admin) return; + const { botAccountId, peerId, personaId, memoryId } = req.body ?? {}; + if (!botAccountId || !peerId || !personaId || !memoryId) { + return reply + .code(400) + .send({ error: "botAccountId, peerId, personaId, memoryId required" }); + } + const bot = await getBotAccount(ctx.db, botAccountId); + if (!bot) return reply.code(404).send({ error: "bot not found" }); + const ok = await deleteMemory( + ctx.db, + botAccountId, + peerId, + personaId, + memoryId, + ); + if (!ok) return reply.code(404).send({ error: "memory not found" }); + await writeAudit(ctx.db, "admin_memory_delete", admin.id, { + botAccountId, + peerId, + personaId, + memoryId, + }); + return { ok: true }; + }); +} diff --git a/apps/api/src/runtime-config-apply.ts b/apps/api/src/runtime-config-apply.ts new file mode 100644 index 0000000..dc99863 --- /dev/null +++ b/apps/api/src/runtime-config-apply.ts @@ -0,0 +1,260 @@ +import type { ChatService, TryChatService } from "@wechat-ai/core"; +import type { ActivityBus } from "./activity-stream.js"; +import type { AppConfig } from "./config.js"; +import type { RuntimeSettingKey } from "./runtime-settings-spec.js"; +import type { BotWorkerManager } from "./worker.js"; + +export interface RuntimeConfigTargets { + chat: ChatService; + tryChat: TryChatService; + worker: BotWorkerManager; + activityBus: ActivityBus; +} + +/** + * Keys whose change requires re-pushing a subsystem's options. + * + * Deliberately coarse: when any key in a set changes we re-push that whole + * subsystem from the current `cfg`. The push is idempotent and cheap, and it + * keeps this file from drifting into a per-key dispatch table that silently + * misses a field when someone adds one. + */ +const CHAT_KEYS: RuntimeSettingKey[] = [ + "shortHistoryLimit", + "memoryExtractEveryN", + "allowUnapproved", + "multiBubbleJson", + "replyFilterEnabled", + "maxReplyChunks", + "maxChunkChars", + "maxStickersPerReply", + "stickerSendEnabled", + "memoryTopK", + "memoryFullInjectMax", + "memoryMaxItems", + "timeToolEnabled", + "timeToolTimeZone", + "webSearchEnabled", + "webSearchMaxResults", + "toolsBaseUrl", + "toolsApiKey", + "toolsTimeoutMs", + "chatflowHttpAllowlist", + "chatflowMaxSteps", + "chatflowMaxNodes", + "visionMode", + "visionModel", + "visionCaptionMaxTokens", +]; + +const TRYCHAT_KEYS: RuntimeSettingKey[] = [ + "tryChatSessionTtlSec", + "tryChatMaxHistory", + "tryChatMaxUserMsgsPerDay", + "tryChatMaxUserMsgsPerSession", + "multiBubbleJson", + "replyFilterEnabled", + "maxReplyChunks", + "maxChunkChars", + "timeToolEnabled", + "timeToolTimeZone", + "toolsBaseUrl", + "toolsApiKey", + "toolsTimeoutMs", + "webSearchEnabled", + "webSearchMaxResults", + "chatflowHttpAllowlist", + "chatflowMaxSteps", + "chatflowMaxNodes", +]; + +const WORKER_KEYS: RuntimeSettingKey[] = [ + "stickerSendEnabled", + "maxStickersPerReply", + "visionEnabled", + "visionMaxImages", + "voiceTranscriptEnabled", + "inboundMediaMaxBytes", + "splitReply", + "replyDelayMsPerChar", + "replyDelayMinMs", + "replyDelayMaxMs", + "replyDelayFirstMinMs", + "replyDelayFirstMaxMs", + "replyDelayThinkExtraMs", + "peerRatePerMinute", + "maxBotsPerWorker", + "leaseTtlSec", + "leaseRenewSec", + "rebalanceEnabled", + "rebalanceIntervalSec", + "rebalanceSlack", + "rebalanceMaxPerTick", + "workerWeightTtlSec", + "inboxMaxLen", + "proactiveEnabled", + "proactiveIdleHours", + "proactiveMinIntervalHours", + "proactiveMaxPerDay", + "proactiveQuietHours", + "proactiveScanIntervalSec", + "proactiveMaxPerScan", + "proactiveLockTtlSec", + "proactiveAttemptCooldownHours", + "broadcastIntervalMs", + "p2pEnabled", + "p2pBindCodeTtlSec", + "p2pRequestTtlSec", + "p2pSessionIdleSec", + "p2pRelayMaxChars", + "p2pMaxRequestsPerDay", + "nodeLabel", + "nodeRegion", + "otaEnabled", + "otaAllowInstall", + "otaStagingDir", +]; + +const STREAM_KEYS: RuntimeSettingKey[] = [ + "dataStreamEnabled", + "dataStreamMaxEps", + "dataStreamRedisSample", +]; + +function touched( + changed: Set, + keys: RuntimeSettingKey[], +): boolean { + return keys.some((k) => changed.has(k)); +} + +/** + * Push the current effective config into services that snapshot their options + * at construction. Route handlers need nothing here — they read `ctx.cfg.*` + * per request and `cfg` is mutated in place by the settings manager. + */ +export function applyRuntimeConfigToServices( + changed: Set, + cfg: AppConfig, + targets: RuntimeConfigTargets, +): void { + if (touched(changed, CHAT_KEYS)) { + targets.chat.applyRuntimeOptions({ + shortHistoryLimit: cfg.shortHistoryLimit, + memoryExtractEveryN: cfg.memoryExtractEveryN, + allowUnapproved: cfg.allowUnapproved, + multiBubbleJson: cfg.multiBubbleJson, + replyFilterEnabled: cfg.replyFilterEnabled, + maxReplyBubbles: cfg.maxReplyChunks, + maxChunkChars: cfg.maxChunkChars, + maxStickersPerReply: cfg.maxStickersPerReply, + stickersEnabled: cfg.stickerSendEnabled, + memoryTopK: cfg.memoryTopK, + memoryFullInjectMax: cfg.memoryFullInjectMax, + memoryMaxItems: cfg.memoryMaxItems, + timeToolEnabled: cfg.timeToolEnabled, + timeToolTimeZone: cfg.timeToolTimeZone, + webSearchEnabled: cfg.webSearchEnabled, + webSearchMaxResults: cfg.webSearchMaxResults, + toolsBaseUrl: cfg.toolsBaseUrl || undefined, + toolsApiKey: cfg.toolsApiKey || undefined, + toolsTimeoutMs: cfg.toolsTimeoutMs, + chatflowHttpAllowHosts: cfg.chatflowHttpAllowlist, + chatflowMaxSteps: cfg.chatflowMaxSteps, + chatflowMaxNodes: cfg.chatflowMaxNodes, + visionMode: cfg.visionMode, + visionModel: cfg.visionModel || undefined, + visionCaptionMaxTokens: cfg.visionCaptionMaxTokens, + }); + } + + if (touched(changed, TRYCHAT_KEYS)) { + targets.tryChat.applyRuntimeOptions({ + sessionTtlSec: cfg.tryChatSessionTtlSec, + maxHistory: cfg.tryChatMaxHistory, + maxUserMsgsPerDay: cfg.tryChatMaxUserMsgsPerDay, + maxUserMsgsPerSession: cfg.tryChatMaxUserMsgsPerSession, + multiBubbleJson: cfg.multiBubbleJson, + replyFilterEnabled: cfg.replyFilterEnabled, + maxReplyBubbles: cfg.maxReplyChunks, + maxChunkChars: cfg.maxChunkChars, + timeToolEnabled: cfg.timeToolEnabled, + timeToolTimeZone: cfg.timeToolTimeZone, + toolsBaseUrl: cfg.toolsBaseUrl || undefined, + toolsApiKey: cfg.toolsApiKey || undefined, + toolsTimeoutMs: cfg.toolsTimeoutMs, + webSearchEnabled: cfg.webSearchEnabled, + webSearchMaxResults: cfg.webSearchMaxResults, + chatflowHttpAllowHosts: cfg.chatflowHttpAllowlist, + chatflowMaxSteps: cfg.chatflowMaxSteps, + chatflowMaxNodes: cfg.chatflowMaxNodes, + }); + } + + if (touched(changed, WORKER_KEYS)) { + targets.worker.applyRuntimeConfig({ + stickerSendEnabled: cfg.stickerSendEnabled, + maxStickersPerReply: cfg.maxStickersPerReply, + visionEnabled: cfg.visionEnabled, + visionMaxImages: cfg.visionMaxImages, + voiceTranscriptEnabled: cfg.voiceTranscriptEnabled, + inboundMediaMaxBytes: cfg.inboundMediaMaxBytes, + splitReply: cfg.splitReply, + peerRatePerMinute: cfg.peerRatePerMinute, + maxBotsPerWorker: cfg.maxBotsPerWorker, + leaseTtlSec: cfg.leaseTtlSec, + leaseRenewSec: cfg.leaseRenewSec, + rebalanceEnabled: cfg.rebalanceEnabled, + rebalanceIntervalSec: cfg.rebalanceIntervalSec, + rebalanceSlack: cfg.rebalanceSlack, + rebalanceMaxPerTick: cfg.rebalanceMaxPerTick, + workerWeightTtlSec: cfg.workerWeightTtlSec, + inboxMaxLen: cfg.inboxMaxLen, + replyDelay: { + msPerChar: cfg.replyDelayMsPerChar, + minMs: cfg.replyDelayMinMs, + maxMs: cfg.replyDelayMaxMs, + firstMinMs: cfg.replyDelayFirstMinMs, + firstMaxMs: cfg.replyDelayFirstMaxMs, + thinkExtraMs: cfg.replyDelayThinkExtraMs, + }, + proactive: { + globalEnabled: cfg.proactiveEnabled, + defaultIdleHours: cfg.proactiveIdleHours, + defaultMinIntervalHours: cfg.proactiveMinIntervalHours, + defaultMaxPerDay: cfg.proactiveMaxPerDay, + defaultQuietHours: cfg.proactiveQuietHours, + scanIntervalSec: cfg.proactiveScanIntervalSec, + maxPerScan: cfg.proactiveMaxPerScan, + lockTtlSec: cfg.proactiveLockTtlSec, + attemptCooldownHours: cfg.proactiveAttemptCooldownHours, + }, + broadcast: { + intervalMs: cfg.broadcastIntervalMs, + pollIntervalMs: 2_000, + lockTtlSec: 60, + }, + p2pEnabled: cfg.p2pEnabled, + p2p: { + bindCodeTtlSec: cfg.p2pBindCodeTtlSec, + requestTtlSec: cfg.p2pRequestTtlSec, + sessionIdleSec: cfg.p2pSessionIdleSec, + relayMaxChars: cfg.p2pRelayMaxChars, + maxRequestsPerDay: cfg.p2pMaxRequestsPerDay, + }, + nodeLabel: cfg.nodeLabel, + nodeRegion: cfg.nodeRegion, + otaEnabled: cfg.otaEnabled, + otaAllowInstall: cfg.otaAllowInstall, + otaStagingDir: cfg.otaStagingDir, + }); + } + + if (touched(changed, STREAM_KEYS)) { + targets.activityBus.applyRuntimeOptions({ + enabled: cfg.dataStreamEnabled, + maxEps: cfg.dataStreamMaxEps, + redisSample: cfg.dataStreamRedisSample, + }); + } +} diff --git a/apps/api/src/runtime-config.test.ts b/apps/api/src/runtime-config.test.ts new file mode 100644 index 0000000..4b079e6 --- /dev/null +++ b/apps/api/src/runtime-config.test.ts @@ -0,0 +1,476 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import type { Db } from "@wechat-ai/db"; +import { loadConfig, type AppConfig } from "./config.js"; +import { + RuntimeConfigManager, + RuntimeSettingsUnavailableError, + SECRET_CLEAR, + SECRET_MASK, +} from "./runtime-config.js"; +import type { RuntimeSettingKey } from "./runtime-settings-spec.js"; +import { + coerceSetting, + SETTING_SPECS, + SETTING_SPEC_BY_KEY, +} from "./runtime-settings-spec.js"; + +/** + * Minimal in-memory stand-in for the Redis surface the manager touches: + * one JSON doc plus the SET NX lock guarding its read-modify-write. + */ +function fakeDb(): Db & { store: Map; strings: Map } { + const store = new Map(); + const strings = new Map(); + return { + store, + strings, + async getJson(key: string): Promise { + return (store.get(key) as T) ?? null; + }, + async setJson(key: string, value: unknown): Promise { + store.set(key, JSON.parse(JSON.stringify(value))); + }, + async del(...keys: string[]): Promise { + for (const k of keys) { + store.delete(k); + strings.delete(k); + } + }, + redis: { + async set( + key: string, + value: string, + _ex?: string, + _ttl?: number, + nx?: string, + ): Promise { + if (nx === "NX" && strings.has(key)) return null; + strings.set(key, value); + return "OK"; + }, + async get(key: string): Promise { + return strings.get(key) ?? null; + }, + }, + } as unknown as Db & { store: Map; strings: Map }; +} + +function baseConfig(env: Record = {}): AppConfig { + return loadConfig({ + REDIS_URL: "redis://127.0.0.1:6379", + ...env, + } as NodeJS.ProcessEnv); +} + +describe("runtime settings spec", () => { + it("every spec key exists on AppConfig", () => { + const cfg = baseConfig() as unknown as Record; + for (const spec of SETTING_SPECS) { + assert.ok( + spec.key in cfg, + `${spec.key} is declared in the registry but missing from AppConfig`, + ); + } + }); + + it("excludes bootstrap-critical config", () => { + const forbidden = [ + "redisUrl", + "llmBaseUrl", + "llmApiKey", + "llmModel", + "llmProviderSecret", + "sessionCookieName", + "cookieSecure", + "publicBaseUrl", + "corsOrigins", + "host", + "port", + "token", + "adminIds", + "repoRoot", + "appVersion", + ]; + for (const k of forbidden) { + assert.ok( + !SETTING_SPEC_BY_KEY.has(k as RuntimeSettingKey), + `${k} must stay env-only`, + ); + } + }); + + /** + * Guard rail: every AppConfig field must be a deliberate decision — either + * admin-editable or explicitly listed as env-only. Adding a new config field + * fails this test until it is classified, which is what keeps the panel from + * silently drifting out of date. + */ + it("classifies every AppConfig field as editable or env-only", () => { + const ENV_ONLY = new Set([ + // bootstrap / connection + "redisUrl", + "host", + "port", + "repoRoot", + // platform LLM credentials + "llmBaseUrl", + "llmApiKey", + "llmModel", + // vision endpoint credentials — same trust level as the platform LLM + "visionBaseUrl", + "visionApiKey", + // vision endpoint credentials — dialed directly like the platform LLM, + // and they fall back to it when empty, so same rule applies + "visionBaseUrl", + "visionApiKey", + // secrets whose rotation breaks stored data or locks admins out + "llmProviderSecret", + "token", + "adminIds", + // cookie / URL surface + "sessionCookieName", + "cookieSecure", + "publicBaseUrl", + "corsOrigins", + // derived, not directly settable + "appVersion", + "uploadBodyLimit", + // no consumer: the default persona comes from the DB is_default flag + "defaultPersonaSlug", + ]); + /** + * Consumer not written yet. Exposing one of these would give the operator + * a control that reports success and changes nothing — the same defect we + * removed DEFAULT_PERSONA_SLUG for. Move to SETTING_SPECS once the read + * site lands. + */ + const PENDING_CONSUMER = new Set([ + // declared on ChatServiceOptions, but nothing reads this.opts.visionMode + "visionMode", + ]); + const cfg = baseConfig() as unknown as Record; + const unclassified = Object.keys(cfg).filter( + (k) => + !ENV_ONLY.has(k) && + !PENDING_CONSUMER.has(k) && + !SETTING_SPEC_BY_KEY.has(k as RuntimeSettingKey), + ); + assert.deepEqual( + unclassified, + [], + `add these to SETTING_SPECS or to the ENV_ONLY list: ${unclassified.join(", ")}`, + ); + }); + + it("has no duplicate keys or env names", () => { + const keys = new Set(); + const envs = new Set(); + for (const spec of SETTING_SPECS) { + assert.ok(!keys.has(spec.key), `duplicate key ${spec.key}`); + assert.ok(!envs.has(spec.env), `duplicate env ${spec.env}`); + keys.add(spec.key); + envs.add(spec.env); + } + }); + + it("clamps numbers into range and parses booleans/csv", () => { + const steps = SETTING_SPEC_BY_KEY.get("chatflowMaxSteps")!; + assert.equal(coerceSetting(steps, 9999), 200); + assert.equal(coerceSetting(steps, -5), 1); + assert.equal(coerceSetting(steps, "12"), 12); + assert.equal(coerceSetting(steps, "abc"), null); + + const bool = SETTING_SPEC_BY_KEY.get("webSearchEnabled")!; + assert.equal(coerceSetting(bool, "true"), true); + assert.equal(coerceSetting(bool, "0"), false); + + const csv = SETTING_SPEC_BY_KEY.get("chatflowHttpAllowlist")!; + assert.equal(coerceSetting(csv, " a.com , ,b.com "), "a.com,b.com"); + }); +}); + +describe("RuntimeConfigManager", () => { + let db: ReturnType; + let cfg: AppConfig; + let applied: Array>; + let mgr: RuntimeConfigManager; + + beforeEach(async () => { + db = fakeDb(); + cfg = baseConfig({ WEB_SEARCH_ENABLED: "false", CHATFLOW_MAX_STEPS: "32" }); + applied = []; + mgr = new RuntimeConfigManager( + db, + cfg, + (changed) => applied.push(new Set(changed)), + () => {}, + ); + await mgr.init(); + }); + + it("starts from env with no overrides", () => { + const v = mgr.view(); + assert.equal(v.overriddenCount, 0); + assert.equal(cfg.chatflowMaxSteps, 32); + assert.equal(applied.length, 0); + }); + + it("mutates the live cfg object in place and reports the diff", async () => { + const r = await mgr.patch({ + patch: { chatflowMaxSteps: 64, webSearchEnabled: true }, + actor: "tester", + }); + assert.deepEqual(r.changed.sort(), ["chatflowMaxSteps", "webSearchEnabled"]); + assert.equal(cfg.chatflowMaxSteps, 64); + assert.equal(cfg.webSearchEnabled, true); + assert.equal(applied.length, 1); + assert.ok(applied[0]!.has("chatflowMaxSteps")); + }); + + it("clamps out-of-range input instead of storing it", async () => { + await mgr.patch({ patch: { chatflowMaxSteps: 100000 }, actor: "t" }); + assert.equal(cfg.chatflowMaxSteps, 200); + }); + + it("drops the override when a value is set back to the env default", async () => { + await mgr.patch({ patch: { chatflowMaxSteps: 64 }, actor: "t" }); + assert.equal(mgr.view().overriddenCount, 1); + await mgr.patch({ patch: { chatflowMaxSteps: 32 }, actor: "t" }); + assert.equal(mgr.view().overriddenCount, 0); + assert.equal(cfg.chatflowMaxSteps, 32); + }); + + it("reset restores the env default", async () => { + await mgr.patch({ patch: { chatflowMaxSteps: 64 }, actor: "t" }); + const r = await mgr.patch({ reset: ["chatflowMaxSteps"], actor: "t" }); + assert.deepEqual(r.changed, ["chatflowMaxSteps"]); + assert.equal(cfg.chatflowMaxSteps, 32); + }); + + it("resetAll clears every override", async () => { + await mgr.patch({ + patch: { chatflowMaxSteps: 64, memoryTopK: 33 }, + actor: "t", + }); + await mgr.patch({ resetAll: true, actor: "t" }); + assert.equal(mgr.view().overriddenCount, 0); + assert.equal(cfg.chatflowMaxSteps, 32); + assert.equal(cfg.memoryTopK, 12); + }); + + it("csv settings round-trip into a string array on cfg", async () => { + await mgr.patch({ + patch: { chatflowHttpAllowlist: "a.com, b.com" }, + actor: "t", + }); + assert.deepEqual(cfg.chatflowHttpAllowlist, ["a.com", "b.com"]); + }); + + it("never leaks a secret and treats blank as no-change", async () => { + await mgr.patch({ patch: { toolsApiKey: "sk-real" }, actor: "t" }); + assert.equal(cfg.toolsApiKey, "sk-real"); + const item = mgr.view().items.find((i) => i.key === "toolsApiKey")!; + assert.equal(item.value, SECRET_MASK); + + // A blank submit must not wipe the stored key… + await mgr.patch({ patch: { toolsApiKey: "" }, actor: "t" }); + assert.equal(cfg.toolsApiKey, "sk-real"); + // …and echoing the mask back must not become the literal value. + await mgr.patch({ patch: { toolsApiKey: SECRET_MASK }, actor: "t" }); + assert.equal(cfg.toolsApiKey, "sk-real"); + // Explicit clear. + await mgr.patch({ patch: { toolsApiKey: SECRET_CLEAR }, actor: "t" }); + assert.equal(cfg.toolsApiKey, ""); + }); + + it("ignores unknown keys", async () => { + const r = await mgr.patch({ + patch: { redisUrl: "redis://evil", notAKey: 1 }, + actor: "t", + }); + assert.deepEqual(r.changed, []); + assert.equal(cfg.redisUrl, "redis://127.0.0.1:6379"); + }); + + it("picks up a peer node's write on the next refresh", async () => { + const peerDb = db; + const peerCfg = baseConfig({ CHATFLOW_MAX_STEPS: "32" }); + const peer = new RuntimeConfigManager(peerDb, peerCfg, () => {}, () => {}); + await peer.init(); + + await mgr.patch({ patch: { chatflowMaxSteps: 77 }, actor: "node-a" }); + assert.equal(peerCfg.chatflowMaxSteps, 32, "not yet refreshed"); + + const changed = await peer.refresh(); + assert.equal(changed, true); + assert.equal(peerCfg.chatflowMaxSteps, 77); + assert.equal(await peer.refresh(), false, "second refresh is a no-op"); + }); + + it("surfaces cross-field warnings without rewriting input", async () => { + await mgr.patch({ + patch: { leaseTtlSec: 20, leaseRenewSec: 30 }, + actor: "t", + }); + assert.equal(cfg.leaseTtlSec, 20); + assert.equal(cfg.leaseRenewSec, 30); + assert.ok( + mgr.currentWarnings().some((w) => w.includes("租约 TTL")), + "expected a lease TTL warning", + ); + }); + + it("keeps uploadBodyLimit consistent with stickerMaxBytes", async () => { + await mgr.patch({ patch: { stickerMaxBytes: 20 * 1024 * 1024 }, actor: "t" }); + assert.equal(cfg.uploadBodyLimit, 40 * 1024 * 1024); + }); + + it("releases the RMW lock so a second write is not blocked", async () => { + await mgr.patch({ patch: { chatflowMaxSteps: 40 }, actor: "a" }); + assert.equal(db.strings.has("wa:settings:runtime:lock"), false); + await mgr.patch({ patch: { memoryTopK: 20 }, actor: "b" }); + assert.equal(cfg.chatflowMaxSteps, 40); + assert.equal(cfg.memoryTopK, 20); + }); + + it("concurrent writes on two nodes do not lose each other's edits", async () => { + const peerCfg = baseConfig({ CHATFLOW_MAX_STEPS: "32" }); + const peer = new RuntimeConfigManager(db, peerCfg, () => {}, () => {}); + await peer.init(); + + // Both patches race against the same shared store. + await Promise.all([ + mgr.patch({ patch: { chatflowMaxSteps: 50 }, actor: "node-a" }), + peer.patch({ patch: { memoryTopK: 40 }, actor: "node-b" }), + ]); + + await mgr.refresh(); + await peer.refresh(); + assert.equal(cfg.chatflowMaxSteps, 50, "node A's edit survived"); + assert.equal(cfg.memoryTopK, 40, "node B's edit survived"); + assert.equal(mgr.view().overriddenCount, 2); + }); + + it("view() payload matches what the admin page renders", () => { + const v = mgr.view(); + const groupIds = new Set(v.groups.map((g) => g.id)); + assert.ok(v.items.length > 50, "expected a broad settings surface"); + for (const item of v.items) { + for (const f of [ + "key", + "env", + "group", + "label", + "type", + "value", + "envDefault", + "overridden", + "restart", + ]) { + assert.ok(f in item, `${item.key} is missing ${f}`); + } + assert.ok( + groupIds.has(item.group as never), + `${item.key} points at unknown group ${item.group}`, + ); + assert.ok( + ["bool", "int", "float", "string", "csv", "secret"].includes(item.type), + `${item.key} has unrenderable type ${item.type}`, + ); + } + // Every declared group must actually hold at least one row. + for (const g of v.groups) { + assert.ok( + v.items.some((i) => i.group === g.id), + `group ${g.id} would render empty`, + ); + } + }); + + it("rejects a log level outside the allowed set", async () => { + // A bad level would only bite on the NEXT boot (restart:true), where pino + // throws at Fastify construction and crash-loops every node. + const r = await mgr.patch({ patch: { logLevel: "verbose" }, actor: "t" }); + assert.deepEqual(r.changed, []); + assert.equal(cfg.logLevel, "info"); + await mgr.patch({ patch: { logLevel: "debug" }, actor: "t" }); + assert.equal(cfg.logLevel, "debug"); + }); + + it("a Redis read failure never wipes stored overrides", async () => { + await mgr.patch({ + patch: { chatflowMaxSteps: 64, memoryTopK: 33 }, + actor: "t", + }); + const stored = JSON.parse( + JSON.stringify(db.store.get("wa:settings:runtime")), + ); + + // Simulate a transient Redis error on the next read. + const realGet = db.getJson.bind(db); + let failNext = true; + (db as { getJson: unknown }).getJson = async (key: string) => { + if (failNext) { + failNext = false; + throw new Error("ETIMEDOUT"); + } + return realGet(key); + }; + + // refresh() must keep the last good state, not revert to .env. + assert.equal(await mgr.refresh(), false); + assert.equal(cfg.chatflowMaxSteps, 64); + assert.equal(cfg.memoryTopK, 33); + + // patch() must refuse to write rather than persist an empty base. + failNext = true; + await assert.rejects( + () => mgr.patch({ patch: { memoryTopK: 44 }, actor: "t" }), + (e: unknown) => e instanceof RuntimeSettingsUnavailableError, + ); + assert.deepEqual(db.store.get("wa:settings:runtime"), stored); + }); + + it("resetAll still works when the stored document is unreadable", async () => { + await mgr.patch({ patch: { chatflowMaxSteps: 64 }, actor: "t" }); + const realGet = db.getJson.bind(db); + let fail = true; + (db as { getJson: unknown }).getJson = async (key: string) => { + if (fail) throw new Error("bad json"); + return realGet(key); + }; + const r = await mgr.patch({ resetAll: true, actor: "t" }); + assert.ok(r.changed.includes("chatflowMaxSteps")); + fail = false; + await mgr.refresh(); + assert.equal(cfg.chatflowMaxSteps, 32); + assert.equal(mgr.view().overriddenCount, 0); + }); + + it("resetAll clears a corrupt document even with nothing loaded locally", async () => { + // The node that boots into a corrupt doc has no overrides of its own, so + // `changed` is empty — the write must happen anyway or recovery is a no-op. + db.store.set("wa:settings:runtime", { values: { chatflowMaxSteps: 64 } }); + const realGet = db.getJson.bind(db); + let fail = true; + (db as { getJson: unknown }).getJson = async (key: string) => { + if (fail) throw new Error("Unexpected token in JSON"); + return realGet(key); + }; + await mgr.patch({ resetAll: true, actor: "t" }); + fail = false; + const doc = db.store.get("wa:settings:runtime") as { values: object }; + assert.deepEqual(doc.values, {}, "corrupt doc was replaced with an empty one"); + }); + + it("survives a malformed stored document", async () => { + db.store.set("wa:settings:runtime", { + values: { chatflowMaxSteps: "not-a-number", bogus: 1 }, + updatedAt: "x", + updatedBy: "y", + }); + await mgr.refresh(); + assert.equal(cfg.chatflowMaxSteps, 32); + }); +}); diff --git a/apps/api/src/runtime-config.ts b/apps/api/src/runtime-config.ts new file mode 100644 index 0000000..f339c5e --- /dev/null +++ b/apps/api/src/runtime-config.ts @@ -0,0 +1,476 @@ +import { K, type Db } from "@wechat-ai/db"; +import type { AppConfig } from "./config.js"; +import { + coerceSetting, + configToSettingValue, + isRuntimeSettingKey, + SETTING_GROUPS, + SETTING_SPEC_BY_KEY, + SETTING_SPECS, + settingValueToConfig, + type RuntimeSettingKey, + type SettingSpec, + type SettingValue, +} from "./runtime-settings-spec.js"; + +/** Stored document at `wa:settings:runtime`. */ +export interface RuntimeSettingsDoc { + values: Partial>; + updatedAt: string; + updatedBy: string; +} + +/** + * Per-node poll interval. Chosen over pub/sub: one GET every 5s per node is + * negligible next to the request-path Redis traffic, and it needs no extra + * subscriber connection. Worst-case propagation across the fleet is 5s. + */ +export const RUNTIME_SETTINGS_REFRESH_MS = 5_000; + +/** Stored overrides could not be read; the caller must not write. */ +export class RuntimeSettingsUnavailableError extends Error { + constructor(public readonly cause: string) { + super(`无法读取运行时配置（Redis）：${cause}`); + this.name = "RuntimeSettingsUnavailableError"; + } +} + +/** Placeholder returned instead of secret values. */ +export const SECRET_MASK = "••••••••"; +/** Typing this into a secret field clears it. */ +export const SECRET_CLEAR = "-"; + +export interface SettingItemView { + key: RuntimeSettingKey; + env: string; + group: string; + label: string; + type: SettingSpec["type"]; + min?: number; + max?: number; + step?: number; + /** Closed set for string settings; rendered as a ` 末尾注入： + + ```html + + ``` + +2. 直接响应 **`/ads.txt`**（不必源站提供），默认内容： + + ```text + google.com, pub-…, DIRECT, f08c47fec0942fa0 + ``` + +3. **默认跳过** `/admin`、`/api/*`、`/__lb/*`、`/cdn/*`、`/health`（管理后台与 API 不插广告） + +4. 注入后的响应带 `X-WeChat-AI-Adsense: 1`；`GET /__lb/health` 的 JSON 含 `adsense.enabled` / `adsense.client` + +控制台粘贴 `worker.js` 时，在 Variables 里加上 `ADSENSE_CLIENT`。关闭：清空 `ADSENSE_CLIENT` 或设 `ADSENSE_ENABLED=false` 后重新 Deploy。 + +自检： + +```bash +# 应看到 adsense.enabled / client +curl -s https://你的主域名/__lb/health + +# 应返回 google.com, pub-… 行 +curl -s https://你的主域名/ads.txt + +# 首页 HTML 的 应含 pagead2.googlesyndication.com +curl -s https://你的主域名/ | head -n 40 +``` + +说明：这是 **Auto ads 用的全局脚本**；若要用手动广告位，仍需在页面 HTML 里放 ``（或再扩展 Worker 注入广告位）。 + +### 本地调试 + +```bash +# 终端 1/2：两台源站（不同端口，同一 REDIS_URL） +# 终端 3： +npx wrangler dev --var ORIGINS:http://127.0.0.1:8787,http://127.0.0.1:8788 +``` + +Worker 自检：`GET /__lb/health` → `mode: non_blocking_probe`。 + +## DNS + +1. 主域名 **Custom Domain** 绑到本 Worker（或 CNAME 到 workers.dev 路由）。 +2. 源站 IP **不必** 橙云代理到用户；可仅允许 Cloudflare IP 访问 8787。 +3. 应用 `.env`： + +```env +PUBLIC_BASE_URL=https://你的主域名 +LINUXDO_REDIRECT_URI=https://你的主域名/api/v1/auth/callback +COOKIE_SECURE=true +WORKER_ID=node-01 # 每机唯一 +``` + +## 与 Cache Rules + +源站仍输出 `Cache-Control` / `Cloudflare-CDN-Cache-Control`。全站经 Worker 时，可继续用 Dashboard Cache Rules 缓存 HTML 壳与 `/cdn/s/*`（见 `docs/cloudflare.md`）。 + +## 算法摘要 + +1. 解析 `ORIGINS` +2. 按间隔对 `HEALTH_PATH` 探活，标记 healthy +3. 在 healthy 集合 round-robin；全挂时仍尝试一次 +4. 转发 Cookie / Method / Body；附加 `CF-Connecting-IP`、`X-Forwarded-*` +5. 响应增加 `X-WeChat-AI-LB: 1`（标识经 LB）；不向客户端暴露源站主机名 +6. 若配置了 AdSense：公开 HTML 注入脚本；`/ads.txt` 由边缘直接返回 + +## 扩缩容 + +1. 新机器 `docker run` 同镜像，配同一 `REDIS_URL` / `PUBLIC_BASE_URL`，唯一 `WORKER_ID` +2. 更新 Worker `ORIGINS` 并 `wrangler deploy` +3. 管理后台 **节点** 页应出现新 `WORKER_ID` 心跳 + +下线：从 `ORIGINS` 移除 → deploy；停容器后租约 TTL 过期自动转移 bot。 diff --git a/cloudflare-worker/package.json b/cloudflare-worker/package.json new file mode 100644 index 0000000..669ae72 --- /dev/null +++ b/cloudflare-worker/package.json @@ -0,0 +1,14 @@ +{ + "name": "wechat-ai-cf-worker", + "private": true, + "version": "0.1.0", + "description": "Cloudflare Worker reverse-proxy load balancer for WeChat-AI multi-node origins", + "scripts": { + "dev": "wrangler dev", + "deploy": "wrangler deploy", + "tail": "wrangler tail" + }, + "devDependencies": { + "wrangler": "^4.20.0" + } +} diff --git a/cloudflare-worker/src/adsense.ts b/cloudflare-worker/src/adsense.ts new file mode 100644 index 0000000..28bb7e2 --- /dev/null +++ b/cloudflare-worker/src/adsense.ts @@ -0,0 +1,156 @@ +import type { LbEnv } from "./origins"; + +/** Normalize to `ca-pub-…` or null if unset/invalid. */ +export function normalizeAdsenseClient( + raw: string | undefined | null, +): string | null { + const s = (raw || "").trim(); + if (!s) return null; + const ca = s.match(/^ca-pub-(\d+)$/i); + if (ca) return `ca-pub-${ca[1]}`; + const pub = s.match(/^pub-(\d+)$/i); + if (pub) return `ca-pub-${pub[1]}`; + if (/^\d+$/.test(s)) return `ca-pub-${s}`; + // Strict: only accept known shapes (avoid injecting arbitrary URLs) + return null; +} + +/** Publisher id for ads.txt (`pub-…`). */ +export function publisherIdFromClient(client: string): string { + return client.replace(/^ca-/i, ""); +} + +export function adsTxtFromClient(client: string): string { + const pub = publisherIdFromClient(client); + // Standard AdSense authorization record (CERTIFICATION_AUTHORITY_ID is fixed for Google) + return `google.com, ${pub}, DIRECT, f08c47fec0942fa0\n`; +} + +/** + * Paths that should never get ads (app consoles, API, LB internals, CDN, health). + * Comma-separated env `ADSENSE_SKIP_PATHS` overrides the default list. + * + * /app and /chatflow are logged-in consoles with no ad slots — injecting the + * loader there only cost a third-party DNS+TLS+~100KB (and a hanging socket + * where googlesyndication is unreachable), and the HTMLRewriter pass strips + * Content-Encoding, defeating the origin's pre-compressed shells. + */ +export function parseSkipPaths(env: LbEnv): string[] { + const raw = + env.ADSENSE_SKIP_PATHS?.trim() || + "/admin,/app,/chatflow,/api/,/__lb/,/cdn/,/health"; + return raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +} + +export function pathSkipped(pathname: string, skips: string[]): boolean { + const path = pathname.toLowerCase() || "/"; + for (const skip of skips) { + const sk = skip.toLowerCase(); + if (sk.endsWith("/")) { + if (path === sk.slice(0, -1) || path.startsWith(sk)) return true; + } else if (path === sk || path.startsWith(`${sk}/`)) { + return true; + } + } + return false; +} + +export function adsenseEnabled(env: LbEnv): boolean { + if (String(env.ADSENSE_ENABLED || "true").toLowerCase() === "false") { + return false; + } + return Boolean(normalizeAdsenseClient(env.ADSENSE_CLIENT)); +} + +export function shouldInjectAdsense( + pathname: string, + env: LbEnv, +): boolean { + if (!adsenseEnabled(env)) return false; + return !pathSkipped(pathname, parseSkipPaths(env)); +} + +/** + * Inject AdSense loader into `` of successful HTML responses. + * Strips Content-Length / Content-Encoding because the body changes. + */ +export function injectAdsenseScript( + response: Response, + client: string, +): Response { + const ct = (response.headers.get("content-type") || "").toLowerCase(); + if (!ct.includes("text/html")) return response; + if (response.status < 200 || response.status >= 300) return response; + + const safe = client.replace(/[^a-zA-Z0-9-]/g, ""); + if (!/^ca-pub-\d+$/i.test(safe)) return response; + + const snippet = + `\n\n`; + + const headers = new Headers(response.headers); + headers.delete("content-length"); + headers.delete("content-encoding"); + headers.set("X-WeChat-AI-Adsense", "1"); + + return new HTMLRewriter() + .on("head", { + element(el) { + el.append(snippet, { html: true }); + }, + }) + .transform( + new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }), + ); +} + +/** Apply injection when request path + env allow it. */ +export function maybeInjectAdsense( + request: Request, + response: Response, + env: LbEnv, +): Response { + if (request.method !== "GET") return response; + const pathname = new URL(request.url).pathname; + if (!shouldInjectAdsense(pathname, env)) return response; + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + if (!client) return response; + return injectAdsenseScript(response, client); +} + +/** Serve `/ads.txt` from Worker when AdSense is configured (or custom body). */ +export function tryServeAdsTxt( + request: Request, + env: LbEnv, +): Response | null { + const url = new URL(request.url); + if (url.pathname !== "/ads.txt") return null; + if (request.method !== "GET" && request.method !== "HEAD") return null; + + const custom = env.ADSENSE_ADS_TXT?.trim(); + let body: string | null = null; + if (custom) { + body = custom.endsWith("\n") ? custom : `${custom}\n`; + } else { + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + if (client) body = adsTxtFromClient(client); + } + if (!body) return null; + + const headers = { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "public, max-age=3600", + "X-WeChat-AI-Adsense": "ads.txt", + }; + if (request.method === "HEAD") { + return new Response(null, { status: 200, headers }); + } + return new Response(body, { status: 200, headers }); +} diff --git a/cloudflare-worker/src/index.ts b/cloudflare-worker/src/index.ts new file mode 100644 index 0000000..888d32d --- /dev/null +++ b/cloudflare-worker/src/index.ts @@ -0,0 +1,121 @@ +import { + getOrCreateStates, + parseOrigins, + pickCandidates, + nextRoundRobinIndex, + runBackgroundProbes, + probeOrigin, + type LbEnv, +} from "./origins"; +import { proxyRequest } from "./proxy"; +import { + adsenseEnabled, + maybeInjectAdsense, + normalizeAdsenseClient, + tryServeAdsTxt, +} from "./adsense"; + +export default { + async fetch( + request: Request, + env: LbEnv, + ctx: { waitUntil: (p: Promise) => void }, + ): Promise { + const url = new URL(request.url); + + if (url.pathname === "/__lb/health") { + const origins = parseOrigins(env.ORIGINS); + const states = getOrCreateStates(origins); + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + return Response.json({ + ok: true, + service: "wechat-ai-lb", + mode: "non_blocking_probe", + adsense: { + enabled: adsenseEnabled(env), + client: client || null, + }, + origins: states.map((s) => ({ + url: s.url, + healthy: s.healthy, + lastCheck: s.lastCheck, + failCount: s.failCount, + })), + }); + } + + // Edge-served ads.txt (AdSense site verification / authorization) + const adsTxt = tryServeAdsTxt(request, env); + if (adsTxt) return adsTxt; + + if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { + return new Response("WebSocket not supported", { status: 426 }); + } + + const origins = parseOrigins(env.ORIGINS); + if (!origins.length) { + return new Response( + JSON.stringify({ + error: "ORIGINS not configured", + hint: "Set wrangler vars ORIGINS to comma-separated node base URLs", + }), + { + status: 503, + headers: { "Content-Type": "application/json" }, + }, + ); + } + + const states = getOrCreateStates(origins); + + // Never block user requests on health probes (default). + if (String(env.HEALTH_ON_REQUEST || "").toLowerCase() === "true") { + const timeout = Math.max( + 300, + Number(env.HEALTH_TIMEOUT_MS ?? "1500") || 1500, + ); + const healthPath = (env.HEALTH_PATH || "/health").trim() || "/health"; + await Promise.all( + states.map((s) => probeOrigin(s, healthPath, timeout)), + ); + } else { + ctx.waitUntil(runBackgroundProbes(states, env)); + } + + const candidates = pickCandidates(states); + const start = nextRoundRobinIndex(candidates.length); + const order = + candidates.length <= 1 + ? candidates + : candidates.slice(start).concat(candidates.slice(0, start)); + + let lastErr: unknown; + for (const origin of order) { + try { + const res = await proxyRequest(request, origin.url, env); + if (res.status >= 502 && res.status <= 504 && order.length > 1) { + lastErr = new Error(`upstream ${res.status}`); + origin.failCount += 1; + if (origin.failCount >= 2) origin.healthy = false; + continue; + } + origin.healthy = true; + origin.failCount = 0; + return maybeInjectAdsense(request, res, env); + } catch (err) { + origin.failCount += 1; + if (origin.failCount >= 2) origin.healthy = false; + lastErr = err; + } + } + + const msg = lastErr instanceof Error ? lastErr.message : String(lastErr); + return new Response( + JSON.stringify({ error: "all origins failed", detail: msg }), + { + status: 502, + headers: { "Content-Type": "application/json" }, + }, + ); + }, +}; diff --git a/cloudflare-worker/src/origins.ts b/cloudflare-worker/src/origins.ts new file mode 100644 index 0000000..d5d3d78 --- /dev/null +++ b/cloudflare-worker/src/origins.ts @@ -0,0 +1,149 @@ +export interface OriginState { + url: string; + healthy: boolean; + lastCheck: number; + failCount: number; +} + +export interface LbEnv { + ORIGINS: string; + HEALTH_PATH?: string; + HEALTH_INTERVAL_MS?: string; + HEALTH_TIMEOUT_MS?: string; + /** If "true", await probes on every request (slow — debug only) */ + HEALTH_ON_REQUEST?: string; + ORIGIN_HOST_MODE?: string; + ORIGIN_PROXY_SECRET?: string; + /** + * Google AdSense client id, e.g. `ca-pub-…`. + * When set, Worker injects adsbygoogle.js into public HTML and may serve /ads.txt. + */ + ADSENSE_CLIENT?: string; + /** Set to "false" to disable injection even if ADSENSE_CLIENT is set */ + ADSENSE_ENABLED?: string; + /** + * Comma-separated path prefixes that never get ads. + * Default: /admin,/api/,/__lb/,/cdn/,/health + */ + ADSENSE_SKIP_PATHS?: string; + /** Optional full ads.txt body; if empty, generated from ADSENSE_CLIENT */ + ADSENSE_ADS_TXT?: string; +} + +const states = new Map(); +let rr = 0; +let bgProbeRunning = false; + +export function parseOrigins(raw: string | undefined): string[] { + if (!raw?.trim()) return []; + const out: string[] = []; + for (const part of raw.split(/[,\n]/)) { + let s = part.trim(); + if (!s) continue; + s = s.replace(/\/$/, ""); + if (!/^https?:\/\//i.test(s)) s = "http://" + s; + try { + out.push(new URL(s).origin); + } catch { + /* skip */ + } + } + return [...new Set(out)]; +} + +export function getOrCreateStates(origins: string[]): OriginState[] { + for (const url of origins) { + if (!states.has(url)) { + states.set(url, { + url, + healthy: true, + lastCheck: 0, + failCount: 0, + }); + } + } + for (const key of [...states.keys()]) { + if (!origins.includes(key)) states.delete(key); + } + return origins.map((u) => states.get(u)!); +} + +export async function probeOrigin( + state: OriginState, + healthPath: string, + timeoutMs: number, +): Promise { + const path = healthPath.startsWith("/") ? healthPath : `/${healthPath}`; + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const res = await fetch(`${state.url}${path}`, { + method: "GET", + redirect: "manual", + signal: ctrl.signal, + headers: { Accept: "application/json, */*", "Cache-Control": "no-store" }, + }); + try { + await res.arrayBuffer(); + } catch { + /* */ + } + const ok = res.status >= 200 && res.status < 400; + state.healthy = ok; + state.failCount = ok ? 0 : state.failCount + 1; + state.lastCheck = Date.now(); + return ok; + } catch { + state.healthy = false; + state.failCount += 1; + state.lastCheck = Date.now(); + return false; + } finally { + clearTimeout(t); + } +} + +/** Background probes only — do not await on request path. */ +export async function runBackgroundProbes( + origins: OriginState[], + env: LbEnv, +): Promise { + if (bgProbeRunning || !origins.length) return; + bgProbeRunning = true; + try { + const interval = Math.max( + 3000, + Number(env.HEALTH_INTERVAL_MS ?? "15000") || 15_000, + ); + const timeout = Math.max( + 300, + Number(env.HEALTH_TIMEOUT_MS ?? "1500") || 1500, + ); + const healthPath = (env.HEALTH_PATH || "/health").trim() || "/health"; + const now = Date.now(); + const due = origins.filter((s) => now - s.lastCheck >= interval); + for (let i = 0; i < due.length; i += 4) { + const chunk = due.slice(i, i + 4); + await Promise.all(chunk.map((s) => probeOrigin(s, healthPath, timeout))); + } + } finally { + bgProbeRunning = false; + } +} + +export function pickCandidates(origins: OriginState[]): OriginState[] { + const healthy = origins.filter((s) => s.healthy); + return healthy.length ? healthy : origins; +} + +export function nextRoundRobinIndex(len: number): number { + if (len <= 1) return 0; + const i = rr % len; + rr = (rr + 1) % len; + return i; +} + +export function pickOrigin(candidates: OriginState[]): OriginState { + if (candidates.length === 1) return candidates[0]!; + return candidates[nextRoundRobinIndex(candidates.length)]!; +} diff --git a/cloudflare-worker/src/proxy.ts b/cloudflare-worker/src/proxy.ts new file mode 100644 index 0000000..1f99837 --- /dev/null +++ b/cloudflare-worker/src/proxy.ts @@ -0,0 +1,124 @@ +import type { LbEnv } from "./origins"; + +const HOP_BY_HOP = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "host", + "cf-connecting-ip", + "cf-ray", + "cf-visitor", + "cf-ipcountry", + "cdn-loop", + "x-forwarded-for", + "x-forwarded-proto", + "x-real-ip", +]); + +/** Response headers from origin that must not reach clients */ +const STRIP_RESPONSE_HEADERS = [ + "x-wechat-ai-origin", + "x-railway-edge", + "x-railway-request-id", + "x-railway-cdn-edge", + "x-hikari-trace", + "server-timing", +] as const; + +export function buildUpstreamHeaders( + req: Request, + originUrl: string, + env: LbEnv, + clientHost: string, +): Headers { + const out = new Headers(); + req.headers.forEach((value, key) => { + const lk = key.toLowerCase(); + if (HOP_BY_HOP.has(lk)) return; + if (lk.startsWith("cf-")) return; + out.set(key, value); + }); + + const mode = (env.ORIGIN_HOST_MODE || "preserve").toLowerCase(); + if (mode === "origin") { + try { + out.set("Host", new URL(originUrl).host); + } catch { + /* */ + } + } else if (clientHost) { + out.set("Host", clientHost); + } + + const clientIp = + req.headers.get("CF-Connecting-IP") || + req.headers.get("X-Forwarded-For")?.split(",")[0]?.trim() || + ""; + if (clientIp) { + out.set("CF-Connecting-IP", clientIp); + out.set("X-Real-IP", clientIp); + const prior = req.headers.get("X-Forwarded-For"); + out.set( + "X-Forwarded-For", + prior ? `${clientIp}, ${prior}` : clientIp, + ); + } + + const proto = + req.headers.get("X-Forwarded-Proto") || + (new URL(req.url).protocol === "https:" ? "https" : "http"); + out.set("X-Forwarded-Proto", proto); + if (clientHost) out.set("X-Forwarded-Host", clientHost); + + if (env.ORIGIN_PROXY_SECRET) { + out.set("X-WeChat-AI-Proxy-Secret", env.ORIGIN_PROXY_SECRET); + } + + return out; +} + +export async function proxyRequest( + req: Request, + originBase: string, + env: LbEnv, +): Promise { + const incoming = new URL(req.url); + const target = new URL( + incoming.pathname + incoming.search, + originBase.endsWith("/") ? originBase : originBase + "/", + ); + + const clientHost = incoming.host; + const headers = buildUpstreamHeaders(req, originBase, env, clientHost); + + const init: RequestInit = { + method: req.method, + headers, + redirect: "manual", + }; + + if (req.method !== "GET" && req.method !== "HEAD") { + init.body = req.body; + // @ts-expect-error duplex required for streaming body in Workers + init.duplex = "half"; + } + + const upstream = await fetch(target.toString(), init); + const resHeaders = new Headers(upstream.headers); + resHeaders.set("X-WeChat-AI-LB", "1"); + // Strip origin infra / ops headers (do not leak Railway host, edge, etc.) + for (const h of STRIP_RESPONSE_HEADERS) { + resHeaders.delete(h); + } + + return new Response(upstream.body, { + status: upstream.status, + statusText: upstream.statusText, + headers: resHeaders, + }); +} diff --git a/cloudflare-worker/worker.js b/cloudflare-worker/worker.js new file mode 100644 index 0000000..bc89608 --- /dev/null +++ b/cloudflare-worker/worker.js @@ -0,0 +1,440 @@ +const HOP_BY_HOP = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + "host", + "cf-connecting-ip", + "cf-ray", + "cf-visitor", + "cf-ipcountry", + "cdn-loop", + "x-forwarded-for", + "x-forwarded-proto", + "x-real-ip", +]); + +/** @type {Map} */ +const states = new Map(); +let rr = 0; +let bgProbeRunning = false; + +function parseOrigins(raw) { + if (!raw || !String(raw).trim()) return []; + const out = []; + for (const part of String(raw).split(/[,\n]/)) { + let s = part.trim(); + if (!s) continue; + s = s.replace(/\/$/, ""); + if (!/^https?:\/\//i.test(s)) s = "http://" + s; + try { + out.push(new URL(s).origin); + } catch { + /* skip */ + } + } + return [...new Set(out)]; +} + +function getOrCreateStates(origins) { + for (const url of origins) { + if (!states.has(url)) { + // Start optimistic healthy — never block first request on probe + states.set(url, { url, healthy: true, lastCheck: 0, failCount: 0 }); + } + } + for (const key of [...states.keys()]) { + if (!origins.includes(key)) states.delete(key); + } + return origins.map((u) => states.get(u)); +} + +async function probeOrigin(state, healthPath, timeoutMs) { + const path = healthPath.startsWith("/") ? healthPath : `/${healthPath}`; + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const res = await fetch(`${state.url}${path}`, { + method: "GET", + redirect: "manual", + signal: ctrl.signal, + headers: { Accept: "application/json, */*", "Cache-Control": "no-store" }, + // Avoid CF caching the health fetch oddly + cf: { cacheTtl: 0, cacheEverything: false }, + }); + const ok = res.status >= 200 && res.status < 400; + // drain body so connection can reuse + try { + await res.arrayBuffer(); + } catch { + /* */ + } + state.healthy = ok; + state.failCount = ok ? 0 : state.failCount + 1; + state.lastCheck = Date.now(); + return ok; + } catch { + state.healthy = false; + state.failCount += 1; + state.lastCheck = Date.now(); + return false; + } finally { + clearTimeout(t); + } +} + +/** + * Background-only probes. Never call this with await on the request path + * unless HEALTH_ON_REQUEST=true (debug). + */ +async function runBackgroundProbes(origins, env) { + if (bgProbeRunning || !origins.length) return; + bgProbeRunning = true; + try { + const interval = Math.max( + 3000, + Number(env.HEALTH_INTERVAL_MS || "15000") || 15000, + ); + const timeout = Math.max( + 300, + Number(env.HEALTH_TIMEOUT_MS || "1500") || 1500, + ); + // Lightweight process check by default (NOT /health/ready + Redis) + const healthPath = (env.HEALTH_PATH || "/health").trim() || "/health"; + const now = Date.now(); + const due = origins.filter((s) => now - s.lastCheck >= interval); + if (!due.length) return; + // Cap concurrency: probe at most 4 at a time to avoid stampede + for (let i = 0; i < due.length; i += 4) { + const chunk = due.slice(i, i + 4); + await Promise.all(chunk.map((s) => probeOrigin(s, healthPath, timeout))); + } + } finally { + bgProbeRunning = false; + } +} + +/** Prefer known-healthy; if none known yet, use all (optimistic). */ +function pickCandidates(origins) { + const healthy = origins.filter((s) => s.healthy); + if (healthy.length) return healthy; + // All marked bad — still try them (avoid total outage from stale probes) + return origins; +} + +function buildUpstreamHeaders(req, originUrl, env, clientHost) { + const out = new Headers(); + req.headers.forEach((value, key) => { + const lk = key.toLowerCase(); + if (HOP_BY_HOP.has(lk)) return; + if (lk.startsWith("cf-")) return; + out.set(key, value); + }); + + const mode = String(env.ORIGIN_HOST_MODE || "preserve").toLowerCase(); + if (mode === "origin") { + try { + out.set("Host", new URL(originUrl).host); + } catch { + /* */ + } + } else if (clientHost) { + out.set("Host", clientHost); + } + + const clientIp = + req.headers.get("CF-Connecting-IP") || + req.headers.get("X-Forwarded-For")?.split(",")[0]?.trim() || + ""; + if (clientIp) { + out.set("CF-Connecting-IP", clientIp); + out.set("X-Real-IP", clientIp); + const prior = req.headers.get("X-Forwarded-For"); + out.set("X-Forwarded-For", prior ? `${clientIp}, ${prior}` : clientIp); + } + + const proto = + req.headers.get("X-Forwarded-Proto") || + (new URL(req.url).protocol === "https:" ? "https" : "http"); + out.set("X-Forwarded-Proto", proto); + if (clientHost) out.set("X-Forwarded-Host", clientHost); + + if (env.ORIGIN_PROXY_SECRET) { + out.set("X-WeChat-AI-Proxy-Secret", env.ORIGIN_PROXY_SECRET); + } + + return out; +} + +async function proxyRequest(req, originBase, env) { + const incoming = new URL(req.url); + const target = new URL( + incoming.pathname + incoming.search, + originBase.endsWith("/") ? originBase : originBase + "/", + ); + + const headers = buildUpstreamHeaders(req, originBase, env, incoming.host); + /** @type {RequestInit} */ + const init = { + method: req.method, + headers, + redirect: "manual", + }; + + if (req.method !== "GET" && req.method !== "HEAD") { + init.body = req.body; + // @ts-ignore + init.duplex = "half"; + } + + const upstream = await fetch(target.toString(), init); + const resHeaders = new Headers(upstream.headers); + resHeaders.set("X-WeChat-AI-LB", "1"); + // Strip origin infra / ops headers (do not leak Railway host, edge, etc.) + for (const h of [ + "x-wechat-ai-origin", + "x-railway-edge", + "x-railway-request-id", + "x-railway-cdn-edge", + "x-hikari-trace", + "server-timing", + ]) { + resHeaders.delete(h); + } + + return new Response(upstream.body, { + status: upstream.status, + statusText: upstream.statusText, + headers: resHeaders, + }); +} + +// ── Google AdSense (edge inject + ads.txt) ────────────────────────── + +function normalizeAdsenseClient(raw) { + const s = (raw || "").trim(); + if (!s) return null; + const ca = s.match(/^ca-pub-(\d+)$/i); + if (ca) return `ca-pub-${ca[1]}`; + const pub = s.match(/^pub-(\d+)$/i); + if (pub) return `ca-pub-${pub[1]}`; + if (/^\d+$/.test(s)) return `ca-pub-${s}`; + return null; +} + +function adsTxtFromClient(client) { + const pub = client.replace(/^ca-/i, ""); + return `google.com, ${pub}, DIRECT, f08c47fec0942fa0\n`; +} + +function parseSkipPaths(env) { + const raw = + (env.ADSENSE_SKIP_PATHS && String(env.ADSENSE_SKIP_PATHS).trim()) || + "/admin,/api/,/__lb/,/cdn/,/health"; + return raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean); +} + +function pathSkipped(pathname, skips) { + const path = (pathname || "/").toLowerCase(); + for (const skip of skips) { + const sk = skip.toLowerCase(); + if (sk.endsWith("/")) { + if (path === sk.slice(0, -1) || path.startsWith(sk)) return true; + } else if (path === sk || path.startsWith(sk + "/")) { + return true; + } + } + return false; +} + +function adsenseEnabled(env) { + if (String(env.ADSENSE_ENABLED || "true").toLowerCase() === "false") { + return false; + } + return Boolean(normalizeAdsenseClient(env.ADSENSE_CLIENT)); +} + +function shouldInjectAdsense(pathname, env) { + if (!adsenseEnabled(env)) return false; + return !pathSkipped(pathname, parseSkipPaths(env)); +} + +function injectAdsenseScript(response, client) { + const ct = (response.headers.get("content-type") || "").toLowerCase(); + if (!ct.includes("text/html")) return response; + if (response.status < 200 || response.status >= 300) return response; + + const safe = client.replace(/[^a-zA-Z0-9-]/g, ""); + if (!/^ca-pub-\d+$/i.test(safe)) return response; + + const snippet = + `\n\n`; + + const headers = new Headers(response.headers); + headers.delete("content-length"); + headers.delete("content-encoding"); + headers.set("X-WeChat-AI-Adsense", "1"); + + return new HTMLRewriter() + .on("head", { + element(el) { + el.append(snippet, { html: true }); + }, + }) + .transform( + new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }), + ); +} + +function maybeInjectAdsense(request, response, env) { + if (request.method !== "GET") return response; + const pathname = new URL(request.url).pathname; + if (!shouldInjectAdsense(pathname, env)) return response; + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + if (!client) return response; + return injectAdsenseScript(response, client); +} + +function tryServeAdsTxt(request, env) { + const url = new URL(request.url); + if (url.pathname !== "/ads.txt") return null; + if (request.method !== "GET" && request.method !== "HEAD") return null; + + const custom = env.ADSENSE_ADS_TXT && String(env.ADSENSE_ADS_TXT).trim(); + let body = null; + if (custom) { + body = custom.endsWith("\n") ? custom : custom + "\n"; + } else { + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + if (client) body = adsTxtFromClient(client); + } + if (!body) return null; + + const headers = { + "Content-Type": "text/plain; charset=utf-8", + "Cache-Control": "public, max-age=3600", + "X-WeChat-AI-Adsense": "ads.txt", + }; + if (request.method === "HEAD") { + return new Response(null, { status: 200, headers }); + } + return new Response(body, { status: 200, headers }); +} + +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + + if (url.pathname === "/__lb/health") { + const origins = parseOrigins(env.ORIGINS); + const list = getOrCreateStates(origins); + const client = normalizeAdsenseClient(env.ADSENSE_CLIENT); + return Response.json({ + ok: true, + service: "wechat-ai-lb", + mode: "non_blocking_probe", + adsense: { + enabled: adsenseEnabled(env), + client: client || null, + }, + origins: list.map((s) => ({ + url: s.url, + healthy: s.healthy, + lastCheck: s.lastCheck, + failCount: s.failCount, + })), + }); + } + + const adsTxt = tryServeAdsTxt(request, env); + if (adsTxt) return adsTxt; + + if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { + return new Response("WebSocket not supported", { status: 426 }); + } + + const origins = parseOrigins(env.ORIGINS); + if (!origins.length) { + return new Response( + JSON.stringify({ + error: "ORIGINS not configured", + hint: "Settings → Variables → ORIGINS = http://ip1:8787,http://ip2:8787", + }), + { status: 503, headers: { "Content-Type": "application/json" } }, + ); + } + + const list = getOrCreateStates(origins); + + // Optional debug: old blocking path (DO NOT use in production) + if (String(env.HEALTH_ON_REQUEST || "").toLowerCase() === "true") { + const timeout = Math.max( + 300, + Number(env.HEALTH_TIMEOUT_MS || "1500") || 1500, + ); + const healthPath = (env.HEALTH_PATH || "/health").trim() || "/health"; + await Promise.all( + list.map((s) => probeOrigin(s, healthPath, timeout)), + ); + } else if (ctx && typeof ctx.waitUntil === "function") { + // Fire-and-forget: never await on request path + ctx.waitUntil(runBackgroundProbes(list, env)); + } + + const candidates = pickCandidates(list); + // Try primary + failover without long waits (only real proxy errors) + const order = [...candidates]; + // rotate start + if (order.length > 1) { + const start = rr % order.length; + rr = (rr + 1) % order.length; + const rotated = order.slice(start).concat(order.slice(0, start)); + let lastErr; + for (const origin of rotated) { + try { + const res = await proxyRequest(request, origin.url, env); + // Soft signal: 5xx from origin → try next if multi + if (res.status >= 502 && res.status <= 504 && rotated.length > 1) { + lastErr = new Error(`upstream ${res.status}`); + continue; + } + // mark success + origin.healthy = true; + origin.failCount = 0; + return maybeInjectAdsense(request, res, env); + } catch (err) { + origin.failCount += 1; + if (origin.failCount >= 2) origin.healthy = false; + lastErr = err; + } + } + const msg = lastErr instanceof Error ? lastErr.message : String(lastErr); + return new Response( + JSON.stringify({ error: "all origins failed", detail: msg }), + { status: 502, headers: { "Content-Type": "application/json" } }, + ); + } + + try { + const res = await proxyRequest(request, order[0].url, env); + return maybeInjectAdsense(request, res, env); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return new Response( + JSON.stringify({ error: "origin failed", detail: msg }), + { status: 502, headers: { "Content-Type": "application/json" } }, + ); + } + }, +}; diff --git a/cloudflare-worker/wrangler.toml b/cloudflare-worker/wrangler.toml new file mode 100644 index 0000000..fe21381 --- /dev/null +++ b/cloudflare-worker/wrangler.toml @@ -0,0 +1,30 @@ +# Cloudflare Worker: WeChat-AI multi-origin load balancer +# +# 1. Set ORIGINS to comma-separated origin base URLs (direct node addresses). +# 2. wrangler deploy +# 3. Bind your main domain (Custom Domain) to this Worker. +# +# Application PUBLIC_BASE_URL must be the main domain, not individual origins. + +name = "wechat-ai-lb" +main = "src/index.ts" +compatibility_date = "2025-07-01" +compatibility_flags = ["nodejs_compat"] + +[vars] +# Comma-separated origins, e.g. "http://1.2.3.4:8787,http://5.6.7.8:8787" +ORIGINS = "" +# Lightweight process health (background only — does not block user requests) +HEALTH_PATH = "/health" +HEALTH_INTERVAL_MS = "15000" +HEALTH_TIMEOUT_MS = "1500" +# preserve = forward client Host (recommended when PUBLIC_BASE_URL is main domain) +# origin = rewrite Host to origin hostname +ORIGIN_HOST_MODE = "preserve" +# Google AdSense: inject adsbygoogle.js into public HTML + serve /ads.txt +# Clear ADSENSE_CLIENT or set ADSENSE_ENABLED=false to disable +ADSENSE_CLIENT = "" +ADSENSE_ENABLED = "false" +# 登录态控制台（/app /chatflow /admin）不投广告：无广告位，且注入会去掉 +# Content-Encoding，让源站启动时预压缩的 shell 失效 +ADSENSE_SKIP_PATHS = "/admin,/app,/chatflow,/api/,/__lb/,/cdn/,/health" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e64a1d2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,87 @@ +# WeChat-AI +# +# 1. 配置 .env（REDIS_URL / LINUXDO_* / LLM_* / PUBLIC_BASE_URL） +# 用户自定义 LLM + 联网搜索：TOOLS_BASE_URL / TOOLS_API_KEY → wechat-ai-tools +# 2. pnpm docker:up +# 升版 + 本地 OTA pack + compose up --build +# 通道发布：/admin → 上传 files.json（无需 CLI Cookie） +# （直接 docker compose up -d --build 不会升版本、不会 pack） +# 3. 打开 https://你的域名/app +# +# 注意：容器内 WECHAT_AI_HOST 强制 0.0.0.0；OAuth 回调请用公网域名。 +# +# 多节点：本 compose 默认单副本。多机时在各服务器分别 docker run 同镜像， +# 共享 REDIS_URL / PUBLIC_BASE_URL（主域名），每机设唯一 WORKER_ID， +# 主域名反代用 cloudflare-worker（静态 ORIGINS）。见 docs/docker.md。 +# +# 工具网关（可选 profile tools）： +# docker compose --profile tools up -d --build +# 主站 .env: TOOLS_BASE_URL=http://wechat-ai-tools:7860 TOOLS_API_KEY=... + +services: + wechat-ai: + build: + context: . + dockerfile: Dockerfile + image: wechat-ai:latest + container_name: wechat-ai + restart: unless-stopped + # Give the graceful shutdown time to release bot leases before SIGKILL + stop_grace_period: 30s + ports: + - "${WECHAT_AI_PORT:-8787}:8787" + env_file: + - .env + environment: + WECHAT_AI_HOST: "0.0.0.0" + WECHAT_AI_PORT: "8787" + NODE_ENV: production + # 单机默认；多节点请在各机覆盖为唯一 ID + # WORKER_ID: node-01 + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:8787/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))", + ] + interval: 30s + timeout: 8s + retries: 3 + start_period: 50s + + # User custom LLM egress + web search (platform LLM stays on wechat-ai) + wechat-ai-tools: + profiles: ["tools"] + build: + context: ./huggingface/wechat-ai-tools + dockerfile: Dockerfile + image: wechat-ai-tools:latest + container_name: wechat-ai-tools + restart: unless-stopped + ports: + - "${TOOLS_PORT:-7860}:7860" + environment: + HOST: "0.0.0.0" + PORT: "7860" + TOOLS_API_KEY: "${TOOLS_API_KEY:-}" + ALLOW_REQUEST_UPSTREAM: "true" + UPSTREAM_DENY_PRIVATE: "true" + SEARCH_PROVIDER: "${SEARCH_PROVIDER:-ddg}" + # Optional demo fallback only (main site uses its own LLM_* for platform) + UPSTREAM_LLM_BASE_URL: "${UPSTREAM_LLM_BASE_URL:-}" + UPSTREAM_LLM_API_KEY: "${UPSTREAM_LLM_API_KEY:-}" + UPSTREAM_LLM_MODEL: "${UPSTREAM_LLM_MODEL:-gpt-4o-mini}" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/health', timeout=5)", + ] + interval: 30s + timeout: 8s + retries: 3 + start_period: 25s diff --git a/docs/ACCEPTANCE.md b/docs/ACCEPTANCE.md new file mode 100644 index 0000000..44a121b --- /dev/null +++ b/docs/ACCEPTANCE.md @@ -0,0 +1,70 @@ +# 验收状态 + +**日期**：2026-07-19 +**结论**：**系统已完成并可验收**（离线/自动化门禁 + 文档化真机清单） + +## 已交付范围（v1） + +| 能力 | 实现 | +|------|------| +| 直连 iLink（无 OpenClaw） | `packages/ilink` | +| 多用户隔离会话/记忆 | `packages/core` + DB | +| 人设模板与发布 | seed 猫娘/女友 + Admin/API | +| 仅后台分配角色 | `/角色` 拒绝自助 | +| OpenAI 兼容 LLM | `packages/llm` | +| 多 Bot 登录 | `pnpm ilink:login -- --name …` | +| Admin Web | `/admin` | +| 限流 / typing / 非文字提示 | worker | +| 语音转写文本（有则用） | `extractText` | +| 运维 | `pnpm diag`、`docs/runbook.md` | +| E2E 清单 | `docs/e2e-checklist.md` | + +## 自动化门禁 + +```bash +pnpm accept +``` + +包含：全仓单元测试、migrate/seed、diag、关键文件存在性。 + +## 真机项（人工） + +见 `docs/e2e-checklist.md` 章节 C–E（需扫码 + 真实 LLM Key）。 + +## 非目标 / 可选后续 + +- ~~图片/文件 CDN 加解密全链路 / 入站 Vision~~ → **入站图片已实现**（CDN 下载 + AES-128-ECB 解密 + 视觉模型，`VISION_ENABLED`，默认关）。仍未做：入站语音 ASR（微信 SILK/AMR 无 OpenAI 兼容端点可用，现依赖 iLink 自带转写）、入站文件解析 +- Embedding 记忆检索（无 Embedding API；已用文本 top-k 替代） +- 多模型路由 +- 联网搜索工具 +- React 工程化 Admin +- Windows 服务安装包 +- 固定时段问候（可叠在主动联系调度器上） + +## 已补充（2026-07-20） + +- 无向量记忆 top-k（`MEMORY_TOP_K` / `MEMORY_FULL_INJECT_MAX`）+ 条数上限/去重 +- 记忆治理：单条删除 + 用户中心「记忆」面板 +- `get_current_time` 工具（`TIME_TOOL_ENABLED`，默认开） + +## 主动找用户（空闲） + +- 全局 `PROACTIVE_ENABLED` + Bot 开关 + peer「允许主动」 +- LLM 生成；可 skip；见 `docs/runbook.md` §2.1 + +## AI 网关 + Chatflow（2026-07-26） + +| 能力 | 实现 | +|------|------| +| 唯一外网 AI 出口 | `huggingface/wechat-ai-tools`（FastAPI + Dockerfile） | +| 用户自定义 OpenAI 兼容 API | 加密存储；**仅** HF 代发，主站不 dial 用户 base_url | +| 联网搜索 | 仅 HF `/v1/web-search`；全局 + 人设双开关 | +| 「我的模型」UI | `/app` → 我的模型（增删改 / 启停 / 掩码 Key） | +| Chatflow 引擎 | `packages/core/src/chatflow/*`；节点 start/llm/answer/if-else/http/memory/search | +| Chatflow 编辑器 | `/chatflow`（拖拽 + 连线 + 属性 + 保存启用） | +| 试聊支持 chatflow | 强制平台上游，不消耗作者额度 | +| 就绪探针 | `/health/ready` 纳入 tools 健康（缓存 15s） | + +约束：chatflow 不做主动联系；Fork 不继承作者密钥；http 节点默认仅工具域。 + +文档：`docs/ai-gateway.md`、`docs/chatflow.md`；验收项见 `docs/e2e-checklist.md` §I / §J。 diff --git a/docs/admin-api.md b/docs/admin-api.md new file mode 100644 index 0000000..ce60f24 --- /dev/null +++ b/docs/admin-api.md @@ -0,0 +1,280 @@ +# API 概览（Redis + LINUX DO OAuth） + +Base: `http://127.0.0.1:8787` +Auth: **Cookie 会话**（OAuth 登录后 `wa_session`），`credentials: include` + +## 认证 + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/auth/config` | `{ oauthEnabled, provider, localAuthEnabled, inviteRequiredForLocal, passwordMinLength }` | +| GET | `/api/v1/auth/login` | 跳转 LINUX DO OAuth | +| GET | `/api/v1/auth/callback` | OAuth 回调（新 OAuth 用户**不需要**邀请码） | +| POST | `/api/v1/auth/register` | 本地注册：`{ inviteCode, username, password, name? }` → cookie | +| POST | `/api/v1/auth/password-login` | 用户名密码登录：`{ username, password }` → cookie | +| GET | `/api/v1/auth/invite/:code` | 预检邀请码（不消费） | +| POST | `/api/v1/auth/logout` | 退出 | +| GET | `/api/v1/auth/me` | 当前用户（含 `authProvider`、`isSuperAdmin`，不含密码） | + +## 邀请（登录用户） + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/me/invites` | pending 列表 + 配额 + 邀请链接 | +| POST | `/api/v1/me/invites` | 生成一次性码；受「每 X 小时 N 个」滑动窗口限制 | +| DELETE | `/api/v1/me/invites/:code` | 撤销未使用码 | + +邀请链接形态：`{PUBLIC_BASE_URL}/app?invite={CODE}`（打开后自动填入注册表单）。 + +## 用户 + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/me/bots` | 我的机器人 | +| POST | `/api/v1/me/bots/login/start` | 扫码添加机器人（新建 botId） | +| POST | `/api/v1/me/bots/:botId/relogin/start` | **重新扫码绑定**（更新 token，保留 peers/记忆/分配） | +| GET | `/api/v1/me/bots/login/:sessionId` | 扫码状态 | +| DELETE | `/api/v1/me/bots/:botId` | 删除自己的机器人（含 Redis token） | +| GET | `/api/v1/me/peers` | 私聊用户 | +| POST | `/api/v1/me/peers/approve` | 批准 | +| PUT | `/api/v1/me/assignments` | 分配人设（须在库中） | +| GET | `/api/v1/me/personas` | 我的库 + 我创建的 | +| POST | `/api/v1/me/personas/:id/add` | 添加人设到库 | +| DELETE | `/api/v1/me/personas/:id` | 从库移除 | +| GET | `/api/v1/me/memories?botAccountId=&peerId=&personaId?` | 长期记忆；无 personaId 时返回 `{total,groups}` | +| POST | `/api/v1/me/memories/reset` | `{ botAccountId, peerId, personaId? }` 清空记忆 | +| DELETE | `/api/v1/me/memories` | `{ botAccountId, peerId, personaId, memoryId }` 删单条 | +| GET | `/api/v1/me/wechat-bind` | 当前 LINUX DO ↔ 微信绑定状态（`reachable` 表示是否有 context_token） | +| POST | `/api/v1/me/wechat-bind/code` | 生成 6 位绑定码（微信 `/绑定 CODE`） | +| DELETE | `/api/v1/me/wechat-bind` | 解除绑定并结束相关用户对话 | +| GET | `/api/v1/me/blocks` | 我的黑名单（LINUX DO 用户） | +| POST | `/api/v1/me/blocks` | `{ username }` 或 `{ userId }` 拉黑 | +| DELETE | `/api/v1/me/blocks/:userId` | 取消拉黑 | + +## 人设广场 + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/square/personas?q=&page=&limit=&sort=heat\|use\|recent\|name` | 搜索公开人设（默认 sort=heat；字段含 useCount / assignCount / forkCount / heatScore / forkedFrom） | +| GET | `/api/v1/square/personas/:id` | 详情（含 systemPrompt） | +| POST | `/api/v1/square/personas` | 发布（public/private） | +| POST | `/api/v1/square/personas/:id/fork` | Fork 为当前用户私有草稿（`PERSONA_FORK_ENABLED`，默认开） | +| PUT | `/api/v1/square/personas/:id` | 作者更新 | +| DELETE | `/api/v1/square/personas/:id` | 作者软删除 | +| GET | `/api/v1/square/stickers?q=&page=&limit=&sort=use\|recent\|name` | 已审核公开表情包（默认 sort=use） | +| GET | `/api/v1/square/stickers/:id` | 详情（`imageUrl` 对公开已审为 CDN 路径） | +| GET | `/api/v1/square/stickers/:id/image` | 预览图（**鉴权**；私有/待审/作者预览） | +| GET | `/cdn/s/:id?v={content_hash}` | **公开 CDN**（无 Cookie）：仅 `public`+`approved`+`enabled`；长缓存 immutable | +| POST | `/api/v1/square/stickers` | 投稿（public→待审 / private 自用） | +| PUT | `/api/v1/square/stickers/:id` | 作者更新（公开改动回待审） | +| DELETE | `/api/v1/square/stickers/:id` | 作者软删除 | +| GET | `/api/v1/me/stickers` | 我的库 + 我创建的 | +| POST | `/api/v1/me/stickers/:id/add` | 加入表情库 | +| DELETE | `/api/v1/me/stickers/:id` | 从库移除（对称递减 use_count） | + +## 网页试聊 + +不经微信，在 `/app` 内限量体验公开/可用人设。计入用户 Token 用量；会话存 Redis TTL。 + +| Method | Path | 说明 | +|--------|------|------| +| POST | `/api/v1/try-chat/sessions` | `{ personaId, botName? }` → `{ sessionId, persona, remainingToday, expiresInSec }` | +| POST | `/api/v1/try-chat/sessions/:sessionId/messages` | `{ text }` → `{ messages[], remainingToday, remainingSession, usage }` | +| DELETE | `/api/v1/try-chat/sessions/:sessionId` | 结束会话 | + +环境变量：`TRY_CHAT_ENABLED`、`TRY_CHAT_MAX_USER_MSGS_PER_DAY`、`TRY_CHAT_MAX_USER_MSGS_PER_SESSION`、`TRY_CHAT_SESSION_TTL_SEC`、`TRY_CHAT_MAX_HISTORY`。 + +热度公式：`heatScore = use_count*2 + assign_count*5 + fork_count*3`（分配仅在 persona 变化时 +1）。 + +## 管理员 + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/admin/dashboard` | 仪表盘 + 今日 Token + **全舰队** workerStats + nodes 摘要 + Redis | +| GET | `/api/v1/admin/system` | 系统健康 / doctor / **全舰队** workerStats / nodes 摘要 | +| GET | `/api/v1/admin/nodes` | **仅超管** 部署节点列表（含 fenced） | +| POST | `/api/v1/admin/nodes/:workerId/force-offline` | **仅超管** 强制下线 | +| POST | `/api/v1/admin/nodes/:workerId/clear-fence` | **仅超管** 解除封锁 | +| POST | `/api/v1/admin/nodes/:workerId/weight` | **仅超管** 设负载权重 `{ weight: 0..500 }` | +| DELETE | `/api/v1/admin/nodes/:workerId/weight` | **仅超管** 恢复默认权重（100%） | +| POST | `/api/v1/admin/nodes/weights/prune` | **仅超管** 立即清理失效权重（平时会自动过期） | +| POST | `/api/v1/admin/workers/restart-all` | **仅超管** 批量恢复 pollable 并尝试认领（有 token 的 active bot） | +| POST | `/api/v1/admin/workers/stop-all` | **仅超管** 暂停全部可轮询 bot（写 pause 标记，不删账号） | +| POST | `/api/v1/admin/system/seed-personas` | 幂等补种官方人设 | +| GET | `/api/v1/admin/memories?botAccountId=&peerId=` | 查看 peer 长期记忆（按人设分组） | +| DELETE | `/api/v1/admin/memories` | `{ botAccountId, peerId, personaId, memoryId }` 删单条 | +| POST | `/api/v1/admin/memories/reset` | `{ botAccountId, peerId, personaId? }` 清记忆 | +| POST | `/api/v1/admin/messages/clear` | `{ botAccountId, peerId }` 清除短期对话历史（保留长期记忆） | +| GET | `/api/v1/admin/peers?status=unapproved\|approved\|all` | 全站私聊 peer（默认待批准） | +| POST | `/api/v1/admin/peers/approve` | `{ botAccountId, peerId }` 管理员代批 | +| POST | `/api/v1/admin/peers/approve-all` | 批准全部待批准 peer | +| GET | `/api/v1/admin/usage?day=` | 按日用量 | +| GET | `/api/v1/admin/usage?days=7` | 近 N 日用量列表 | +| GET | `/api/v1/admin/users` | 用户全量列表（含 botCount、isBanned、authProvider、isSuperAdmin） | +| GET | `/api/v1/admin/users/:id` | 用户详情 + 名下机器人 | +| PATCH | `/api/v1/admin/users/:id` | **仅超管** `{ isAdmin }` 授予/撤销管理员（不可撤自己/最后一位；**不可撤超管**） | +| POST | `/api/v1/admin/users/:id/ban` | `{ reason?, cascadeBots? }` 封禁（踢 session；默认级联停用 bot；**不可封超管**） | +| POST | `/api/v1/admin/users/:id/unban` | 解封（不自动启 bot） | +| DELETE | `/api/v1/admin/users/:id?confirm=username` | 删除用户（级联删 bot/session/索引；**不可删超管**） | + +> **超管**：仍为管理员的用户中 `created_at` 最早者（系统首位管理员）。对超管的撤销管理 / 封禁 / 删除一律拒绝（`cannot_revoke_super_admin` / `cannot_ban_super_admin` / `cannot_delete_super_admin`）。 +| GET | `/api/v1/admin/settings/invites` | 邀请策略（配额窗口小时 / 每窗口上限 / TTL / pending） | +| PATCH | `/api/v1/admin/settings/invites` | 更新邀请策略（Redis 覆盖 env 默认） | +| GET | `/api/v1/admin/settings/runtime` | **仅超管** 运行时配置：分组 + 全部项（当前值 / env 默认 / 是否已覆盖 / 是否需重启）+ 警告 | +| PATCH | `/api/v1/admin/settings/runtime` | **仅超管** `{ patch: {key: value}, reset: [key] }` 写 Redis 覆盖 | +| POST | `/api/v1/admin/settings/runtime/reset` | **仅超管** 删除全部覆盖，回到 `.env` | + +> 运行时配置详见 [`docs/runtime-settings.md`](./runtime-settings.md)：`.env` 是默认值，Redis 存覆盖，各节点 5 秒内同步。密钥字段 GET 只返回掩码；PATCH 传空 = 不改，传 `-` = 清空。 +| GET | `/api/v1/admin/bots` | 全部机器人（owner、worker、hasToken、peer 计数；前端分页，后端批量读） | +| GET | `/api/v1/admin/bots/:botId` | 机器人详情 + peers | +| PATCH | `/api/v1/admin/bots/:botId` | 改名 / `{ status: active\|inactive }` 启停 | +| POST | `/api/v1/admin/bots/:botId/stop-worker` | **仅超管** 停止 Worker 轮询 | +| POST | `/api/v1/admin/bots/:botId/start-worker` | **仅超管** 启动/重启 Worker（需已有 Redis token） | +| DELETE | `/api/v1/admin/bots/:botId` | 删除机器人 | +| GET | `/api/v1/admin/bots/:botId/send-targets` | **仅超管** 该 bot 的 peers + `hasContextToken`（广播勾选） | +| POST | `/api/v1/admin/broadcast` | **仅超管** 创建广播任务或 `preview:true` 仅预估人数 | +| GET | `/api/v1/admin/broadcast?limit=` | **仅超管** 最近广播任务列表 | +| GET | `/api/v1/admin/broadcast/:id` | **仅超管** 任务详情与进度 | +| POST | `/api/v1/admin/broadcast/:id/cancel` | **仅超管** 取消 pending/running 任务 | +| GET | `/api/v1/admin/personas?q=&includeDisabled=1` | 人设列表 | +| GET | `/api/v1/admin/personas/:id` | 详情（含 prompt） | +| POST | `/api/v1/admin/personas` | 创建官方人设（可带 tags / isDefault） | +| PUT | `/api/v1/admin/personas/:id` | 更新元信息 / prompt | +| POST | `/api/v1/admin/personas/:id/publish` | 发布新版本 prompt | +| POST | `/api/v1/admin/personas/:id/takedown` | 下架非官方人设 | +| POST | `/api/v1/admin/personas/:id/restore` | 恢复已下架 | +| POST | `/api/v1/admin/personas/:id/set-default` | 设为默认人设 | +| GET | `/api/v1/admin/audit?limit=` | 审计 | +| GET | `/api/v1/admin/stream/recent?limit=&types=&full=` | **仅超管** 活动数据流 backlog（消息 / Worker / LLM；`full=1` 消息预览加长；`types=message,redis,worker,llm`） | +| GET | `/api/v1/admin/stream?types=&full=&heartbeat=` | **仅超管** SSE 实时数据流（`text/event-stream`；Redis 命令为进程内抽样，不写 Redis backlog） | +| GET | `/api/v1/admin/stickers?q=&enabled=` | 表情包列表 | +| GET | `/api/v1/admin/stickers/:id` | 表情详情 | +| GET | `/api/v1/admin/stickers/:id/image` | 预览原图（admin cookie） | +| POST | `/api/v1/admin/stickers` | 上传：`{ slug, displayName, description?, tags?, mime?, dataBase64, enabled? }` | +| PUT | `/api/v1/admin/stickers/:id` | 更新元信息 / 可选换图 | +| DELETE | `/api/v1/admin/stickers/:id` | 删除 meta + 本地文件 | + +页面：`/` 功能介绍 · `/app` 用户中心 · `/admin` 管理后台 · `/og.jpg` 社交分享图 + +### 表情包广场 / 回图 + +- **元数据 + 图片二进制均在 Redis**（`wa:sticker:{id}` / `wa:sticker:{id}:blob`）。 +- Meta 含 **`content_hash`**（blob sha256 前缀）；换图会更新 hash，CDN 用 `?v=` 缓存破坏。 +- **公开已审图**可走无登录 `GET /cdn/s/:id?v=hash`（`public, max-age=31536000, immutable`），供 Cloudflare 边缘缓存。 +- **公开投稿必须管理员审核**（`pending` → `approve` 后进广场与 CDN）；私有仅作者 bot 可用。 +- 上传强制安全扫描：禁 SVG、magic/mime 校验、脚本/PHP/polyglot 尾部检测（非杀软）。 +- 运行时按 **机器人主人** 的表情库 + 自建可用表情注入 LLM；`{"messages":["文字",{"type":"sticker","slug":"..."}]}`。 +- Admin：`GET /admin/stickers?status=pending`、`POST .../approve|reject|takedown|restore`。 +- 环境变量：`STICKER_SEND_ENABLED`、`MAX_STICKERS_PER_REPLY`、`STICKER_MAX_BYTES`（默认 2MB）。 +- Cloudflare 部署：见 `docs/cloudflare.md`。 +### Worker 相关配置（与 API 同进程 / 单镜像） + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `WORKER_ENABLED` | `true` | 是否在本进程跑 iLink 轮询与回复 | +| `MAX_BOTS_PER_WORKER` | `500` | 本进程最多同时 long-poll 的 bot 数 | +| `REPLY_CONCURRENCY` | `16` | 进程内 LLM/发送并发 | +| `INBOX_MAX_LEN` | `20000` | 入站队列深度上限 | + +### 管理后台广播（纯文本推送） + +- **仅纯文本**；不经 LLM、不写短期对话 / 长期记忆。 +- **必须有 `context_token`**（peer 曾给该 bot 发过消息）；无 token 的目标在展开时跳过。 +- **不按批准过滤**：未批准但可触达的 peer 也会进入队列。 +- 异步任务：`POST` 只写 Redis job；Worker 进程内 `BroadcastRunner` 限速发送（`BROADCAST_INTERVAL_MS`，默认 200ms；生产可调到 50–100ms 加速，注意 iLink 限流）。 +- `scope`： + - `all_bots` — 全部机器人下可触达 peer + - `bots` — `botIds[]` 指定机器人 + - `targets` — 显式 `{ botId, peerId }[]` +- `POST` body 带 `preview: true` 时只返回 `{ deliverable, skippedNoToken, missingBots }`，不创建任务。 +- 需 `WORKER_ENABLED=true`（默认）才会实际发送；否则任务保持 `pending`。 +- 环境变量：`BROADCAST_INTERVAL_MS`、`BROADCAST_MAX_TEXT`（默认 2000）、`BROADCAST_HISTORY`（默认 100）。 + +### `workerStats` 字段（dashboard / system） + +**全舰队**租约汇总（`scope: "fleet"`）： + +| 字段 | 说明 | +|------|------| +| `scope` | `fleet`（管理页）或 `local` | +| `leasedLocal` / `leasedFleet` | 全舰队正在 poll 的 bot 数（fleet 模式下两者相同） | +| `maxBots` | 全舰队容量合计（各节点 maxBots 之和） | +| `pollable` | 应被轮询的 bot（active + token + 未 pause） | +| `nodesOnline` / `nodesTotal` | 在线 / 注册部署节点数 | +| `atCapacity` | 任一点触顶或舰队已满 | +| `inboxDepth` 等 | **当前应答节点本机** inbox/任务计数（非全舰队加总） | + +### 超管（super admin） + +`created_at` 最早的 `is_admin` 用户。节点管理 API 与后台「节点」页仅超管可用；普通管理员仍可看 Workers 全舰队 bot 列表。 + +### `nodes` / `GET /admin/nodes`（fleet） + +Redis 心跳注册的**部署进程**（多机时每台一条），**不包含**源站公网 URL（用户统一走主域名；源站 IP 只在 CF Worker `ORIGINS`）。 + +| 字段 | 说明 | +|------|------| +| `id` | `WORKER_ID` | +| `hostname` / `pid` | 主机与进程 | +| `botCount` / `maxBots` / `leasedCount` / `leasedBotIds` | 容量与租约 | +| `label` / `region` / `version` | 可选运维标签；`version` 为进程 appVersion | +| `online` / `isSelf` | 心跳是否新鲜；是否为当前应答节点 | +| `fenced` / `fenceReason` / `fencedAt` / `fencedBy` | 管理员强制下线封锁（进程可仍在跑 HTTP，但不 poll） | +| `weight` / `weightOverride` / `weightUpdatedAt` / `weightBy` | 负载权重（百分比，默认 100）与是否管理员覆盖 | +| `weightLastSeenAt` | 舰队最后一次确认该节点存活的时间（自动过期计时起点） | +| `targetShare` | 按权重应分配的 bot 数（受 `maxBots` 约束）；离线/封锁节点为 `null` | +| `update` | OTA：`outdated` / `desiredVersion` / `status` / `progress` / `error` | +| `startedAt` / `updatedAt` | 启动与心跳时间 | + +响应另含 `release`（通道当前版本摘要）、`appVersion`、`otaEnabled`、`weights`、`weightTotal`（在线节点权重和）、`weightLimits`、`weightTtlSec`（权重自动清理宽限期）、`pollableTotal`、`rebalanceEnabled`、`rebalanceIntervalSec`。 + +**强制下线说明：** 不停止 Docker/OS 进程；目标 `WORKER_ID` 在 fence 清除前不会 re-register / claim。租约由其他节点接管。从 CF Worker `ORIGINS` 移除源站 IP 是流量侧下线，与本 API 独立。 + +### 节点负载权重（Worker 负载调节，仅超管） + +后台「节点」页 → **负载权重**列 → 滑杆 / 预设 / 手填百分比，保存前可预览各节点调整后的目标 bot 数。 + +- **相对值**：在线节点按各自权重占比分摊 `wa:bots:pollable`。A=200%、B=100% → 2:1；两个节点都是 200% 与都是 100% 等价。 +- **范围** 0–500，默认 100。写入 100 等同删除覆盖（`wa:workers:weights` 只保存真正调过的节点）。 +- **0% = 腾空**：节点保持在线心跳、继续处理 HTTP，但不再认领 bot，并会把已持有的租约全部释放（此时忽略 `REBALANCE_SLACK`，能真正归零）。与「强制下线」不同：不写 fence，改回非 0 即刻恢复。 +- **上限仍然生效**：实际数量取 `min(权重份额, MAX_BOTS_PER_WORKER)`。权重只在容量内重新分配，不能突破单进程上限。 +- **单节点例外**：只有一个在线节点时它拿全部（权重是节点之间的比例）。全舰队都设 0% 时退化为均分，避免所有 bot 停止轮询。 +- **生效时机**：写入后 publish `wa:worker:wake`，各节点丢弃权重缓存；认领在下一个 `LEASE_RENEW_SEC` 生效，多余租约按 `REBALANCE_INTERVAL_SEC` / `REBALANCE_MAX_PER_TICK` 逐步释放。`REBALANCE_ENABLED=false` 时已认领的租约不迁移，权重只影响后续认领（后台会提示）。 + +#### 存储与自动清理 + +存储在 Redis，不进 env，节点重启不丢失。两个 hash 分开写，互不干扰： + +| Key | 内容 | 写入方 | +|-----|------|--------| +| `wa:workers:weights` | `workerId` → JSON `{ percent, updatedAt, byUserId, byUsername }` | 仅管理员操作 | +| `wa:workers:weights:seen` | `workerId` → 最后一次确认存活的 ISO 时间 | 仅 GC | + +> 拆成两个 hash 是必需的：GC 若为了盖时间戳而回写整条权重记录，会把管理员同一时刻的改动覆盖掉（read-modify-write 丢失更新）。 + +**节点消失后权重会自动删除**，无需人工干预： + +- 判活以心跳 meta（`wa:worker:`，TTL `WORKER_STALE_SEC`）为准，不看时间戳——所有版本的进程都会写 meta，因此 OTA 混版滚动期间旧版本节点不会被误判为消失。 +- 任一在线节点每 5 分钟扫一次（`HSET`/`HDEL` 幂等，多节点并发无害），给还活着的节点刷新时间戳，给已消失且超过宽限期的删除记录，并顺带清掉没有对应权重的孤儿时间戳。权重 hash 为空时完全不产生 Redis 调用。 +- 宽限期 `WORKER_WEIGHT_TTL_SEC`（默认 3600 秒，下限 60 秒）必须长于一次重启 / OTA 应用，否则每次发版都会重置调节。 +- **已封锁（force-offline）节点永不过期**：封锁是临时的，解除后权重仍在。 +- 「立即清理权重」按钮跳过宽限期，立刻删除所有无心跳且未封锁节点的权重（`POST /admin/nodes/weights/prune`）。 + +> **注意**：未在环境变量固定 `WORKER_ID` 时，进程每次重启都会生成新的随机 ID（`w___`），权重不会跟随到新 ID——这也正是必须自动清理的原因，否则 hash 会随重启次数无限增长。需要权重长期生效，请为每个节点固定 `WORKER_ID`。 + +### OTA 发布与节点更新（仅超管） + +| Method | Path | 说明 | +|--------|------|------| +| GET | `/api/v1/admin/releases/current` | 通道当前 release | +| GET | `/api/v1/admin/releases` | 最近版本列表 | +| POST | `/api/v1/admin/releases` | `mode=blob\|publish\|pack` 上传/注册 | +| POST | `/api/v1/admin/releases/current` | `{ version }` 切换通道版本 | +| POST | `/api/v1/admin/nodes/:workerId/update` | 对该节点下发更新 job | +| POST | `/api/v1/admin/nodes/update-outdated` | 批量更新落后在线节点 | +| GET | `/api/v1/admin/nodes/:workerId/update-status` | 单节点进度 | + +CLI 打包：`pnpm release:pack` / `pnpm docker:build`（见 `docs/docker.md`）。 +发布通道：`/admin` → 部署节点 → **上传通道包**（`files.json`，浏览器超管会话）→ 节点「更新」。 + +另有公开探活：`GET /health`（进程）、`GET /health/ready`（含 Redis，供 LB）。 diff --git a/docs/adr/0001-ilink-direct.md b/docs/adr/0001-ilink-direct.md new file mode 100644 index 0000000..a270066 --- /dev/null +++ b/docs/adr/0001-ilink-direct.md @@ -0,0 +1,32 @@ +# ADR-0001: 直连 iLink，不依赖 OpenClaw Gateway + +## Status + +Accepted (2026-07-19) + +## Context + +产品需要接入微信 ClawBot / OpenClaw 生态下的官方通道，并支持多人角色扮演。 +曾考虑以 OpenClaw Gateway + `@tencent-weixin/openclaw-weixin` 为运行时。 + +## Decision + +**本系统直接调用腾讯 iLink Bot HTTP API**（`https://ilinkai.weixin.qq.com`）： + +- 扫码登录拿 `bot_token` +- `getupdates` 长轮询收消息 +- `sendmessage` 回复（必带 `context_token`） + +会话、人设、记忆、LLM 均在本仓库实现。OpenClaw **不是运行时依赖**。 + +## Consequences + +- 运维更轻：只需本服务 + LLM Key +- 需自行维护 iLink 适配与协议变更 +- 多人隔离用 DB 行级 `(bot_account_id, peer_id[, persona_id])` 即可 +- 协议细节以 PR0 实测与官方插件行为为准；社区文档可能滞后 + +## References + +- OpenClaw WeChat channel docs (plugin path) +- Community iLink protocol notes / `@tencent-weixin/openclaw-weixin` behavior diff --git a/docs/ai-gateway.md b/docs/ai-gateway.md new file mode 100644 index 0000000..21691df --- /dev/null +++ b/docs/ai-gateway.md @@ -0,0 +1,59 @@ +# AI 网关：平台 LLM vs 用户自定义 / 搜索 + +## 原则 + +| 流量 | 出站位置 | +|------|----------| +| **管理员配置的平台 LLM**（`LLM_BASE_URL` / `LLM_API_KEY` / `LLM_MODEL`） | **主站直连** | +| **用户自定义 OpenAI 兼容 API** | **仅** `huggingface/wechat-ai-tools`（HF / Docker） | +| **联网搜索** | **仅** tools 服务 | + +主站 Node 进程**不会** `fetch(用户的 base_url)`，也不会直连搜索引擎。 + +``` +主站 ──平台 LLM──► 管理员配置的上游 + │ + └──TOOLS_BASE_URL──► wechat-ai-tools + ├── /v1/chat/completions + body.upstream ──► 用户 API + └── /v1/web-search ──► DDG / SearXNG / … +``` + +## 主站环境变量 + +```env +# 平台（管理员） +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=sk-... +LLM_MODEL=gpt-4o-mini + +# 工具网关 +TOOLS_BASE_URL=http://127.0.0.1:7860 # 或 https://xxx.hf.space +TOOLS_API_KEY=shared-secret +WEB_SEARCH_ENABLED=true +LLM_PROVIDER_SECRET=... # 加密用户保存的 API Key +``` + +## 打包 tools 镜像 + +```bash +# 在 tools 目录 +docker build -t wechat-ai-tools:latest -f huggingface/wechat-ai-tools/Dockerfile huggingface/wechat-ai-tools + +# 或 compose profile +docker compose --profile tools up -d --build +# 主站 .env 使用：TOOLS_BASE_URL=http://wechat-ai-tools:7860 +``` + +详见 `huggingface/wechat-ai-tools/README.md`。 + +## Chatflow + +编排图中的 `llm` / `search` / `http` 同样遵守上表：用户自定义与搜索只经 tools；`http` 节点默认仅 tools host。见 `docs/chatflow.md`。 + +## 诊断 + +```bash +pnpm diag +``` + +会检查平台 `LLM_API_KEY`，以及（若配置了）`TOOLS_BASE_URL/health`。 diff --git a/docs/chatflow.md b/docs/chatflow.md new file mode 100644 index 0000000..b7c4829 --- /dev/null +++ b/docs/chatflow.md @@ -0,0 +1,134 @@ +# Chatflow + +可视化对话编排（MVP）。人设可在 **prompt** 与 **chatflow** 两种模式间切换。 + +## 入口 + +- 用户中心「我的人设」：运行模式选 **Chatflow 流程** +- 编辑器：`/chatflow?persona=` +- API： + - `GET /api/v1/square/personas/:id/graph` + - `PUT /api/v1/square/personas/:id/graph`（保存图并切换为 chatflow） + +## 节点 + +| 类型 | 说明 | 出站 | +|------|------|------| +| `start` | 唯一入口 | 本地 | +| `llm` | 调用模型 | 平台 LLM 直连，或用户自定义 → **仅** HF tools `/v1/chat/completions` | +| `search` | 联网搜索 | **仅** HF tools `/v1/web-search` | +| `http` | HTTP 请求 | **仅** `TOOLS` 主机 + `CHATFLOW_HTTP_ALLOWLIST`（`*` = 任意公网，见下）| +| `memory` | 注入本轮已选记忆 | 本地 | +| `if-else` | 条件分支（true/false 句柄） | 本地 | +| `answer` | 最终回复模板 | 本地 | + +默认图：`start → llm → answer`。 + +## 模板变量 + +节点文案支持 `{{query}}`、`{{system_prompt}}`、`{{history}}`、`{{memories}}`、`{{bot_name}}`、`{{llm_text}}`、`{{节点id.text}}` 等。 + +## 试聊 + +网页试聊支持 chatflow 人设：执行同一张已发布的图，但 **强制平台上游**（`upstream: null`）， +避免消耗或泄露作者的自定义模型额度。 + +## 限制（MVP） + +- Chatflow 人设 **禁用主动联系**（`skipReason: chatflow_no_proactive`） +- Fork 复制 graph/mode/web_search，**不复制** `llm_provider_id` +- 试聊不注入长期记忆（`memories` 为空）与表情目录 +- LLM 节点的 `temperature` 暂用客户端默认值 +- 完整 Dify（知识库、代码节点等）不做 + +## 环境变量 + +```env +CHATFLOW_HTTP_ALLOWLIST= # 额外允许的 http 节点 host；`*` = 任意公网 +CHATFLOW_MAX_STEPS=32 +CHATFLOW_MAX_NODES=40 +CHATFLOW_HTTP_TIMEOUT_MS=15000 # 单个 http 节点的墙钟上限 +TOOLS_BASE_URL=... # 搜索 / 用户自定义 LLM 必需 +TOOLS_API_KEY=... +WEB_SEARCH_ENABLED=true # 全局搜索开关；人设还需 webSearchEnabled +WEB_SEARCH_MAX_RESULTS=5 # 默认条数；search 节点可用 max_results 覆盖 +``` + +以上都是**默认值**：`/admin` → 设置 → 「Chatflow」「联网搜索与工具网关」可直接改， +覆盖存 Redis，各节点 5 秒内生效，无需重启。详见 `docs/runtime-settings.md`。 + +## http 节点的出站边界 + +图是**人设作者**写的（`PUT .../graph` 仅 owner，`requireUser` 之外无审核）， +响应正文还会回灌进 `vars`（截断 50KB）供 `answer` 引用。也就是说 http 节点等于把一次 +**可读的**服务端请求交到普通注册用户手里，所以出站要按白名单管。 + +两个放大器值得单独记住： + +- URL 是**模板**：`renderTemplate(node.data.url, vars)`，而 `vars.query` 就是原始 + 用户消息（`engine.ts:127-133`）、`vars.llm_text` 是模型输出。作者写成 + `https://{{query}}/` 就等于把 host 的选择权交给**任何发消息的人**；写成引用上游 + `llm` 节点的变量，就等于交给可被 prompt 注入的模型输出。所以校验必须发生在 + **渲染之后**（现实现即如此），作者时校验 URL 是没用的。 +- 触发不需要微信：网页试聊走同一张图，人设可以一直是 private，作者从不出现在广场 + 或任何审核面上。 + +`CHATFLOW_HTTP_ALLOWLIST` 的两种写法： + +| 值 | 含义 | +|----|------| +| 空 | http 节点只能打 `TOOLS_BASE_URL` 的 host:port。没配 TOOLS 就完全不可用 | +| `a.com,b.com` | 精确匹配 hostname，**不支持通配**：`a.com` 不覆盖 `api.a.com` | +| `*` | 任意**公网**地址 | + +`*` 不是「不做检查」。放行前仍然逐项拦掉（`packages/core/src/chatflow/http-guard.ts`）： + +**第一道：地址段**（写死的 IP 字面量） + +- 回环 `127.0.0.0/8`、`::1`、`localhost` +- 私有段 `10/8`、`172.16/12`、`192.168/16` +- 链路本地 `169.254/16` —— 含 AWS/Azure IMDS `169.254.169.254` +- 运营商级 NAT `100.64/10` —— 含阿里云元数据 `100.100.100.200` +- IPv6 `fc00::/7`、**整个 `fe00::/8`**（不只 `fe80::/10`，站点本地 `fec0::/10` 同样在内）、 + `ff00::/8`，以及 `::ffff:`、`64:ff9b::` 里嵌的 IPv4 +- IPv6 兜底：**不在全球单播 `2000::/3` 内的一律拒**，免得再漏某个保留前缀 +- `0/8`、`192.0/16`、`198.18/15`、多播与保留段 + +判断走**地址段**而不是字面量，因为 `new URL()` 会把 `2130706433`、`0x7f.0.0.1`、 +`0177.0.0.1`、`::ffff:127.0.0.1` 归一成别的写法 —— 只比字符串是拦不住的。 + +**第二道：主机名** + +- 单标签主机名（docker service 名、`metadata`、`instance-data`） +- `.local` `.localhost` `.internal` `.svc` `.lan` `.intranet` `.corp` `.private` `.home.arpa` +- 长得像公网、其实是云元数据的名字：`metadata.tencentyun.com`（腾讯云 CVM，直接给 + 角色凭证，无 token 步骤）、`metadata.goog` + +**第三道：解析结果** + +只看字面量挡不住 `169-254-169-254.nip.io` 这种通配 DNS（一个普通 `.io` 域名，解析到 +IMDS 地址，不需要攻击者自建任何东西），也挡不住攻击者把自家 A 记录指向 `10.0.0.5`， +或 docker 内嵌 DNS 把 `service.network` 解析成网桥地址。所以放行前会 `dns.lookup` +一次，**任一**返回地址落在上面的段里就拒。 + +解析失败**不**算拦截：解析不出来的名字 fetch 也出不去，没有任何流量发生，不该把一次 +DNS 抖动升级成 `http_blocked` 把整个流程打断。 + +**第四道：重定向与凭证** + +- **每一跳重定向都重新过一遍全部检查**（`redirect: "manual"`，最多 3 跳）。否则一个允许的 + 公网域名只要回一个 `302 → http://169.254.169.254/`，前面三道全白做。 +- `Authorization` 跨 origin 会被摘掉；`TOOLS_API_KEY` 只发给 `TOOLS_BASE_URL` 的 + **host:port** —— 同一台机器换个端口都不给，`TOOLS_BASE_URL` 指向 loopback 时尤其重要。 + +### 已知边界 + +**DNS rebinding 仍有窗口。** 这里解析一次、`fetch` 自己再解析一次，控制着权威 DNS 且 +TTL 压到极短的攻击者可以在两次之间翻记录。要堵死得把 socket 钉在**已经校验过的那个 +地址**上，即 undici `Agent` 的 `connect.lookup` —— 需要把 undici 收成直接依赖，当前没做。 +现在挡住的是所有直接写内网地址、以及所有**静态**解析到内网的名字。 + +**白名单是全站共享的**：加进去的 host 等于允许所有人的人设去请求它，别放带鉴权的内部接口。 + +联网搜索是**两道闸的与**：全局开关 + 人设自身的「联网」开关（用户中心「我的人设」里勾选）。 +详见 `docs/ai-gateway.md`。 diff --git a/docs/cloudflare.md b/docs/cloudflare.md new file mode 100644 index 0000000..a4db0f0 --- /dev/null +++ b/docs/cloudflare.md @@ -0,0 +1,195 @@ +# Cloudflare Business 缓存适配 + +本项目源站（Fastify Docker）已输出 **Cloudflare 友好** 的缓存头。你只需在 Cloudflare Dashboard 配好 **Cache Rules** 与区域级开关，即可最大化边缘命中。 + +相关代码： + +| 能力 | 位置 | +|------|------| +| HTML / OG 内存缓冲 + ETag | `apps/api/src/static-pages.ts`, `index.ts` | +| 统一 Cache-Control | `apps/api/src/cache-headers.ts` | +| 公开表情 CDN | `GET /cdn/s/:id?v={content_hash}` | +| 默认 API `private, no-store` | `apps/api/src/routes.ts` `onSend` | +| CORS 白名单 | `CORS_ORIGINS` + `PUBLIC_BASE_URL` | + +**本阶段不上 R2**；公开表情仍由源站 Redis blob 提供，靠边缘长缓存降压。 + +--- + +## 1. 源站响应策略（已实现） + +| 路径 | Cache-Control | Cloudflare-CDN-Cache-Control | 鉴权 | +|------|---------------|------------------------------|------| +| `/` `/docs` | `public, max-age=300` | `max-age=3600, swr=86400` | 无 | +| `/app` `/admin` | `public, max-age=60` | `max-age=3600, swr=86400` | 无（壳静态；数据走 API） | +| `/og.jpg` | `public, max-age=86400, immutable` | `max-age=604800` | 无 | +| `/cdn/s/:id?v=` | `public, max-age=31536000, immutable` | 同左 | **无**；仅 public+approved+enabled | +| `/api/v1/auth/config` | `public, max-age=60` | `max-age=300` | 无 | +| `/health` | `no-store` | — | 无 | +| 其余 `/api/v1/**` | `private, no-store` | — | Cookie 会话 | +| 鉴权表情图 | `private, no-store` | — | 登录 / admin | + +HTML 带 `ETag` + `Cache-Tag: html-shell`；公开表情带 `Cache-Tag: sticker-{id}`（便于定向 Purge）。 + +--- + +## 2. DNS / SSL + +1. 域名 A/AAAA/CNAME 橙云代理到源站 +2. SSL/TLS → **Full (strict)** + - 源站用有效证书，或 [Cloudflare Origin CA](https://developers.cloudflare.com/ssl/origin-configuration/origin-ca/) +3. Always Use HTTPS = On +4. 源站 `.env`： + - `PUBLIC_BASE_URL=https://你的域名` + - `COOKIE_SECURE=true` + - `LINUXDO_REDIRECT_URI=https://你的域名/api/v1/auth/callback` + - 可选 `CORS_ORIGINS=https://你的域名`（默认已含 PUBLIC_BASE_URL origin） + +--- + +## 3. Cache Rules（按顺序创建，先匹配先生效） + +路径：**Caching → Cache Rules**（Business 推荐用 Cache Rules，而不是旧 Page Rules）。 + +### Rule 1 — Bypass 动态 API + +- **Name:** `bypass-api-private` +- **When:** + `(starts_with(http.request.uri.path, "/api/v1/") and not http.request.uri.path eq "/api/v1/auth/config")` + 或方法为 `POST` / `PUT` / `PATCH` / `DELETE` +- **Then:** Cache eligibility = **Bypass cache** + +### Rule 2 — HTML 壳 Cache Everything + 忽略 Cookie + +- **Name:** `cache-html-shells` +- **When:** + `http.request.uri.path in {"/" "/app" "/docs" "/admin"}` +- **Then:** + - Eligible for cache + - Edge TTL: **Respect origin**（或 Override 1 hour） + - Browser TTL: Respect origin + - **Cache key → Ignore query string**（可选） + - **Cookie handling → Ignore presence of cookies**（**关键**：否则带 `wa_session` 永远 MISS） + +### Rule 3 — OG 图 + +- **Name:** `cache-og` +- **When:** `http.request.uri.path eq "/og.jpg"` +- **Then:** Eligible for cache；Edge TTL Respect origin 或 7d；**Ignore cookies** + +### Rule 4 — 公开表情 CDN + +- **Name:** `cache-cdn-stickers` +- **When:** `starts_with(http.request.uri.path, "/cdn/s/")` +- **Then:** Eligible for cache；Edge TTL Respect origin；**Ignore cookies** +- Cache key **保留 query string**（`v=` 内容哈希） + +### Rule 5 — auth/config 短缓存 + +- **Name:** `cache-auth-config` +- **When:** `http.request.uri.path eq "/api/v1/auth/config"` +- **Then:** Eligible for cache；Edge TTL 5 minutes 或 Respect origin；Ignore cookies + +### 默认 + +未匹配规则时：尊重源站 `Cache-Control`；无 `public` 的不进共享缓存。 + +--- + +## 4. 区域级推荐（Business） + +| 设置 | 建议 | +|------|------| +| HTTP/3 (QUIC) | On | +| Brotli | On | +| Tiered Cache | On | +| Early Hints | 可选（当前 HTML 内联，收益有限） | +| Auto Minify | **Off**（避免改 HTML 导致 ETag 与部署不一致） | +| Rocket Loader | **Off** | +| Polish | 可选；若开 WebP，注意 `Accept` 变体，或只对 `/cdn/s/*` 试 | +| Mirage | 可选（移动列表） | +| Cache Reserve | 可选（长尾表情） | +| Argo Smart Routing | 源站距用户远时可选 | +| WAF Managed Rules | On | +| Rate limiting | 建议：`/api/v1/auth/*`、try-chat、上传 POST | +| Bot Fight | 按需；勿误伤 OAuth 回调 | + +--- + +## 5. 部署后刷新缓存 + +| 变更 | 做法 | +|------|------| +| 新版本 HTML（app/admin/docs/landing） | Purge by URL：`/` `/app` `/docs` `/admin`，或 Purge by Tag：`html-shell` | +| 换图表情 | **无需 purge**：`?v={content_hash}` 自动新键 | +| 下架 / 拒绝公开表情 | 源站 404；边缘可能仍 HIT 至 TTL → Purge URL 或 Tag `sticker-{id}` | +| 紧急全站 | Purge Everything（会短暂升高源站压力） | + +API Purge 示例（可选）： + +```bash +# 需要 Zone ID + API Token (Cache Purge 权限) +curl -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/purge_cache" \ + -H "Authorization: Bearer $CF_API_TOKEN" \ + -H "Content-Type: application/json" \ + --data '{"files":["https://你的域名/","https://你的域名/app","https://你的域名/docs","https://你的域名/admin"]}' +``` + +--- + +## 6. 验收 curl + +```bash +HOST=https://你的域名 + +# HTML：第二次应 CF-Cache-Status: HIT（即使带假 cookie） +curl -sI "$HOST/" | grep -iE 'cf-cache-status|cache-control|etag|content-encoding' +curl -sI "$HOST/app" -H "Cookie: wa_session=fake" | grep -i cf-cache-status + +# 公开表情 +curl -sI "$HOST/cdn/s/$STICKER_ID?v=$HASH" | grep -iE 'cf-cache-status|cache-control' +curl -sI "$HOST/cdn/s/$STICKER_ID?v=$HASH" | grep -i cf-cache-status # HIT + +# 私有 API +curl -sI "$HOST/api/v1/auth/me" -H "Cookie: wa_session=..." | grep -iE 'cache-control|cf-cache-status' +# 期望: private, no-store；DYNAMIC 或 BYPASS + +# 压缩 +curl -sI -H 'Accept-Encoding: br' "$HOST/app" | grep -i content-encoding + +# 私有 / 未审表情不得公开 +curl -sI "$HOST/cdn/s/$PRIVATE_ID" # 404 +``` + +Dashboard → Caching → Cache Analytics 可看 HIT 率。 + +--- + +## 7. 反代注意 + +- 优先 **CF 橙云 → 源站 443/8787**，中间少一层 +- **多节点负载均衡**：使用仓库内 **`cloudflare-worker/`**（静态 `ORIGINS` + 健康检查 + 轮询），主域名绑 Worker；详见 `cloudflare-worker/README.md` +- 若 Nginx 仅 TLS 终结再反代 Node： + - 转发 `Host`、`X-Forwarded-Proto`、`CF-Connecting-IP` + - **不要**强行剥掉 `Accept-Encoding` 导致双重压缩混乱 +- 源站健康检查：`GET /health`（进程）或 `GET /health/ready`（含 Redis，推荐给 LB） +- 应用侧 `PUBLIC_BASE_URL` 始终为主域名；源站 IP 不必暴露在后台 UI +--- + +## 8. 安全边界 + +- `/cdn/s/*` **仅** `visibility=public` 且 `review_status=approved` 且 `enabled` +- 私有 / pending / rejected → 统一 404 +- 用户 JSON、admin、try-chat、OAuth callback **永不**边缘共享缓存 +- 广场列表含 `inLibrary` 个性化，**不**做公共 CDN 缓存（升级路径：拆无个性化的 public catalog API） + +--- + +## 9. 可选升级路径（未实现） + +1. 公开表情同步 **R2** + 自定义域，源站只写 meta +2. 拆 `GET /api/v1/public/square/*` 无 `inLibrary` 的目录接口短缓存 +3. 内联 CSS/JS 拆出带 content-hash 的静态文件 +4. Cache Reserve + Image Resizing + +详见主 README / 本文件与 `docs/docker.md` 交叉引用。 diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..43386b3 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,244 @@ +# Docker 部署 + +## 前置条件 + +| 依赖 | 说明 | +|------|------| +| Docker + Compose | 本机或服务器已安装 | +| Upstash Redis | `.env` 中 `REDIS_URL=rediss://...` | +| LINUX DO OAuth | Client ID/Secret + **公网回调地址** | +| 平台 LLM（主站直连） | `LLM_API_KEY` / `LLM_BASE_URL` / `LLM_MODEL` | +| 工具网关（用户自定义 API + 搜索） | `TOOLS_BASE_URL` / `TOOLS_API_KEY`；镜像见 `huggingface/wechat-ai-tools` | + + +## 快速启动 + +```bash +cd /path/to/WeChat-AI + +# 配置环境变量（勿提交 .env） +# 生产务必修改： +# PUBLIC_BASE_URL=https://你的域名 +# LINUXDO_REDIRECT_URI=https://你的域名/api/v1/auth/callback +# REDIS_URL / LLM_* / LINUXDO_CLIENT_* +# 用户自定义 API + 搜索：TOOLS_BASE_URL / TOOLS_API_KEY +# docker compose --profile tools up -d --build +# TOOLS_BASE_URL=http://wechat-ai-tools:7860 + +# 推荐：升版 + 打 OTA 包 + 构建（无需 Cookie） +pnpm docker:up +# 自定义镜像名： +pnpm docker:build -- -- docker build -t e51l6pwpe/wxai:latest . + +# 包在 dist/release/<版本>/files.json +# 浏览器登录 /admin → 部署节点 →「上传通道包」→ 再点节点「更新」 + +docker compose logs -f wechat-ai +``` + +### 单独构建 tools 镜像 + +```bash +docker build -t wechat-ai-tools:latest -f huggingface/wechat-ai-tools/Dockerfile huggingface/wechat-ai-tools +docker run --rm -p 7860:7860 -e TOOLS_API_KEY=secret -e ALLOW_REQUEST_UPSTREAM=true wechat-ai-tools:latest +``` + +详见 `docs/ai-gateway.md`、`huggingface/wechat-ai-tools/README.md`。 + + +> **版本与通道：** `pnpm docker:build` 默认：升版 → **本地 pack** → Docker。 +> 通道发布只走网页：`/admin` → 上传 `files.json`（无 CLI Cookie）。 +> 直接跑 `docker build` **不会**升版、也**不会**打通道包。 + +| 地址 | 说明 | +|------|------| +| `/` | 功能介绍落地页（OG 分享图 `/og.jpg`） | +| `/app` | 用户中心（LINUX DO 登录、加机器人） | +| `/admin` | 管理后台（仪表盘 / Token） | +| `/health` | 健康检查 | + +## 仅用 Dockerfile + +```bash +# 升版 + docker build -t wechat-ai . +pnpm docker:build -- --raw +# 或自定义：node scripts/docker-build.mjs -- docker build -t wechat-ai:0.2.1 . + +docker run -d --name wechat-ai --restart unless-stopped \ + --env-file .env \ + -e WECHAT_AI_HOST=0.0.0.0 \ + -e WECHAT_AI_PORT=8787 \ + -p 8787:8787 \ + wechat-ai +``` + +Bot token 与表情包均存 **Redis**（与 `REDIS_URL` 同库），容器重建不丢；无需本地数据卷。 + +## 生产环境检查清单 + +1. **LINUX DO** 应用回调与 `.env` 完全一致： + `https://你的域名/api/v1/auth/callback` +2. **`PUBLIC_BASE_URL`** = `https://你的域名`（无尾斜杠） +3. **HTTPS** 时设置 `COOKIE_SECURE=true` +4. **Redis** 使用 Upstash `rediss://`，服务器能访问外网 +5. Bot **token 已写入 Redis**（与 `REDIS_URL` 同库），重建容器不会丢登录 + +## 常用命令 + +```bash +docker compose ps +docker compose logs -f +docker compose restart +docker compose down # 停服务（Bot token 在 Redis，不受影响） +docker compose down -v # 同 down（本服务无持久化 volume） +pnpm docker:up # 升版+本地 pack+compose up --build +pnpm docker:build -- -- docker build -t e51l6pwpe/wxai:latest . +# 然后 /admin → 上传通道包 (files.json) → 更新节点 +pnpm docker:build -- --no-channel # 只升版构建，不 pack +``` + +## OTA 增量更新（多节点日常热修） + +业务源码小改可不必每台 `docker build`：本地 pack → 管理后台上传通道包 → 对落后节点点「更新」。 + +```bash +# 构建顺带打通道包 +pnpm docker:build -- -- docker build -t e51l6pwpe/wxai:latest . +# 或仅 pack： +pnpm release:pack + +# 浏览器 /admin → 部署节点 →「上传通道包」选 dist/release//files.json +# →「更新全部落后」 +``` + +| 项 | 说明 | +|----|------| +| 差量 | 按文件 sha256 比对，只下发变更文件 | +| 重启 | 节点 `process.exit(0)`，依赖 `restart: unless-stopped` 拉起**同一容器**（可写层保留补丁） | +| 版本 | 心跳 `version`：`.wa-version`（OTA 写入）→ `APP_VERSION` → 根 `package.json` | +| 仍需镜像 | Node 基础镜像、系统包、Dockerfile、`OTA_ALLOW_INSTALL=false` 时的依赖大变 | + +环境变量：`OTA_ENABLED`（默认 true）、`OTA_ALLOW_INSTALL`、`APP_VERSION`（无 OTA 戳时可选）、`OTA_STAGING_DIR`。 +**注意：** OTA 只改文件、不改环境变量；版本靠 `/app/.wa-version` 上报，无需、也不应靠 `APP_VERSION` 跟版。 +`docker compose up --build` / 重建容器会丢掉仅靠 OTA 写入的补丁；长期仍以镜像为 source of truth。 + +## 反代示例 + +生产推荐把域名挂在 **Cloudflare**（橙云代理 + Cache Rules），见 **`docs/cloudflare.md`**（Business 缓存规则、忽略 Cookie、Purge 清单）。 + +### Caddy（无 CF 时） + +```caddy +your.domain.com { + reverse_proxy 127.0.0.1:8787 +} +``` + +### Nginx（无 CF 时，或 CF → Nginx → Node） + +```nginx +server { + listen 443 ssl; + server_name your.domain.com; + # ssl_certificate ...; + + location / { + proxy_pass http://127.0.0.1:8787; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # Cloudflare 还原真实 IP 时可用： + # proxy_set_header CF-Connecting-IP $http_cf_connecting_ip; + } +} +``` + +源站已输出 `Cache-Control` / `Cloudflare-CDN-Cache-Control`、HTML ETag、公开表情 `/cdn/s/:id?v=`。反代层**不必**再写 `proxy_cache`，除非你不用 Cloudflare。 +## 镜像说明 + +- 基础镜像：`node:22-bookworm-slim` +- 包管理：pnpm monorepo +- 启动：`pnpm db:seed`（幂等）→ `pnpm --filter @wechat-ai/api start`（**API + Worker 同进程**） +- 非 root 用户 `appuser` 运行 +- 健康检查：`GET /health` + +## Worker 规模（单镜像） + +默认仍是 **一个容器跑全部**：HTTP + iLink 长轮询 + AI 回复。 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `MAX_BOTS_PER_WORKER` | `500` | 本进程最多同时 long-poll 的 bot 数 | +| `REPLY_CONCURRENCY` | `16` | 同时进行的 LLM/发送任务数 | +| `LEASE_TTL_SEC` | `45` | 租约 TTL（同镜像多副本时防重复 poll） | + +机器人很多时优先调高 `MAX_BOTS_PER_WORKER` 与系统 `ulimit -n`（注意内存与出站连接数）。 +默认 **单副本一体部署** 即可；同镜像多副本已支持（Redis 租约分片 poll）。 + +## 多节点同构部署（10+ 台） + +每台服务器跑**同一镜像**（API + Worker），共用一个 Upstash Redis；用户只访问**主域名**。 + +### 应用 env（全站一致） + +```env +PUBLIC_BASE_URL=https://你的主域名 +LINUXDO_REDIRECT_URI=https://你的主域名/api/v1/auth/callback +REDIS_URL=rediss://... +COOKIE_SECURE=true +WORKER_ENABLED=true +``` + +### 每节点不同 + +```env +WORKER_ID=node-01 # 必填且唯一 +NODE_LABEL=cn-east-1a # 可选，管理后台展示 +NODE_REGION=cn-east # 可选 +``` + +**不要**给每台设不同的 `PUBLIC_BASE_URL`。源站直连地址（IP:8787）只写在 Cloudflare Worker 的 `ORIGINS`，见 `cloudflare-worker/README.md`。 + +### 部署步骤摘要 + +1. 各机：`docker run ... --env-file .env -e WORKER_ID=node-0N -p 8787:8787 wechat-ai` +2. 配置并部署 `cloudflare-worker`，`ORIGINS=http://ip1:8787,http://ip2:8787,...` +3. 主域名绑到 Worker +4. 打开 `/admin` → **节点**：应看到各 `WORKER_ID` 心跳与租约 bot 数 + +| 探活 | 路径 | +|------|------| +| Docker / 轻量 | `GET /health` | +| LB 就绪（含 Redis） | `GET /health/ready` | + +管理 API：`GET /api/v1/admin/nodes`（Cookie 管理员）。 + +扫码加机器人会话状态在 Redis，HTTP 无需粘性会话。 + +### 租约自动再平衡（rebalance） + +多节点默认 **开启**：租约偏多的进程会周期性 **主动释放** 多余 lease（不 pause bot），空闲节点下一轮 `claim` 捡走，使各节点 bot 数接近均分。 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `REBALANCE_ENABLED` | `true` | 设为 `false` 关闭（租约粘在首占节点） | +| `REBALANCE_INTERVAL_SEC` | `60` | 同一进程两次 shed 最小间隔 | +| `REBALANCE_SLACK` | `2` | 允许高出均分多少个再释放 | +| `REBALANCE_MAX_PER_TICK` | `50` | 每次最多释放数（避免瞬间空窗过大） | + +日志关键字：`[worker] rebalance shed N bot(s)`。约 `ceil(超额 / 50)` 分钟内收敛。 + +### 强制下线节点 + +管理后台 **节点** 页 → **强制下线**： + +1. 写入 Redis fence（`wa:worker:{id}:fence`） +2. 释放该 WORKER_ID 下全部 bot 租约 +3. 从 `wa:workers:reg` 移除 + +目标进程下一轮 reconcile 发现 fence 后停止认领；其他节点 claim 这些 bot。 +**解除下线** 后该节点可重新加入。 + +这**不会** `docker stop`；若要从 LB 摘流量，还要从 Cloudflare Worker `ORIGINS` 去掉该源站。 \ No newline at end of file diff --git a/docs/e2e-checklist.md b/docs/e2e-checklist.md new file mode 100644 index 0000000..e92fc41 --- /dev/null +++ b/docs/e2e-checklist.md @@ -0,0 +1,113 @@ +# E2E 验收清单 + +环境：已配置 `.env`、完成 `db:migrate` + `db:seed`、至少一个 Bot 登录、LLM 可用。 + +## A. 安装与诊断 + +- [ ] `pnpm install` 成功 +- [ ] `pnpm db:migrate` / `pnpm db:seed` 成功 +- [ ] `pnpm diag` 在配置正确时 PASS（弱 token / 无 bot 仅警告） +- [ ] `pnpm test` 全部通过 + +## B. Admin + +- [ ] 打开 `http://127.0.0.1:8787/admin` 可加载 +- [ ] 填入正确 Token 后总览卡片有数据 +- [ ] 可见默认人设 `catgirl` / `girlfriend` +- [ ] 可创建新人设并出现在列表 +- [ ] 可发布人设新版本 + +## C. Bot 与多人 + +- [ ] `pnpm ilink:login` 扫码成功，Admin → Bot 可见账号 +- [ ] 微信用户 A 发消息后，Admin → 用户出现 pending +- [ ] 批准 A 后，A 收到角色扮演回复（默认猫娘风格） +- [ ] 用户 B 同时发消息：会话与记忆不与 A 串味 +- [ ] Admin 将 A 分配为女友人设，A 回复风格变化 +- [ ] A 发送 `/角色 xxx` 收到「仅后台分配」提示 +- [ ] 未批准用户收到开通引导（或被忽略） + +## D. 记忆 + +- [ ] A 陈述偏好（如「我叫小明」）后，多轮后记忆列表出现相关事实（每 N 轮抽取） +- [ ] 重启 `pnpm dev` 后 A 的记忆仍在 +- [ ] Admin 重置 A 记忆后列表清空 + +## E. 多 Bot（可选） + +- [ ] 第二次 `pnpm ilink:login -- --name bot2` 增加第二个 Bot +- [ ] 两个 Bot 的 peer 互不混淆 + +## F. 限流与体验 + +- [ ] 同一用户短时间连发超限，收到「发得太快」提示 +- [ ] typing 调用失败不影响正常回复（best-effort） + +### F.1 输入状态（getconfig + status 1/2） + +- [ ] 发消息后微信显示「对方正在输入中」 +- [ ] 回复送达后指示器**消失**（不是等服务端超时） +- [ ] 多气泡回复期间指示器在气泡之间持续显示 +- [ ] 被限流 / 未批准被拒 / 用户互聊中继后，指示器也停止 +- [ ] 抓包确认每个 peer 只有一次 `getconfig`（票据缓存生效），之后只有 `sendtyping` +- [ ] `sendtyping` 携带 `typing_ticket` 与 `status` +- [ ] 人为让 `getconfig` 失败（改 baseUrl）→ 回复仍正常送达 + +### F.2 入站媒体 + +- [ ] `VISION_ENABLED=false` 时发图 → 回「看不了图片」，且**没有**产生 LLM 调用 +- [ ] `VISION_ENABLED=true` 且模型支持视觉时发图 → 回复据实描述图中内容 +- [ ] 发图 + 文字说明 → 两者一起进入同一轮对话 +- [ ] 一条消息发 3+ 张图 → 只识别 `VISION_MAX_IMAGES` 张，其余按「看不了」告知 +- [ ] 超过 `INBOUND_MEDIA_MAX_BYTES` 的图 → 降级为「看不了」，回复不失败 +- [ ] 发视频 / 文件 → 按类型回话，且模型**没有**编造内容 +- [ ] 发带转写的微信语音 → 按文字正常对话，不出现「我听不到」 +- [ ] `VOICE_TRANSCRIPT_ENABLED=false` 后同一条语音 → 回「没听清，麻烦打字」，且不走 LLM +- [ ] 该开关与 `VISION_ENABLED` 互不影响（图片关、语音开是默认组合） +- [ ] 发图后下一轮追问 → 历史里是 `[图片]` 占位符，上下文没丢 +- [ ] chatflow 模式人设收到图 → 看到 `[图片]` 占位符，不报错 + +## G. 安全 + +- [ ] 无 Token 访问 `/api/v1/*` 返回 401 +- [ ] Token 不出现在审计全文中；仅存 Redis `wa:bot:{id}:creds` + +## H. 用户对话（@LINUX DO） + +- [ ] 用户中心可生成绑定码；微信 `/绑定 CODE` 成功；`/我的身份` 显示用户名 +- [ ] 双方均绑定且各聊过机器人后，A 发 `@B用户名`，B 收到请求 +- [ ] B `/同意` 后互发文字，内容带 `[用户名]` 前缀 +- [ ] `/断开` 后普通消息恢复 AI 角色扮演 +- [ ] 未绑定 / 无 context_token 的目标：发起方收到明确错误提示 +- [ ] `hello @user` 不触发请求（整条消息匹配） + +## I. AI 网关（自定义模型 / 联网搜索） + +前置：已部署 `huggingface/wechat-ai-tools`，主站配置 `TOOLS_BASE_URL` / `TOOLS_API_KEY` / `LLM_PROVIDER_SECRET`。 + +- [ ] `/app` → **我的模型**：可添加连接（名称 / Base URL / Key / 模型），列表显示掩码 Key +- [ ] 可停用 / 启用 / 删除连接 +- [ ] 未配置 `LLM_PROVIDER_SECRET` 时页面提示明确、不落库 +- [ ] 人设选择「我的连接」后对话正常；抓包确认主站 **未**直连该 base_url（仅打 TOOLS） +- [ ] 人设开启联网搜索 + `WEB_SEARCH_ENABLED=true`：问时事类问题触发 `/v1/web-search` +- [ ] 审计日志 / 应用日志中 **不含** upstream api_key 明文 +- [ ] Fork 他人人设：新人设 `llmProviderId` 为空（不继承作者密钥） + +## J. Chatflow + +- [ ] 人设编辑器切「Chatflow 流程」保存后，卡片显示 Chatflow 徽章 +- [ ] `/chatflow?persona=` 可加载（未保存图时显示默认 start→llm→answer） +- [ ] 拖拽节点、连线、编辑属性后「保存并启用」成功 +- [ ] 非本人人设打开为只读（保存按钮禁用） +- [ ] 微信对该人设发消息，回复由流程产出 +- [ ] 网页试聊 chatflow 人设可正常回复（走平台模型，不消耗作者额度） +- [ ] `search` 节点在人设未开搜索 / 未配 TOOLS 时报错清晰 +- [ ] `http` 节点填非 allowlist 域名 → 被拒（`http_blocked`） +- [ ] chatflow 人设不触发主动联系（skip: `chatflow_no_proactive`） +- [ ] 图校验：无 answer / 多个 start / 超 `CHATFLOW_MAX_NODES` → 400 + +## 验收结论 + +全部关键项（A–D、F–G）通过即可判定：**系统已完成并可验收**。 +E 为多 Bot 增强项；H 为用户对话增强项；真机扫码依赖本机微信与 iLink 可用性。 +I / J 需先部署 HF 工具服务（`docs/ai-gateway.md`、`docs/chatflow.md`）；未部署时这两节整体跳过。 diff --git a/docs/oauth-linuxdo.md b/docs/oauth-linuxdo.md new file mode 100644 index 0000000..ac73ff6 --- /dev/null +++ b/docs/oauth-linuxdo.md @@ -0,0 +1,55 @@ +# LINUX DO OAuth 接入 + +## 1. 申请应用 + +1. 打开 [LINUX DO Connect](https://connect.linux.do/)（或社区应用管理入口） +2. 创建 OAuth 应用，回调地址填： + +```text +http://你的域名/api/v1/auth/callback +``` + +本地开发示例： + +```text +http://127.0.0.1:8787/api/v1/auth/callback +``` + +3. 拿到 `client_id` / `client_secret` + +## 2. 配置 `.env` + +```env +LINUXDO_CLIENT_ID=... +LINUXDO_CLIENT_SECRET=... +LINUXDO_REDIRECT_URI=http://127.0.0.1:8787/api/v1/auth/callback +LINUXDO_ADMIN_IDS=你的LINUXDO数字ID,你的用户名 +REDIS_URL=redis://:密码@远端:6379/0 +PUBLIC_BASE_URL=http://127.0.0.1:8787 +``` + +`LINUXDO_ADMIN_IDS`：匹配 OAuth 返回的 **用户 id** 或 **username** 即视为管理员。 + +## 3. 流程 + +1. 用户访问 `/app` → 可用 **用户名密码** 登录，或点「LINUX DO 登录」 +2. LINUX DO 跳转 `connect.linux.do` 授权（scope：`openid profile`） +3. 回调 `/api/v1/auth/callback` 写 Redis session cookie + - **OAuth 新用户不需要邀请码**（与本地注册不同） +4. 本地注册：需好友分享的一次性邀请码/链接（`/app?invite=CODE`），成功后同样写 session cookie +5. 普通用户：管理自己的机器人（添加/删除、批准 peer、分配人设）、生成邀请 +6. 管理员：`/admin` 仪表盘；可配置「每 X 小时可生成 N 个邀请」、封禁/删除用户、停用/删除机器人 + +未配置 `LINUXDO_ADMIN_IDS` 时，**第一个成功注册/登录的用户**自动成为管理员（`FIRST_USER_IS_ADMIN=true`）。 + +封禁用户后 OAuth 仍可在 LINUX DO 授权，但 callback / 密码登录 / 会话校验会返回 `user_banned`。 + +## 4. 协议端点（默认 / OIDC Discovery） + +| 用途 | URL | +|------|-----| +| Discovery | `https://connect.linux.do/.well-known/openid-configuration` | +| 授权 | `https://connect.linux.do/oauth2/authorize` | +| Token | `https://connect.linux.do/oauth2/token` | +| 用户信息 | `https://connect.linux.do/api/user` | +| 支持 scope | `openid` `profile` `email` | diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..1c2eaca --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,289 @@ +# WeChat-AI 运维手册 + +## 1. 首次部署清单 + +### 1.1 依赖 + +- Node.js 20+ +- pnpm +- **Upstash Redis**（`rediss://...`） +- **LINUX DO OAuth** 应用 +- LLM API Key（OpenAI 兼容） + +### 1.2 `.env` 必填 + +```env +REDIS_URL=rediss://default:密码@xxxx.upstash.io:6379 +LLM_API_KEY=... +LLM_BASE_URL=... +LLM_MODEL=... + +LINUXDO_CLIENT_ID=... +LINUXDO_CLIENT_SECRET=... +LINUXDO_REDIRECT_URI=http://127.0.0.1:8787/api/v1/auth/callback +# 可选：数字 ID 或用户名；留空时「第一个登录的用户」自动成为管理员 +LINUXDO_ADMIN_IDS= + +PUBLIC_BASE_URL=http://127.0.0.1:8787 +``` + +LINUX DO 应用回调地址必须与 `LINUXDO_REDIRECT_URI` **完全一致**。 + +### 1.4 多节点运维要点 + +| 项 | 说明 | +|----|------| +| 共享 | 所有节点同一 `REDIS_URL`、`PUBLIC_BASE_URL`（主域名）、OAuth 回调 | +| 每机 | 唯一 `WORKER_ID`；可选 `NODE_LABEL` / `NODE_REGION` | +| 入口 | 主域名 → `cloudflare-worker` 的 `ORIGINS`（源站 IP:端口） | +| 后台 | `/admin` → **节点**：进程心跳与 bot 租约；**不显示**源站 URL | +| 扫码 | 登录会话在 Redis，无需粘性会话 | +| 探活 | LB 用 `/health/ready`；Docker 可用 `/health` | +| 下线 | 从 Worker `ORIGINS` 移除并 deploy；停容器后租约 TTL 过期自动转移 | +| 扩容 | 新机起容器 + 更新 `ORIGINS`；bot 由租约自动分片 | +| 日常代码热修 | `pnpm release:pack` 或 `pnpm docker:build` → `/admin` 上传通道包 → 节点「更新」 | +| 基础镜像变更 | 仍需各机 `docker build` / 拉新镜像 | + +进程内 HTTP 限流为单机计数；生产建议在 Cloudflare 对 `/api/v1/auth/*` 做 Rate Limiting。 + +### 1.3 启动 + +```powershell +cd F:\Code-Other-4\WeChat-AI +pnpm install +pnpm diag +pnpm db:seed +pnpm dev +``` + +| 页面 | URL | +|------|-----| +| 用户中心 | http://127.0.0.1:8787/app | +| 管理后台 | http://127.0.0.1:8787/admin | + +## 2. 用户操作流程 + +1. 打开 `/app` → **LINUX DO 登录** +2. **扫码添加微信机器人**（ClawBot） +3. 微信好友私聊机器人 → 在用户中心 **批准** +4. 可选：分配人设（猫娘 / 女友） +5. 正常聊天（AI 会分多条气泡回复） + +### 2.0 输入状态（「对方正在输入中」） + +两步协议，`packages/ilink` 内部完成，无需配置： + +1. `POST /ilink/bot/getconfig { ilink_user_id, context_token }` → `typing_ticket` +2. `POST /ilink/bot/sendtyping { ilink_user_id, typing_ticket, status }` — `status: 1` 开始，`status: 2` 停止 + +票据**按用户缓存**（服务端有效期约 24h，本地按 20h 过期后自动重取），所以每个 peer 大约一天一次 `getconfig`；并发的多次输入调用会合并成一次取票。 + +指示器的生命周期：收到消息立刻开始 → 调模型前再次开始 → 多气泡之间每条前再次开始 → **`handleJob` 的 `finally` 统一停止**。回复、拒绝、限流、用户互聊中继、异常，任何出口都会停止，不会把「正在输入中」留在对方屏幕上。主动联系发完也会停止。 + +排查：`getconfig` 失败时会退化成不带票据的 `sendtyping`（指示器可能不显示，但**绝不影响回复**）；票据被服务端提前失效时会强制重取并重试一次。 + +### 2.0.1 图片理解（入站 Vision） + +默认**关闭**，且由 `apps/api/src/shipped-defaults.test.ts` 守着——`VISION_ENABLED` 只认精确的 `"true"`，`"1"` / `"yes"` / `"TRUE"` 都不算开。关闭状态下：不去 CDN 取字节、不调任何模型，收到图片直接回一句按类型区分的话。 + +> **出站语音 / 视频 / 文件**：`sendVoice` / `sendVideo` / `sendFile` 已在 `packages/ilink` 实现并有测试，但**没有任何调用点**，回复路径不会用到。要真正让角色发语音，还差两段：TTS 产出音频、再转成微信用的 SILK 编码（TTS 给的是 mp3/wav/opus，直接发大概率放不出来）。另外出站 voice 的 item type 只在入站验证过，需真机确认。 + + +**关键点：人设模型不需要支持视觉。** 默认的 `caption` 模式先让一个识图端点把图片转成文字描述，只把这段**文字**交给人设模型——所以 `deepseek-v4-flash` 这类纯文本模型照样能"看图"。描述还会写进对话历史，隔几轮再问"刚那张图里的猫呢"仍然接得上。 + +| 模式 | 人设模型要求 | 说明 | +|------|--------------|------| +| `caption`（默认） | 无 | 识图端点出描述 → 文字进人设模型 | +| `direct` | **必须支持视觉** | 图片原样交给人设模型；不支持就直接报错 | + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `VISION_ENABLED` | `false` | 总闸。关闭时不下载、不调模型 | +| `VISION_MODE` | `caption` | 见上表 | +| `VISION_BASE_URL` / `VISION_API_KEY` | 空 | 识图端点。**env-only**（与平台 LLM 同信任级，直连不走 tools）。留空则复用 `LLM_BASE_URL` / `LLM_API_KEY` | +| `VISION_MODEL` | 空 | **必填**，留空则所有图片按「看不了」处理。对「我的模型连接」无效（那条链路只认连接里的模型名） | +| `VISION_CAPTION_MAX_TOKENS` | `300` | 描述长度上限 | +| `VISION_MAX_IMAGES` | `2` | 单条消息最多识别几张（每张都实打实花 token） | +| `INBOUND_MEDIA_MAX_BYTES` | `4194304` | 单个附件下载上限（解密后原始大小；base64 再涨约 1/3） | + +`caption` 模式的成本：每张图多一次识图调用（记在机器人主人账上），人设模型那一轮只多几十个 token 的描述文字——比把 4MB base64 塞进上下文便宜得多。识图失败会降级成「看不了」，**不会让回复失败**。 + +行为说明： + +- **只有图片会被下载**。语音/视频/文件的字节拿来也喂不进模型，所以根本不去 CDN 取——微信语音是 SILK/AMR，没有 OpenAI 兼容端点收。 +- `caption` 模式下人设模型**收不到字节**，只收到方括号里的描述；提示词会要求它「当作亲眼所见，但只依据描述内容，不要往外扩写」。 +- **语音靠微信自带转写**（`VOICE_TRANSCRIPT_ENABLED`，**默认开**）：转写文字随入站消息一起到，`extractText` 把它并入文本，这类语音当文字处理，不再额外列为附件（否则会告诉模型「你听不到」它正要读的内容）。这个开关与 `VISION_ENABLED` **互不相干**——用转写不花钱、不需要任何模型。设 `false` 后语音一律回「没听清，麻烦打字」。 +- 模型看不到的附件仍会写进系统提示，并明确要求**不许猜测内容**——否则人设会张口就编视频里有什么。 +- 整条消息只有看不了的媒体且没有文字时，直接回一句按类型区分的话（不调模型，不花钱）。 +- 附件下载失败不会拖垮回复，降级成「只告知存在」。 +- Chatflow 模式的人设看到的是 `[图片]` 占位符——图执行器没有多模态节点。 +- 历史里存的也是 `[图片]` 占位符（字节不落库），所以下一轮追问「所以呢？」时上下文仍知道发过图。 + +### 2.1 智能体主动找用户（空闲触发） + +默认**全局关闭**。开启后，空闲一段时间的用户可被角色主动联系。 + +1. `.env` 设置 `PROACTIVE_ENABLED=true` 并重启服务 +2. 在 `/app` 机器人卡片中打开 **「主动找用户聊天」** 并保存参数（空闲小时 / 间隔 / 每日上限 / 安静时段） +3. 在用户列表对该 peer 勾选 **「允许主动」**（仅已批准用户） +4. 对方须**曾经聊过**（系统存有 iLink `context_token`）；从未发过消息的人无法冷启动主动触达 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `PROACTIVE_ENABLED` | `false` | 全局总闸 | +| `PROACTIVE_IDLE_HOURS` | `12` | 默认空闲阈值（小时） | +| `PROACTIVE_MIN_INTERVAL_HOURS` | `24` | 两次主动最小间隔 | +| `PROACTIVE_MAX_PER_DAY` | `1` | 每用户每日上限 | +| `PROACTIVE_QUIET_HOURS` | `0-8` | 安静时段（上海时区）；空字符串关闭 | +| `PROACTIVE_SCAN_INTERVAL_SEC` | `300` | 扫描周期 | +| `PROACTIVE_MAX_PER_SCAN` | `10` | 每轮最多发送数 | + +### 2.2 用户之间通过 @LINUX DO 用户名对话 + +机器人可中继两个已绑定用户的文字消息（**不经过 LLM**）。 + +1. 双方均用 LINUX DO 登录 `/app` +2. 用户中心 → **用户对话** → 生成绑定码 → 微信给任意机器人发 `/绑定 ABC123` +3. 双方至少各给机器人发过一次消息（写入 `context_token`，否则不可达） +4. A 发送整条消息 `@对方用户名` → B 收到请求 → `/同意` +5. 会话中直接发文字（前缀 `[用户名]`）；`/断开` 结束；空闲约 30 分钟自动结束 + +| 命令 | 说明 | +|------|------| +| `/绑定 CODE` | 认领绑定码 | +| `/解绑` | 解除绑定 | +| `/我的身份` | 查看绑定与会话状态 | +| `@username` | 发起对话请求（整条消息） | +| `/同意` `/拒绝` | 处理入站请求 | +| `/取消请求` | 取消自己发出的请求 | +| `/断开` | 结束当前用户对话 | +| `/拉黑 用户名` | 拉黑（无法再互相 @） | +| `/取消拉黑 用户名` | 移出黑名单 | +| `/黑名单` | 查看黑名单 | + +### 2.3 管理后台广播（全站 / 单 bot 推送） + +管理员在 `/admin` → **广播** 可向微信用户推送**纯文本**（系统更新、通知等）。 + +1. 登录管理后台 → 侧栏 **广播** +2. 撰写文本，选择范围： + - **全部机器人**：每个 bot 下全部有 `context_token` 的 peer（含未批准） + - **指定机器人**：多选 bot，再向其可触达 peer 群发 + - **指定 Peer**：选 bot → 勾选 peers +3. **预估人数** → 确认后创建异步任务；列表可看进度 / 取消 +4. 机器人详情页也可 **向此 bot 群发** / 对某 peer **发消息**（跳转并预填） + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `BROADCAST_INTERVAL_MS` | `200` | 两条消息间隔（限速，降低 iLink 风险） | +| `BROADCAST_MAX_TEXT` | `2000` | 文本最大长度 | +| `BROADCAST_HISTORY` | `100` | 保留的历史任务数 | + +约束： + +- 从未给 bot 发过消息的用户**无法送达**（无 context_token） +- 需 Worker 开启（`WORKER_ENABLED` 默认 true）；Worker 关则任务一直 `pending` +- 已发出的消息**无法撤回**；取消只停止剩余目标 +- 不进 LLM / 人设记忆 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `P2P_ENABLED` | `true` | 总开关（`false` 关闭） | +| `P2P_BIND_CODE_TTL_SEC` | `600` | 绑定码有效期 | +| `P2P_REQUEST_TTL_SEC` | `300` | 对话请求有效期 | +| `P2P_SESSION_IDLE_SEC` | `1800` | 会话空闲超时 | +| `P2P_RELAY_MAX_CHARS` | `500` | 单条中继最大字数 | +| `P2P_MAX_REQUESTS_PER_DAY` | `20` | 每 peer 每日发起 `@` 次数 | + +跨 Bot 可用：A 绑在 Bot1、B 绑在 Bot2，中继走目标方 bot 凭证 + 其 `context_token`。 + +### 2.4 记忆检索与时间工具 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `MEMORY_TOP_K` | `12` | 记忆超过全量阈值时注入条数 | +| `MEMORY_FULL_INJECT_MAX` | `20` | ≤ 此数时仍全量注入（与旧行为一致） | +| `MEMORY_MAX_ITEMS` | `100` | 每个 peer+人设最多存储条数 | +| `TIME_TOOL_ENABLED` | `true` | 允许模型调用 `get_current_time` | +| `TIME_TOOL_TIMEZONE` | `Asia/Shanghai` | 默认时区 | + +用户中心 → 微信用户行 → **记忆**：查看 / 删单条 / 清空。 + +日志关键字：`[proactive]`（`action=send|skip|lock_miss|no_ctx`）；`[broadcast]`（管理广播任务）。 + +### 2.5 自定义模型与联网搜索（经 HF 工具服务） + +主站**只**直连管理员配置的平台 LLM；用户自定义 API 与联网搜索一律经 +`huggingface/wechat-ai-tools` 出站。 + +| 环境变量 | 说明 | +|----------|------| +| `TOOLS_BASE_URL` | 工具服务根地址（HF Space / 自托管容器） | +| `TOOLS_API_KEY` | 与工具服务共享的调用密钥 | +| `LLM_PROVIDER_SECRET` | 加密用户保存的自定义 API Key（必填，否则无法添加连接） | +| `WEB_SEARCH_ENABLED` | 全局搜索开关；人设还需自行开启 | + +用户路径：`/app` → **我的模型** 添加连接 → 人设编辑器里选择该连接 / 勾选联网搜索。 + +排查： +- `pnpm diag` 会探测 `TOOLS_BASE_URL/health` +- `/health/ready` 在 `WEB_SEARCH_ENABLED=true` 且 tools 不可达时返回 503（结果缓存 15s） +- 日志与审计**不记录** upstream api_key + +### 2.6 Chatflow + +人设可切 `chatflow` 模式，用 `/chatflow?persona=` 编排流程图。 + +| 环境变量 | 默认 | 说明 | +|----------|------|------| +| `CHATFLOW_HTTP_ALLOWLIST` | 空 | http 节点额外允许的 host（tools host:port 始终允许）；`*` = 任意公网，内网/云元数据仍拦 | +| `CHATFLOW_MAX_STEPS` | `32` | 单次执行最大步数 | +| `CHATFLOW_MAX_NODES` | `40` | 图最大节点数 | + +要点：chatflow 人设**不参与主动联系**；试聊强制走平台模型。详见 `docs/chatflow.md`。 + +## 3. 管理员 + +- 配置了 `LINUXDO_ADMIN_IDS`：名单内用户登录后为管理员 +- 未配置：`FIRST_USER_IS_ADMIN` 默认 true，**首个登录用户**自动管理员 +- 打开 `/admin`：今日 Token、用户数、机器人、审计、**广播** + +## 4. Worker 与规模 + +API 与 iLink Worker **同进程**（单镜像 / 单容器）。 + +- 收消息：`getUpdates` 长轮询（每 bot 一路，有 `MAX_BOTS_PER_WORKER` 上限） +- 回消息：进程内 inbox 队列 + `REPLY_CONCURRENCY` 并发，避免 LLM 堵住轮询 +- 日志出现 `at capacity`：提高 `MAX_BOTS_PER_WORKER`，或同镜像多副本分片 + +## 5. 故障 + +| 现象 | 处理 | +|------|------| +| Redis Connection closed / 占位符 | 填真实 Upstash `rediss://` URL | +| OAuth redirect_uri mismatch | 控制台回调与 `.env` 一致 | +| OAuth userinfo 失败 | 确认 scope `openid profile`（已默认） | +| 无管理员 | 清空 Redis 用户或设置 `LINUXDO_ADMIN_IDS` 后重登 | +| 微信 session expired | 在用户中心删除机器人后重新扫码 | +| 无 AI 回复 | 检查 LLM_API_KEY、用户是否批准 | +| `at capacity` / 部分 bot 不 poll | 调高 `MAX_BOTS_PER_WORKER` | +| 自定义模型报 TOOLS_BASE_URL required | 部署工具服务并配置 `TOOLS_BASE_URL` / `TOOLS_API_KEY` | +| 添加模型连接 503 | 未设置 `LLM_PROVIDER_SECRET` | +| chatflow `http_blocked` | 目标域名不在 tools host / `CHATFLOW_HTTP_ALLOWLIST`；或命中内网段（报文里带 `private host blocked` 与原因）；或重定向跳进内网 | +| chatflow http 节点全部 `resolves to ...` 被拦 | 本机 DNS 在劫持解析（把所有域名答成 CGNAT/`198.18` 之类占位地址）。先 `nslookup` 确认，再排查 resolver，不要直接放宽白名单 | +| chatflow `search_disabled` | 人设未开搜索，或 `WEB_SEARCH_ENABLED` / TOOLS 未配 | +| `/health/ready` 503 但 Redis 正常 | tools 网关不可达（见 §2.5） | + +## 6. 备份 + +- Upstash：控制台备份 / 导出（按套餐） +- Redis `wa:bot:{id}:creds`（Bot token，敏感） +- `.env`（勿提交 Git） + +## 7. 文档 + +- Upstash：`docs/upstash-redis.md` +- OAuth：`docs/oauth-linuxdo.md` +- API：`docs/admin-api.md` +- AI 网关（主站↔HF 契约）：`docs/ai-gateway.md` +- Chatflow：`docs/chatflow.md` diff --git a/docs/runtime-settings.md b/docs/runtime-settings.md new file mode 100644 index 0000000..462c312 --- /dev/null +++ b/docs/runtime-settings.md @@ -0,0 +1,145 @@ +# 运行时配置（管理面板） + +`/admin` → **设置** 页可以改绝大多数原本只能写在 `.env` 里的配置。 + +## 优先级 + +``` +.env（进程启动时读一次） ← 默认值 + ↓ 被覆盖 +wa:settings:runtime（Redis JSON） ← 管理面板写入 + ↓ +生效配置（进程内的同一个 cfg 对象） +``` + +- Redis 里**没有**某一项时，就用 `.env` 的值。 +- 面板里把某项改回 `.env` 默认值，会**删除**该覆盖 —— 以后再改 `.env` 又能生效。 +- 「全部恢复默认」删除整份 Redis 覆盖文档。 + +## 传播 + +每个节点每 **5 秒** 读一次 `wa:settings:runtime`，有变化才写入本进程配置并推给各服务。 +所以：改动在**本节点立即生效**，其他节点**最多 5 秒**。没有用 pub/sub —— 一次 GET +相对请求路径上的 Redis 流量可以忽略，也省掉一条订阅连接。 + +## 范围 + +**不可配置（env-only）**，因为它们要么在能读 Redis 之前就得正确，要么改了会破坏已有数据： + +| 变量 | 原因 | +|------|------| +| `REDIS_URL` | 鸡生蛋：覆盖本身存在 Redis 里 | +| `LLM_BASE_URL` / `LLM_API_KEY` / `LLM_MODEL` | 平台模型凭证 | +| `LLM_PROVIDER_SECRET` | 改了之后所有用户已加密的自定义模型 key 全部解不开 | +| `WECHAT_AI_TOKEN` / `LINUXDO_ADMIN_IDS` | 权限根，改错会把自己锁在外面 | +| `SESSION_COOKIE_NAME` / `COOKIE_SECURE` | 改了当场踢掉所有会话 | +| `PUBLIC_BASE_URL` / `CORS_ORIGINS` | 站点自身 URL 与跨域白名单 | +| `WECHAT_AI_HOST` / `WECHAT_AI_PORT` | 监听端口在 `listen()` 时固定 | + +其余全部在面板里，包括 `TOOLS_BASE_URL` / `TOOLS_API_KEY`（密钥在 UI 里只显示掩码）。 + +`apps/api/src/runtime-config.test.ts` 有一条守卫用例：**任何新增的 AppConfig 字段**要么进 +`SETTING_SPECS`，要么进那份 env-only 名单，否则测试直接失败。 + +`DEFAULT_PERSONA_SLUG` 刻意不在面板里：`cfg.defaultPersonaSlug` 在代码里没有任何消费方， +默认人设走的是数据库 `is_default`（管理台「人设」→ 设默认）。放进面板只会得到一个 +「保存成功但什么都没发生」的控件。 + +## 需重启的项 + +绝大多数配置是热生效的。以下几项写入 Redis 后会打上橙色「需重启」徽章： + +| 项 | 为什么 | +|----|--------| +| `WORKER_ENABLED` | `worker.start()` 只在启动时跑一次 | +| `REPLY_CONCURRENCY` | 消费者池在 `start()` 一次性拉起，没有单消费者取消机制，缩容做不到 | +| `LOG_LEVEL` | Fastify 创建实例时固化 logger | +| `STICKER_MAX_BYTES` | 路由注册时固化 `bodyLimit` | + +## 热生效是怎么做到的 + +`cfg` 对象**按引用**传给 `registerRoutes` 和各个服务，所以： + +- **路由**：87 处 `ctx.cfg.*` 是每请求读的，原地改 `cfg` 就够了，零改动。 +- **在构造时快照选项的服务**：由 `runtime-config-apply.ts` 显式推送 —— + `ChatService` / `TryChatService` / `BotWorkerManager` / `ActivityBus` 各有一个 + `applyRuntimeOptions`（或 `applyRuntimeConfig`）。 + +几个需要特殊处理、否则会静默失效的地方： + +- `ChatService.webSearch` 原本在构造函数里判断一次，`WEB_SEARCH_ENABLED=false` 启动就永远 + 是 `null`。现在按 tools 配置的指纹惰性重建。 +- `BotWorkerManager` 的 10 个 `readonly` 标量改成可写，且 setter 里复刻了构造函数的 + clamp —— 面板不能写进一个构造函数本来会拒绝的值（比如 `leaseRenewSec=0`）。 +- `leaseRenewSec` 还决定 `setInterval` 周期，改动时重新装载定时器。 +- `ProactiveScheduler.globalEnabled` 被 `start()` 闩住（早退且留下 `stopped=true`）， + false→true 必须重新进 `start()`，光改标志没用。 +- `setRedisCommandHook` 现在无条件安装，否则 `DATA_STREAM_ENABLED` 这个开关永远是死的。 + +## API + +**整个页面（含只读）限超管。** 超管 = 仍是管理员的用户里 `created_at` 最早那位。 + +| Method | Path | 权限 | 说明 | +|--------|------|------|------| +| GET | `/api/v1/admin/settings/runtime` | **超管** | 分组 + 全部项（当前值 / env 默认值 / 是否已覆盖 / 是否需重启）+ 交叉校验警告 | +| PATCH | `/api/v1/admin/settings/runtime` | **超管** | `{ patch: {key: value}, reset: [key] }` | +| POST | `/api/v1/admin/settings/runtime/reset` | **超管** | 删除全部覆盖 | + +连读也限超管：这个面板能拿到 tools 网关密钥，能整个集群关掉 worker， +payload 本身也把每一个调参项和当前生效值列了出来。普通管理员访问返回 +403 `super_admin_required`。 + +前端跟着服务端一起收口，和「节点 / 广播 / 数据流」用同一套三处门禁： +`switchTab` 拦截并回落到「系统」页、`boot()` 隐藏侧栏按钮与 `
`、 +命令面板（Ctrl/⌘K）过滤掉该条目。 + +两个写操作都会记审计（`runtime_settings_updated` / `runtime_settings_reset`）， +含变更的 key 列表与需重启的 key。 + +密钥字段（`type: "secret"`）的约定： + +- GET 只返回掩码 `••••••••`，永远不回真值 +- PATCH 传空字符串 = **不修改** +- PATCH 传 `-` = **清空** +- 把掩码原样回传会被忽略，不会变成字面值 + +## 交叉校验 + +保存时会返回警告（只提示，不改用户填的值）：租约 TTL ≤ 续约间隔、延迟上下限倒挂、 +`memoryFullInjectMax < memoryTopK`、开了联网但没配 tools 网关、配了网关但密钥为空、 +二次过滤与模型直出 JSON 同时开启。 + +## Redis 读失败时的行为 + +「读不到」和「不存在」必须分开——把两者都当成「没有覆盖」会让一次网络抖动 +把整个节点悄悄退回 `.env`（此前被面板收紧的开关会瞬间放开）。所以： + +- **轮询读失败** → 保持上一份已知配置不动，打一条日志，`refresh()` 返回 false。 +- **写路径读失败** → 直接拒绝，返回 **503**。绝不能基于失败的读做合并再写回， + 那会把其他所有覆盖一起抹掉。 +- **例外：全部恢复默认**。它本来就要丢弃旧文档，所以读不到时也允许执行 —— + 这是文档损坏时唯一的产品内自救路径。 + +## 取值受限的项 + +`SettingSpec.options` 声明闭集，面板渲染成下拉框，服务端对不在集合里的值直接拒绝 +（不做替换，免得把拼写错误藏起来）。目前只有 `LOG_LEVEL` 用到：pino 遇到未知级别会在 +Fastify 建实例时抛错，而这一项又是「需重启」，一个拼写错误会在下次重启时让每个节点 +反复崩溃，而且管理接口起不来、改不回去。 + +## 并发写 + +`patch()` 是读-改-写：读 Redis 当前文档 → 合并 → 整份写回。合并是**按文档**而不是按字段的， +所以两个超管在两个节点同时保存，后写的那份会带着自己的改动覆盖掉前一份的全部改动。 + +为此写路径上加了一把短锁 `wa:settings:runtime:lock`（`SET NX EX 5`，最多重试 5 次 × +120ms），把整个集群的读-改-写串起来。锁只在写时用，读路径完全不碰它。 +拿不到锁时不阻塞管理员，仍然继续写 —— 锁是降低窗口，不是强一致保证；写操作是超管专属 +且极低频，这个取舍是刻意的。 + +## 多节点注意 + +`MAX_BOTS_PER_WORKER`、`LEASE_TTL_SEC`、`REBALANCE_SLACK` 参与的是**集群共享**的租约/再平衡 +协议。因为覆盖存在 Redis、所有节点读同一份，各节点最终看到的是同一组值 —— 但在 5 秒收敛 +窗口内可能短暂不一致，表现为一次多余的再平衡。改这几项建议避开高峰。 diff --git a/docs/upstash-redis.md b/docs/upstash-redis.md new file mode 100644 index 0000000..db4d462 --- /dev/null +++ b/docs/upstash-redis.md @@ -0,0 +1,95 @@ +# 使用 Upstash Redis + +本项目通过 **ioredis + Redis 协议** 连接远端 Redis，兼容 [Upstash](https://upstash.com/)。 + +## 1. 创建数据库 + +1. 登录 [Upstash Console](https://console.upstash.com/) +2. **Create Database** → 选区域（建议离你服务器近的，如 `ap-southeast-1`） +3. 打开数据库 → **Connect** / **REST API & Redis** + +## 2. 复制连接串 + +在 Connect 面板选 **ioredis / Node.js**，复制 **Redis URL**，形如： + +```text +rediss://default:AXxxxx@xxxxxx.upstash.io:6379 +``` + +注意： + +| 项 | 说明 | +|----|------| +| 协议 | 必须是 **`rediss://`**（多一个 s = TLS） | +| 用户名 | 一般为 `default` | +| 密码 | Upstash 提供的 token，当作密码 | +| 端口 | `6379` | + +不要用 Upstash 的 **REST URL**（`https://xxx.upstash.io`）填到 `REDIS_URL`——那是 HTTP REST，不是本项目用的协议。 + +## 3. 写入 `.env` + +```env +REDIS_URL=rediss://default:你的密码@你的主机.upstash.io:6379 +``` + +可选： + +```env +# 连接超时毫秒（默认 15000） +REDIS_CONNECT_TIMEOUT_MS=15000 +# 若 URL 是 redis:// 但强制 TLS（一般不需要，rediss:// 已够） +REDIS_TLS=true +# 并发命令自动 pipeline（默认开；高延迟远端 Redis 强烈建议保持） +REDIS_AUTO_PIPELINE=true +# 进程内 session/user 缓存（默认开；显著降低鉴权 RTT） +REDIS_L1_CACHE=true +# TCP keep-alive 毫秒（默认 10000） +REDIS_KEEPALIVE_MS=10000 +``` + +## 延迟优化建议 + +| 项 | 说明 | +|----|------| +| **区域** | Upstash 选与 API 服务器最近的 region（跨洲 RTT 常 150–300ms，串行几条命令就到秒级） | +| **避免 N+1** | 列表接口应批量 MGET/pipeline；本项目已对 me/bots、me/peers、广场列表等做批处理 | +| **L1 缓存** | 鉴权路径对 session/user 做短 TTL 进程缓存，重复请求几乎零 RTT | +| **命令数** | 免费档按命令计费；pipeline/MGET 既降延迟也降用量 | + +## 4. 验证 + +```powershell +cd F:\Code-Other-4\WeChat-AI +pnpm diag +``` + +应看到类似： + +```text +✓ REDIS_URL=rediss://... +✓ PONG +``` + +然后： + +```powershell +pnpm db:seed +pnpm dev +``` + +## 5. 常见问题 + +| 现象 | 处理 | +|------|------| +| `ECONNREFUSED` / 连不上 | 检查 URL 是否完整、是否 `rediss://`、密码有无复制错 | +| `ENOTFOUND` | 主机名错误 | +| TLS / certificate 错误 | 确认用 `rediss://`；升级 Node 20+ | +| 免费额度耗尽 | Upstash 控制台看 Command 用量；开发时减少轮询/调试 | +| 误填 REST token URL | 改用 **Redis** 协议 URL，不是 `https://...upstash.io` | + +## 6. 安全建议 + +- 不要把 `REDIS_URL` 提交到 Git（已在 `.gitignore` 的 `.env` 中） +- 生产环境在 Upstash 开启合适的 IP 限制（若控制台提供） +- 定期轮换 token diff --git a/huggingface/wechat-ai-tools/.dockerignore b/huggingface/wechat-ai-tools/.dockerignore new file mode 100644 index 0000000..b97a765 --- /dev/null +++ b/huggingface/wechat-ai-tools/.dockerignore @@ -0,0 +1,12 @@ +.env +.venv +venv +__pycache__ +*.py[cod] +.pytest_cache +.mypy_cache +.git +*.md +!README.md +tests +.ruff_cache diff --git a/huggingface/wechat-ai-tools/.env.example b/huggingface/wechat-ai-tools/.env.example new file mode 100644 index 0000000..f752858 --- /dev/null +++ b/huggingface/wechat-ai-tools/.env.example @@ -0,0 +1,28 @@ +# ── Auth (must match main site TOOLS_API_KEY) ─────────── +TOOLS_API_KEY=change-me-shared-with-main-site + +# ── Optional platform fallback when request has no upstream ── +# Main site usually calls platform LLM itself; these are used when +# chat/completions is invoked without body.upstream (e.g. demos). +UPSTREAM_LLM_BASE_URL=https://api.openai.com/v1 +UPSTREAM_LLM_API_KEY= +UPSTREAM_LLM_MODEL=gpt-4o-mini + +# Allow per-request upstream from main site (user custom APIs) +ALLOW_REQUEST_UPSTREAM=true +# Reject private/metadata IPs for upstream base_url +UPSTREAM_DENY_PRIVATE=true +UPSTREAM_TIMEOUT_MS=60000 +MAX_BODY_BYTES=1048576 + +# Search +SEARCH_PROVIDER=ddg +# SEARXNG_URL=https://searx.example/search +# TAVILY_API_KEY= +SEARCH_TIMEOUT_MS=15000 +SEARCH_MAX_RESULTS=8 + +# Server +HOST=0.0.0.0 +PORT=7860 +LOG_LEVEL=info diff --git a/huggingface/wechat-ai-tools/Dockerfile b/huggingface/wechat-ai-tools/Dockerfile new file mode 100644 index 0000000..4bf9cd4 --- /dev/null +++ b/huggingface/wechat-ai-tools/Dockerfile @@ -0,0 +1,46 @@ +# syntax=docker/dockerfile:1 +# wechat-ai-tools — Hugging Face Spaces / self-hosted Docker image +# +# Build: +# docker build -t wechat-ai-tools:latest -f huggingface/wechat-ai-tools/Dockerfile huggingface/wechat-ai-tools +# # or from this directory: +# docker build -t wechat-ai-tools:latest . +# +# Run: +# docker run --rm -p 7860:7860 --env-file .env wechat-ai-tools:latest +# +# HF Spaces: set SDK to Docker; Space will use this Dockerfile. + +FROM python:3.12-slim-bookworm + +LABEL org.opencontainers.image.title="wechat-ai-tools" \ + org.opencontainers.image.description="WeChat-AI tools gateway: web search + user custom LLM proxy" \ + org.opencontainers.image.source="https://github.com/wechat-ai/wechat-ai" + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HOST=0.0.0.0 \ + PORT=7860 + +# Non-root user (HF Spaces compatible) +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --create-home --home-dir /home/app app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY --chown=app:app . . + +USER app + +EXPOSE 7860 + +HEALTHCHECK --interval=30s --timeout=8s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/health', timeout=5)" + +# HF Spaces and local both use PORT (default 7860) +CMD ["sh", "-c", "uvicorn app:app --host ${HOST:-0.0.0.0} --port ${PORT:-7860}"] diff --git a/huggingface/wechat-ai-tools/README.md b/huggingface/wechat-ai-tools/README.md new file mode 100644 index 0000000..96a95dc --- /dev/null +++ b/huggingface/wechat-ai-tools/README.md @@ -0,0 +1,139 @@ +# wechat-ai-tools + +WeChat-AI 的 **工具网关**（可部署到 Hugging Face Spaces 或任意 Docker 主机）。 + +## 职责边界 + +| 调用方 | 走哪里 | +|--------|--------| +| **管理员配置的平台 LLM** | **主站直连**（`LLM_BASE_URL` / `LLM_API_KEY` / `LLM_MODEL`） | +| **用户自定义 OpenAI 兼容 API** | **本服务出站**（主站只请求本服务，body 带 `upstream`） | +| **联网搜索** | **本服务出站**（`POST /v1/web-search`） | + +主站配置： + +```env +# 平台 LLM（管理员） +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=sk-... +LLM_MODEL=gpt-4o-mini + +# 工具网关（用户自定义 API + 搜索） +TOOLS_BASE_URL=http://127.0.0.1:7860 +TOOLS_API_KEY=change-me-shared-with-main-site +WEB_SEARCH_ENABLED=true +``` + +本服务： + +```env +TOOLS_API_KEY=change-me-shared-with-main-site +ALLOW_REQUEST_UPSTREAM=true +UPSTREAM_DENY_PRIVATE=true +# 可选：无 upstream 时的兜底模型（演示用） +# UPSTREAM_LLM_BASE_URL=... +# UPSTREAM_LLM_API_KEY=... +# UPSTREAM_LLM_MODEL=... +``` + +## API + +### `GET /health` + +存活与配置摘要（不含密钥）。 + +### `POST /v1/web-search` + +```json +{ "query": "微信开放平台", "max_results": 5 } +``` + +鉴权：`Authorization: Bearer ` 或 `X-API-Key`。 + +### `POST /v1/chat/completions` + +OpenAI 兼容子集。用户自定义模型示例： + +```json +{ + "model": "gpt-4o-mini", + "messages": [{ "role": "user", "content": "hi" }], + "upstream": { + "base_url": "https://api.siliconflow.cn/v1", + "api_key": "sk-user-key", + "model": "Qwen/Qwen2.5-7B-Instruct" + } +} +``` + +- `upstream` 由主站解密用户连接后注入；本服务**不落库**密钥。 +- 日志只记 host/model，不记 api_key。 +- `UPSTREAM_DENY_PRIVATE=true` 时拒绝指向内网的 base_url。 + +## 本地运行 + +```bash +cd huggingface/wechat-ai-tools +python -m venv .venv +# Windows: .venv\Scripts\activate +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # 编辑 TOOLS_API_KEY 等 +uvicorn app:app --host 0.0.0.0 --port 7860 --reload +``` + +测试： + +```bash +pytest -q +``` + +## Dockerfile 打包镜像 + +在本目录： + +```bash +docker build -t wechat-ai-tools:latest . +docker run --rm -p 7860:7860 --env-file .env wechat-ai-tools:latest +``` + +在仓库根目录： + +```bash +docker build -t wechat-ai-tools:latest -f huggingface/wechat-ai-tools/Dockerfile huggingface/wechat-ai-tools +``` + +推送到私有仓库示例： + +```bash +docker tag wechat-ai-tools:latest registry.example.com/wechat-ai-tools:1.0.0 +docker push registry.example.com/wechat-ai-tools:1.0.0 +``` + +镜像特性： + +- 基础镜像 `python:3.12-slim-bookworm` +- 非 root 用户 `app` (uid 1000) +- 暴露 `7860`（HF Spaces 默认） +- `HEALTHCHECK` → `GET /health` +- 入口：`uvicorn app:app --host 0.0.0.0 --port $PORT` + +## Hugging Face Spaces + +1. 新建 Space，**SDK = Docker** +2. 将本目录文件推到 Space 仓库根（或 monorepo 中指定 Dockerfile 路径） +3. Space Secrets 设置：`TOOLS_API_KEY`、可选 `UPSTREAM_LLM_*` +4. 主站 `TOOLS_BASE_URL=https://.hf.space` + +## 与主站 compose（可选 profile） + +主站 `docker-compose` 可将本镜像作为 `tools` 服务侧车；主站容器只访问 `http://tools:7860`，**不要**把用户自定义 API 的出站放到主站容器。 + +## 安全清单 + +- [x] 共享 `TOOLS_API_KEY` +- [x] upstream SSRF 防护（禁私网 / metadata） +- [x] 请求体大小限制 +- [x] 上游超时 +- [x] 密钥不写日志 +- [ ] 生产建议：仅允许主站出口 IP（反代层） diff --git a/huggingface/wechat-ai-tools/app.py b/huggingface/wechat-ai-tools/app.py new file mode 100644 index 0000000..2d2f340 --- /dev/null +++ b/huggingface/wechat-ai-tools/app.py @@ -0,0 +1,90 @@ +""" +wechat-ai-tools — AI gateway for WeChat-AI user custom LLM + web search. + +Platform (admin) LLM is called by the main site directly. +User-configured custom APIs and web search egress through this service. + +Deploy: Docker image / Hugging Face Spaces (Docker). +""" + +from __future__ import annotations + +import logging + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse + +from config import get_settings +from routers import chat, health, search + +settings = get_settings() +logging.basicConfig( + level=getattr(logging, (settings.log_level or "info").upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +log = logging.getLogger("wechat-ai-tools") + +app = FastAPI( + title="wechat-ai-tools", + description=( + "HTTP tools gateway for WeChat-AI: web search + proxy for user custom " + "OpenAI-compatible LLM APIs. Platform LLM stays on the main site." + ), + version="1.0.0", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=False, + allow_methods=["GET", "POST", "OPTIONS"], + allow_headers=["*"], +) + +app.include_router(health.router) +app.include_router(search.router) +app.include_router(chat.router) + + +@app.middleware("http") +async def limit_body_size(request: Request, call_next): + cl = request.headers.get("content-length") + if cl and cl.isdigit() and int(cl) > settings.max_body_bytes: + return JSONResponse( + status_code=413, + content={"detail": "request body too large"}, + ) + return await call_next(request) + + +@app.get("/") +async def root() -> dict: + return { + "service": "wechat-ai-tools", + "docs": "/docs", + "health": "/health", + "endpoints": [ + "POST /v1/web-search", + "POST /v1/chat/completions", + ], + "note": ( + "Main site uses platform LLM directly; user custom APIs and search " + "go through this gateway." + ), + } + + +def main() -> None: + import uvicorn + + uvicorn.run( + "app:app", + host=settings.host, + port=settings.port, + log_level=(settings.log_level or "info").lower(), + ) + + +if __name__ == "__main__": + main() diff --git a/huggingface/wechat-ai-tools/config.py b/huggingface/wechat-ai-tools/config.py new file mode 100644 index 0000000..fc3e92d --- /dev/null +++ b/huggingface/wechat-ai-tools/config.py @@ -0,0 +1,54 @@ +"""Runtime configuration for wechat-ai-tools (HF / Docker).""" + +from __future__ import annotations + +import os +from functools import lru_cache + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + populate_by_name=True, + ) + + tools_api_key: str = Field(default="", alias="TOOLS_API_KEY") + + upstream_llm_base_url: str = Field( + default="https://api.openai.com/v1", + alias="UPSTREAM_LLM_BASE_URL", + ) + upstream_llm_api_key: str = Field(default="", alias="UPSTREAM_LLM_API_KEY") + upstream_llm_model: str = Field(default="gpt-4o-mini", alias="UPSTREAM_LLM_MODEL") + + allow_request_upstream: bool = Field(default=True, alias="ALLOW_REQUEST_UPSTREAM") + upstream_deny_private: bool = Field(default=True, alias="UPSTREAM_DENY_PRIVATE") + upstream_timeout_ms: int = Field(default=60_000, alias="UPSTREAM_TIMEOUT_MS") + max_body_bytes: int = Field(default=1_048_576, alias="MAX_BODY_BYTES") + + search_provider: str = Field(default="ddg", alias="SEARCH_PROVIDER") + searxng_url: str = Field(default="", alias="SEARXNG_URL") + tavily_api_key: str = Field(default="", alias="TAVILY_API_KEY") + search_timeout_ms: int = Field(default=15_000, alias="SEARCH_TIMEOUT_MS") + search_max_results: int = Field(default=8, alias="SEARCH_MAX_RESULTS") + + host: str = Field(default="0.0.0.0", alias="HOST") + port: int = Field(default=7860, alias="PORT") + log_level: str = Field(default="info", alias="LOG_LEVEL") + + +@lru_cache +def get_settings() -> Settings: + return Settings() + + +def env_bool(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} diff --git a/huggingface/wechat-ai-tools/requirements.txt b/huggingface/wechat-ai-tools/requirements.txt new file mode 100644 index 0000000..ac0cde3 --- /dev/null +++ b/huggingface/wechat-ai-tools/requirements.txt @@ -0,0 +1,9 @@ +# Target Python 3.11–3.12 (Docker image uses 3.12). Avoid 3.14 until wheels catch up. +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +httpx==0.28.1 +pydantic==2.11.3 +pydantic-settings==2.8.1 +duckduckgo-search==8.0.1 +pytest==8.3.5 +pytest-asyncio==0.26.0 diff --git a/huggingface/wechat-ai-tools/routers/__init__.py b/huggingface/wechat-ai-tools/routers/__init__.py new file mode 100644 index 0000000..9c8ddfa --- /dev/null +++ b/huggingface/wechat-ai-tools/routers/__init__.py @@ -0,0 +1 @@ +# routers package diff --git a/huggingface/wechat-ai-tools/routers/chat.py b/huggingface/wechat-ai-tools/routers/chat.py new file mode 100644 index 0000000..2457cb3 --- /dev/null +++ b/huggingface/wechat-ai-tools/routers/chat.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request + +from config import Settings, get_settings +from routers.deps import require_tools_auth +from services.upstream_llm import UpstreamLlmError, chat_completions + +router = APIRouter(prefix="/v1", tags=["chat"]) + + +@router.post("/chat/completions") +async def openai_chat_completions( + request: Request, + settings: Settings = Depends(get_settings), + _: None = Depends(require_tools_auth), +) -> dict[str, Any]: + """ + OpenAI-compatible chat completions proxy. + + Main site uses this for **user custom LLM APIs** by sending: + { "messages": [...], "model": "...", "upstream": { "base_url", "api_key", "model" } } + + Without upstream, falls back to UPSTREAM_LLM_* on this service (optional). + """ + try: + body = await request.json() + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=400, detail="invalid JSON body") from exc + if not isinstance(body, dict): + raise HTTPException(status_code=400, detail="body must be a JSON object") + if "messages" not in body: + raise HTTPException(status_code=400, detail="messages is required") + + try: + return await chat_completions(body, settings) + except UpstreamLlmError as exc: + raise HTTPException(status_code=exc.status_code, detail=str(exc)) from exc diff --git a/huggingface/wechat-ai-tools/routers/deps.py b/huggingface/wechat-ai-tools/routers/deps.py new file mode 100644 index 0000000..218b27d --- /dev/null +++ b/huggingface/wechat-ai-tools/routers/deps.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from fastapi import Depends, HTTPException, Request +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + +from config import Settings, get_settings + +_bearer = HTTPBearer(auto_error=False) + + +def require_tools_auth( + request: Request, + creds: HTTPAuthorizationCredentials | None = Depends(_bearer), + settings: Settings = Depends(get_settings), +) -> None: + expected = (settings.tools_api_key or "").strip() + if not expected: + # Open mode (local demo only). Production should set TOOLS_API_KEY. + return + + token = "" + if creds and creds.scheme.lower() == "bearer": + token = (creds.credentials or "").strip() + if not token: + token = (request.headers.get("x-api-key") or "").strip() + + if token != expected: + raise HTTPException(status_code=401, detail="invalid or missing tools API key") diff --git a/huggingface/wechat-ai-tools/routers/health.py b/huggingface/wechat-ai-tools/routers/health.py new file mode 100644 index 0000000..a16027a --- /dev/null +++ b/huggingface/wechat-ai-tools/routers/health.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from config import get_settings + +router = APIRouter(tags=["health"]) + + +@router.get("/health") +async def health() -> dict: + settings = get_settings() + has_platform = bool( + (settings.upstream_llm_base_url or "").strip() + and (settings.upstream_llm_api_key or "").strip() + ) + return { + "ok": True, + "service": "wechat-ai-tools", + "auth_required": bool((settings.tools_api_key or "").strip()), + "allow_request_upstream": settings.allow_request_upstream, + "platform_upstream_configured": has_platform, + "search_provider": settings.search_provider, + } diff --git a/huggingface/wechat-ai-tools/routers/search.py b/huggingface/wechat-ai-tools/routers/search.py new file mode 100644 index 0000000..8609304 --- /dev/null +++ b/huggingface/wechat-ai-tools/routers/search.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel, Field + +from config import Settings, get_settings +from routers.deps import require_tools_auth +from services.web_search import SearchError, search_web + +router = APIRouter(prefix="/v1", tags=["search"]) + + +class WebSearchRequest(BaseModel): + query: str = Field(..., min_length=1, max_length=500) + max_results: int | None = Field(default=None, ge=1, le=10) + + +@router.post("/web-search") +async def web_search( + body: WebSearchRequest, + request: Request, + settings: Settings = Depends(get_settings), + _: None = Depends(require_tools_auth), +) -> dict[str, Any]: + _ = request + try: + results = await search_web( + body.query, + max_results=body.max_results, + settings=settings, + ) + except SearchError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"query": body.query.strip(), "results": results} diff --git a/huggingface/wechat-ai-tools/services/__init__.py b/huggingface/wechat-ai-tools/services/__init__.py new file mode 100644 index 0000000..0274469 --- /dev/null +++ b/huggingface/wechat-ai-tools/services/__init__.py @@ -0,0 +1 @@ +# services package diff --git a/huggingface/wechat-ai-tools/services/security.py b/huggingface/wechat-ai-tools/services/security.py new file mode 100644 index 0000000..17c5037 --- /dev/null +++ b/huggingface/wechat-ai-tools/services/security.py @@ -0,0 +1,72 @@ +"""SSRF guards for upstream LLM base URLs.""" + +from __future__ import annotations + +import ipaddress +import socket +from urllib.parse import urlparse + + +class UnsafeUpstreamError(ValueError): + pass + + +_BLOCKED_HOSTS = frozenset( + { + "localhost", + "localhost.localdomain", + "metadata.google.internal", + "metadata", + } +) + + +def _is_private_ip(ip: str) -> bool: + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return True + return bool( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_reserved + or addr.is_multicast + or addr.is_unspecified + ) + + +def validate_upstream_base_url(url: str, *, deny_private: bool = True) -> str: + """Return normalized base URL or raise UnsafeUpstreamError.""" + raw = (url or "").strip() + if not raw: + raise UnsafeUpstreamError("upstream base_url is empty") + if len(raw) > 2048: + raise UnsafeUpstreamError("upstream base_url too long") + + parsed = urlparse(raw) + if parsed.scheme not in ("http", "https"): + raise UnsafeUpstreamError("upstream base_url must be http(s)") + if not parsed.hostname: + raise UnsafeUpstreamError("upstream base_url missing host") + if parsed.username or parsed.password: + raise UnsafeUpstreamError("upstream base_url must not embed credentials") + + host = parsed.hostname.lower().rstrip(".") + if host in _BLOCKED_HOSTS or host.endswith(".local"): + raise UnsafeUpstreamError(f"upstream host not allowed: {host}") + + if deny_private: + try: + infos = socket.getaddrinfo(host, parsed.port or 443, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + raise UnsafeUpstreamError(f"upstream host resolve failed: {host}") from exc + for info in infos: + ip = info[4][0] + if _is_private_ip(ip): + raise UnsafeUpstreamError( + f"upstream resolves to blocked address: {host} -> {ip}" + ) + + # Normalize: no trailing slash (callers append /chat/completions) + return raw.rstrip("/") diff --git a/huggingface/wechat-ai-tools/services/upstream_llm.py b/huggingface/wechat-ai-tools/services/upstream_llm.py new file mode 100644 index 0000000..8139331 --- /dev/null +++ b/huggingface/wechat-ai-tools/services/upstream_llm.py @@ -0,0 +1,136 @@ +"""Proxy chat completions to an OpenAI-compatible upstream.""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from config import Settings +from services.security import UnsafeUpstreamError, validate_upstream_base_url + +log = logging.getLogger("wechat-ai-tools.llm") + + +class UpstreamLlmError(RuntimeError): + def __init__(self, message: str, *, status_code: int = 502): + super().__init__(message) + self.status_code = status_code + + +def resolve_upstream( + body: dict[str, Any], + settings: Settings, +) -> tuple[str, str, str]: + """ + Returns (base_url, api_key, model). + Prefer body.upstream when allowed; else platform defaults. + """ + upstream = body.get("upstream") + model_from_body = body.get("model") + + if isinstance(upstream, dict) and settings.allow_request_upstream: + base = str(upstream.get("base_url") or upstream.get("baseUrl") or "").strip() + key = str(upstream.get("api_key") or upstream.get("apiKey") or "").strip() + model = str( + upstream.get("model") + or model_from_body + or settings.upstream_llm_model + or "" + ).strip() + if not base or not key: + raise UpstreamLlmError( + "upstream.base_url and upstream.api_key are required", + status_code=400, + ) + try: + base = validate_upstream_base_url( + base, deny_private=settings.upstream_deny_private + ) + except UnsafeUpstreamError as exc: + raise UpstreamLlmError(str(exc), status_code=400) from exc + if not model: + raise UpstreamLlmError("model is required", status_code=400) + host = base.split("://", 1)[-1].split("/", 1)[0] + log.info("upstream mode=request host=%s model=%s", host, model) + return base, key, model + + if isinstance(upstream, dict) and not settings.allow_request_upstream: + raise UpstreamLlmError( + "per-request upstream is disabled on this tools instance", + status_code=403, + ) + + base = (settings.upstream_llm_base_url or "").strip() + key = (settings.upstream_llm_api_key or "").strip() + model = str(model_from_body or settings.upstream_llm_model or "").strip() + if not base or not key: + raise UpstreamLlmError( + "platform upstream not configured (UPSTREAM_LLM_BASE_URL / API_KEY)", + status_code=503, + ) + try: + base = validate_upstream_base_url( + base, deny_private=settings.upstream_deny_private + ) + except UnsafeUpstreamError as exc: + raise UpstreamLlmError(str(exc), status_code=500) from exc + if not model: + raise UpstreamLlmError("model is required", status_code=400) + host = base.split("://", 1)[-1].split("/", 1)[0] + log.info("upstream mode=platform host=%s model=%s", host, model) + return base, key, model + + +def _strip_gateway_fields(body: dict[str, Any]) -> dict[str, Any]: + """Remove fields that must not be forwarded to real providers.""" + out = dict(body) + out.pop("upstream", None) + return out + + +async def chat_completions( + body: dict[str, Any], + settings: Settings, +) -> dict[str, Any]: + base_url, api_key, model = resolve_upstream(body, settings) + payload = _strip_gateway_fields(body) + payload["model"] = model + + url = f"{base_url}/chat/completions" + timeout = max(1.0, settings.upstream_timeout_ms / 1000.0) + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "User-Agent": "wechat-ai-tools/1.0", + } + + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post(url, json=payload, headers=headers) + except httpx.TimeoutException as exc: + raise UpstreamLlmError("upstream LLM timeout", status_code=504) from exc + except httpx.RequestError as exc: + log.warning("upstream request error: %s", type(exc).__name__) + raise UpstreamLlmError( + f"upstream request failed: {type(exc).__name__}", + status_code=502, + ) from exc + + if resp.status_code >= 400: + # Do not leak upstream auth details + snippet = (resp.text or "")[:300] + log.warning("upstream HTTP %s body_len=%s", resp.status_code, len(resp.text or "")) + raise UpstreamLlmError( + f"upstream LLM error HTTP {resp.status_code}: {snippet}", + status_code=502 if resp.status_code >= 500 else 400, + ) + + try: + data = resp.json() + except Exception as exc: # noqa: BLE001 + raise UpstreamLlmError("upstream returned non-JSON", status_code=502) from exc + if not isinstance(data, dict): + raise UpstreamLlmError("upstream returned invalid JSON object", status_code=502) + return data diff --git a/huggingface/wechat-ai-tools/services/web_search.py b/huggingface/wechat-ai-tools/services/web_search.py new file mode 100644 index 0000000..845122f --- /dev/null +++ b/huggingface/wechat-ai-tools/services/web_search.py @@ -0,0 +1,124 @@ +"""Web search backends (DDG default; optional SearXNG / Tavily).""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from config import Settings + +log = logging.getLogger("wechat-ai-tools.search") + + +class SearchError(RuntimeError): + pass + + +def _clamp_max(n: int | None, settings: Settings) -> int: + if n is None: + return max(1, min(settings.search_max_results, 10)) + return max(1, min(int(n), settings.search_max_results, 10)) + + +async def search_web( + query: str, + *, + max_results: int | None, + settings: Settings, +) -> list[dict[str, str]]: + q = (query or "").strip() + if not q: + raise SearchError("query is required") + if len(q) > 500: + raise SearchError("query too long (max 500)") + + limit = _clamp_max(max_results, settings) + provider = (settings.search_provider or "ddg").strip().lower() + + if provider == "tavily": + return await _search_tavily(q, limit, settings) + if provider == "searxng": + return await _search_searxng(q, limit, settings) + return await _search_ddg(q, limit, settings) + + +async def _search_ddg(query: str, limit: int, settings: Settings) -> list[dict[str, str]]: + try: + from duckduckgo_search import DDGS + except ImportError as exc: + raise SearchError("duckduckgo-search not installed") from exc + + results: list[dict[str, str]] = [] + timeout_s = max(1.0, settings.search_timeout_ms / 1000.0) + try: + # DDGS is sync; run in thread via anyio is ideal — use sync for MVP simplicity + with DDGS() as ddgs: + raw = list(ddgs.text(query, max_results=limit)) + except Exception as exc: # noqa: BLE001 + log.warning("ddg search failed: %s", type(exc).__name__) + raise SearchError(f"search failed: {type(exc).__name__}") from exc + + for item in raw[:limit]: + results.append( + { + "title": str(item.get("title") or ""), + "url": str(item.get("href") or item.get("link") or ""), + "snippet": str(item.get("body") or item.get("snippet") or ""), + } + ) + return results + + +async def _search_searxng( + query: str, limit: int, settings: Settings +) -> list[dict[str, str]]: + base = (settings.searxng_url or "").strip().rstrip("/") + if not base: + raise SearchError("SEARXNG_URL not configured") + timeout = max(1.0, settings.search_timeout_ms / 1000.0) + params = {"q": query, "format": "json", "categories": "general"} + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(base, params=params) + resp.raise_for_status() + data: dict[str, Any] = resp.json() + out: list[dict[str, str]] = [] + for item in (data.get("results") or [])[:limit]: + out.append( + { + "title": str(item.get("title") or ""), + "url": str(item.get("url") or ""), + "snippet": str(item.get("content") or item.get("snippet") or ""), + } + ) + return out + + +async def _search_tavily( + query: str, limit: int, settings: Settings +) -> list[dict[str, str]]: + key = (settings.tavily_api_key or "").strip() + if not key: + raise SearchError("TAVILY_API_KEY not configured") + timeout = max(1.0, settings.search_timeout_ms / 1000.0) + payload = { + "api_key": key, + "query": query, + "max_results": limit, + "include_answer": False, + } + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post("https://api.tavily.com/search", json=payload) + resp.raise_for_status() + data: dict[str, Any] = resp.json() + out: list[dict[str, str]] = [] + for item in (data.get("results") or [])[:limit]: + out.append( + { + "title": str(item.get("title") or ""), + "url": str(item.get("url") or ""), + "snippet": str(item.get("content") or ""), + } + ) + return out diff --git a/huggingface/wechat-ai-tools/tests/test_security.py b/huggingface/wechat-ai-tools/tests/test_security.py new file mode 100644 index 0000000..10700b2 --- /dev/null +++ b/huggingface/wechat-ai-tools/tests/test_security.py @@ -0,0 +1,30 @@ +import pytest + +from services.security import UnsafeUpstreamError, validate_upstream_base_url + + +def test_accepts_https_public_host(): + # deny_private still resolves DNS; use example.com which is public + url = validate_upstream_base_url( + "https://example.com/v1/", + deny_private=False, + ) + assert url == "https://example.com/v1" + + +def test_rejects_localhost(): + with pytest.raises(UnsafeUpstreamError): + validate_upstream_base_url("http://localhost:8080/v1", deny_private=True) + + +def test_rejects_non_http(): + with pytest.raises(UnsafeUpstreamError): + validate_upstream_base_url("ftp://example.com/v1", deny_private=False) + + +def test_rejects_embedded_credentials(): + with pytest.raises(UnsafeUpstreamError): + validate_upstream_base_url( + "https://user:pass@example.com/v1", + deny_private=False, + ) diff --git a/huggingface/wechat-ai-tools/tests/test_upstream_resolve.py b/huggingface/wechat-ai-tools/tests/test_upstream_resolve.py new file mode 100644 index 0000000..5d122b2 --- /dev/null +++ b/huggingface/wechat-ai-tools/tests/test_upstream_resolve.py @@ -0,0 +1,50 @@ +import pytest + +from config import Settings +from services.upstream_llm import UpstreamLlmError, resolve_upstream + + +def test_request_upstream(): + settings = Settings( + ALLOW_REQUEST_UPSTREAM=True, + UPSTREAM_DENY_PRIVATE=False, + UPSTREAM_LLM_API_KEY="", + ) + base, key, model = resolve_upstream( + { + "model": "ignored-if-upstream-has-model", + "upstream": { + "base_url": "https://api.example.com/v1", + "api_key": "sk-test", + "model": "my-model", + }, + }, + settings, + ) + assert base == "https://api.example.com/v1" + assert key == "sk-test" + assert model == "my-model" + + +def test_platform_fallback(): + settings = Settings( + ALLOW_REQUEST_UPSTREAM=True, + UPSTREAM_DENY_PRIVATE=False, + UPSTREAM_LLM_BASE_URL="https://platform.example/v1", + UPSTREAM_LLM_API_KEY="sk-platform", + UPSTREAM_LLM_MODEL="gpt-mini", + ) + base, key, model = resolve_upstream({"messages": []}, settings) + assert "platform.example" in base + assert key == "sk-platform" + assert model == "gpt-mini" + + +def test_missing_platform_raises(): + settings = Settings( + ALLOW_REQUEST_UPSTREAM=True, + UPSTREAM_LLM_API_KEY="", + UPSTREAM_DENY_PRIVATE=False, + ) + with pytest.raises(UpstreamLlmError): + resolve_upstream({"messages": []}, settings) diff --git a/package.json b/package.json new file mode 100644 index 0000000..b97941a --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "wechat-ai", + "private": true, + "version": "0.12.0", + "description": "Multi-user WeChat roleplay via iLink + Redis + LINUX DO OAuth", + "packageManager": "pnpm@11.15.0", + "scripts": { + "build": "pnpm -r run build", + "dev": "pnpm --filter @wechat-ai/api dev", + "db:seed": "pnpm --filter @wechat-ai/db seed", + "db:migrate": "pnpm db:seed", + "diag": "pnpm --filter @wechat-ai/api diag", + "typecheck": "pnpm -r run typecheck", + "test": "pnpm -r run test", + "release:pack": "node scripts/release-pack.mjs", + "docker:build": "node scripts/docker-build.mjs", + "docker:up": "node scripts/docker-build.mjs --up" + }, + "engines": { + "node": ">=20" + } +} diff --git a/packages/core/package.json b/packages/core/package.json new file mode 100644 index 0000000..c37dbc0 --- /dev/null +++ b/packages/core/package.json @@ -0,0 +1,29 @@ +{ + "name": "@wechat-ai/core", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --import tsx --test src/**/*.test.ts" + }, + "dependencies": { + "@wechat-ai/db": "workspace:*", + "@wechat-ai/llm": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.13.10", + "tsx": "^4.19.3", + "typescript": "^5.8.2" + } +} diff --git a/packages/core/src/acceptance.test.ts b/packages/core/src/acceptance.test.ts new file mode 100644 index 0000000..3b869ca --- /dev/null +++ b/packages/core/src/acceptance.test.ts @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + approvePeer, + getPersonaBySlug, + listMemories, + openDatabase, + replaceMemories, + seedPersonas, + setAssignment, + upsertBotAccount, +} from "@wechat-ai/db"; +import type { LlmClient } from "@wechat-ai/llm"; +import { ChatService } from "./chat-service.js"; + +const redisUrl = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; + +class FakeLlm { + lastSystem = ""; + constructor(private reply: string) {} + async chat() { + return this.reply; + } + async chatWithUsage(messages: { role: string; content: string }[]) { + this.lastSystem = messages.find((m) => m.role === "system")?.content ?? ""; + return { + text: this.reply, + promptTokens: 1, + completionTokens: 1, + totalTokens: 2, + model: "fake", + }; + } +} + +describe("acceptance scenarios (Redis)", () => { + it("rejects unapproved and blocks /角色", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const botId = `bot_acc_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u1", + displayName: "t", + botToken: "test-token", + }); + const chat = new ChatService(db, new FakeLlm("ok") as unknown as LlmClient, { + allowUnapproved: false, + memoryExtractEveryN: 999, + }); + const unapproved = await chat.handleInbound({ + botAccountId: botId, + peerId: "p1@im.wechat", + text: "你好", + contextToken: "t1", + }); + assert.equal(unapproved.kind, "reject"); + await approvePeer(db, botId, "p1@im.wechat"); + const switchCmd = await chat.handleInbound({ + botAccountId: botId, + peerId: "p1@im.wechat", + text: "/角色 女友", + contextToken: "t2", + }); + assert.match(switchCmd.text ?? "", /后台|主人/); + await db.close(); + }); + + it("sticky memories per persona", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const botId = `bot_sticky_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u1", + displayName: "t", + botToken: "test-token", + }); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const gf = (await getPersonaBySlug(db, "girlfriend"))!; + const peer = "sticky@im.wechat"; + await approvePeer(db, botId, peer); + await setAssignment(db, botId, peer, cat.id); + await replaceMemories(db, botId, peer, cat.id, ["用户喜欢猫粮"]); + await replaceMemories(db, botId, peer, gf.id, ["用户喜欢约会"]); + await setAssignment(db, botId, peer, gf.id); + assert.equal( + (await listMemories(db, botId, peer, cat.id))[0]?.content, + "用户喜欢猫粮", + ); + await db.close(); + }); +}); diff --git a/packages/core/src/attachments.test.ts b/packages/core/src/attachments.test.ts new file mode 100644 index 0000000..d380cab --- /dev/null +++ b/packages/core/src/attachments.test.ts @@ -0,0 +1,267 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { flattenChatContent } from "@wechat-ai/llm"; +import type { ChatContentPart } from "@wechat-ai/llm"; +import { + buildAttachmentBlock, + buildChatMessages, + buildImageCaptionMessages, + buildUserContent, + describeAttachments, + type PromptAttachment, +} from "./prompt.js"; + +/** Caption mode output: described in words, bytes deliberately not forwarded. */ +const CAPTIONED: PromptAttachment = { + kind: "image", + mime: "image/png", + caption: "一只橘猫躺在键盘上", +}; + +const IMG: PromptAttachment = { + kind: "image", + mime: "image/png", + dataUri: "data:image/png;base64,AAAA", +}; +const BLIND_IMG: PromptAttachment = { kind: "image", mime: "image/png" }; +const VOICE: PromptAttachment = { kind: "voice", mime: "audio/silk" }; +const VIDEO: PromptAttachment = { kind: "video", mime: "video/mp4" }; + +describe("buildAttachmentBlock", () => { + it("is empty without attachments", () => { + assert.equal(buildAttachmentBlock(undefined), ""); + assert.equal(buildAttachmentBlock([]), ""); + }); + + it("tells the model a readable image was actually sent", () => { + const block = buildAttachmentBlock([IMG]); + assert.match(block, /图片 ×1/); + assert.match(block, /已随本条消息发给你/); + assert.match(block, /据实描述/); + }); + + it("forbids inventing contents it cannot perceive", () => { + const block = buildAttachmentBlock([VIDEO]); + assert.match(block, /视频 ×1/); + assert.match(block, /无法查看/); + assert.match(block, /不要猜测或编造/); + assert.doesNotMatch(block, /已随本条消息发给你/); + }); + + it("reports readable and unreadable counts of the same kind", () => { + const block = buildAttachmentBlock([IMG, BLIND_IMG, IMG]); + assert.match(block, /图片 ×3/); + assert.match(block, /其中 2 个已随本条消息发给你/); + }); + + it("groups mixed kinds in a stable order", () => { + const block = buildAttachmentBlock([VIDEO, VOICE, IMG]); + const order = ["图片", "语音", "视频"].map((k) => block.indexOf(k)); + assert.deepEqual( + order, + [...order].sort((a, b) => a - b), + "kinds should be listed image → voice → video → file", + ); + }); +}); + +describe("caption mode", () => { + it("tells the model to treat the description as what it saw", () => { + const block = buildAttachmentBlock([CAPTIONED]); + assert.match(block, /已由识图模型转成文字描述/); + assert.match(block, /当作你亲眼所见/); + assert.match(block, /不要往外扩写细节/); + // Must NOT claim the bytes were attached, and must NOT tell it to refuse. + assert.doesNotMatch(block, /已随本条消息发给你/); + assert.doesNotMatch(block, /无法查看/); + }); + + it("puts the caption in history so later turns keep the content", () => { + // The whole payoff: three turns later the model can still discuss the cat. + assert.equal( + describeAttachments("看这个", [CAPTIONED]), + "看这个\n[图片：一只橘猫躺在键盘上]", + ); + assert.equal( + describeAttachments("", [CAPTIONED]), + "[图片：一只橘猫躺在键盘上]", + ); + }); + + it("keeps the user turn plain text — the roleplay model gets no bytes", () => { + const content = buildUserContent("这是什么", [CAPTIONED]); + assert.equal(typeof content, "string"); + assert.match(String(content), /一只橘猫躺在键盘上/); + }); + + it("mixes captioned and blind attachments correctly", () => { + const block = buildAttachmentBlock([CAPTIONED, VIDEO]); + assert.match(block, /已由识图模型转成文字描述/); + assert.match(block, /视频 ×1[^\n]*无法查看/); + }); + + it("falls back to a bare tag when captioning produced nothing", () => { + const failed: PromptAttachment = { kind: "image", mime: "image/png" }; + assert.equal(describeAttachments("", [failed]), "[图片]"); + assert.match(buildAttachmentBlock([failed]), /无法查看/); + }); + + it("counts several captions separately rather than collapsing them", () => { + const second: PromptAttachment = { ...CAPTIONED, caption: "一杯咖啡" }; + assert.equal( + describeAttachments("", [CAPTIONED, second]), + "[图片：一只橘猫躺在键盘上][图片：一杯咖啡]", + ); + }); +}); + +describe("buildImageCaptionMessages", () => { + it("asks for an objective description, not roleplay", () => { + const msgs = buildImageCaptionMessages({ + dataUri: "data:image/png;base64,AAAA", + }); + assert.equal(msgs.length, 2); + const system = String(msgs[0]!.content); + assert.match(system, /图像描述器/); + assert.match(system, /不要扮演角色/); + assert.match(system, /不要猜测或编造/); + }); + + it("attaches the image to the user turn", () => { + const msgs = buildImageCaptionMessages({ + dataUri: "data:image/png;base64,AAAA", + }); + const parts = msgs[1]!.content as ChatContentPart[]; + assert.equal(Array.isArray(parts), true); + assert.equal(parts[1]!.type, "image_url"); + assert.equal( + (parts[1] as { image_url: { url: string } }).image_url.url, + "data:image/png;base64,AAAA", + ); + }); + + it("folds in the user's own caption to focus the description", () => { + const msgs = buildImageCaptionMessages({ + dataUri: "data:image/png;base64,AAAA", + userText: "这是我家猫", + }); + const parts = msgs[1]!.content as ChatContentPart[]; + assert.match((parts[0] as { text: string }).text, /这是我家猫/); + }); + + it("truncates an overlong user caption", () => { + const msgs = buildImageCaptionMessages({ + dataUri: "data:image/png;base64,AAAA", + userText: "猫".repeat(500), + }); + const text = (msgs[1]!.content as ChatContentPart[])[0] as { text: string }; + assert.ok(text.text.length < 400, `got ${text.text.length} chars`); + }); +}); + +describe("describeAttachments", () => { + it("returns bare text when nothing is attached", () => { + assert.equal(describeAttachments(" 你好 ", []), "你好"); + }); + + it("appends a tag so later turns still see the image happened", () => { + assert.equal(describeAttachments("看这个", [IMG]), "看这个\n[图片]"); + }); + + it("is never empty for a media-only message", () => { + assert.equal(describeAttachments("", [IMG]), "[图片]"); + assert.equal(describeAttachments(" ", [VOICE]), "[语音]"); + }); + + it("counts repeats", () => { + assert.equal(describeAttachments("", [IMG, IMG, VOICE]), "[图片×2][语音]"); + }); +}); + +describe("buildUserContent", () => { + it("stays a plain string when nothing is readable", () => { + assert.equal(buildUserContent("你好", []), "你好"); + // The kind tag is kept so this turn matches what history will show later. + assert.equal(buildUserContent("你好", [VIDEO]), "你好\n[视频]"); + }); + + it("falls back to the placeholder for an unreadable media-only message", () => { + assert.equal(buildUserContent("", [VOICE]), "[语音]"); + }); + + it("emits content parts when an image is readable", () => { + const content = buildUserContent("这是什么", [IMG]); + assert.ok(Array.isArray(content)); + const parts = content as ChatContentPart[]; + assert.equal(parts.length, 2); + assert.deepEqual(parts[0], { type: "text", text: "这是什么" }); + assert.deepEqual(parts[1], { + type: "image_url", + image_url: { url: "data:image/png;base64,AAAA" }, + }); + }); + + it("still leads with a text part when the user sent no caption", () => { + const parts = buildUserContent("", [IMG]) as ChatContentPart[]; + // Some providers reject a user turn that is images only. + assert.equal(parts[0]!.type, "text"); + assert.equal((parts[0] as { text: string }).text, "[图片]"); + }); + + it("includes only readable attachments as image parts", () => { + const parts = buildUserContent("看", [IMG, VIDEO, IMG]) as ChatContentPart[]; + assert.equal(parts.filter((p) => p.type === "image_url").length, 2); + }); +}); + +describe("buildChatMessages with attachments", () => { + const base = { + systemPrompt: "你是猫娘。", + memories: [], + history: [], + botName: "小铃", + multiBubbleJson: false, + }; + + it("injects the attachment block into the system prompt", () => { + const msgs = buildChatMessages({ + ...base, + userText: "这是什么", + attachments: [IMG], + }); + assert.match(flattenChatContent(msgs[0]!.content), /本条消息的附件/); + }); + + it("omits the block entirely for a plain text turn", () => { + const msgs = buildChatMessages({ ...base, userText: "你好" }); + assert.doesNotMatch(flattenChatContent(msgs[0]!.content), /本条消息的附件/); + assert.equal(msgs[msgs.length - 1]!.content, "你好"); + }); + + it("makes the final user turn multimodal", () => { + const msgs = buildChatMessages({ + ...base, + userText: "这是什么", + attachments: [IMG], + }); + const last = msgs[msgs.length - 1]!; + assert.equal(last.role, "user"); + assert.ok(Array.isArray(last.content)); + }); + + it("keeps the final user turn a string when nothing is readable", () => { + const msgs = buildChatMessages({ + ...base, + userText: "看看", + attachments: [VIDEO], + }); + assert.equal(msgs[msgs.length - 1]!.content, "看看\n[视频]"); + }); + + it("tags an unsent attachment even when another one is sent", () => { + const parts = buildUserContent("看看", [IMG, VIDEO]) as ChatContentPart[]; + // The image is right there; the video needs to be named or it vanishes. + assert.equal((parts[0] as { text: string }).text, "看看\n[视频]"); + assert.equal(parts.filter((p) => p.type === "image_url").length, 1); + }); +}); diff --git a/packages/core/src/chat-service.test.ts b/packages/core/src/chat-service.test.ts new file mode 100644 index 0000000..592bf2d --- /dev/null +++ b/packages/core/src/chat-service.test.ts @@ -0,0 +1,337 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + approvePeer, + listMemories, + listRecentMessages, + openDatabase, + replaceMemories, + seedPersonas, + setAssignment, + setPeerProactiveEnabled, + getPersonaBySlug, + upsertBotAccount, +} from "@wechat-ai/db"; +import type { LlmClient } from "@wechat-ai/llm"; +import { ChatService } from "./chat-service.js"; + +const redisUrl = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; + +class FakeLlm implements Pick { + calls = 0; + constructor(private reply: string | string[]) {} + private next(): string { + this.calls++; + if (typeof this.reply === "string") return this.reply; + const i = Math.min(this.calls - 1, this.reply.length - 1); + return this.reply[i] ?? ""; + } + async chat(): Promise { + return this.next(); + } + async chatWithUsage(_messages?: unknown, _opts?: unknown) { + return { + text: this.next(), + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + model: "fake", + }; + } +} + +function asLlm(fake: FakeLlm): LlmClient { + return fake as unknown as LlmClient; +} + +describe("ChatService multi-user isolation (Redis)", () => { + it("keeps separate memories per peer", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const botId = `bot_test_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u_test", + displayName: "test", + botToken: "test-token", + }); + + await approvePeer(db, botId, "user_a@im.wechat"); + await approvePeer(db, botId, "user_b@im.wechat"); + await setAssignment(db, botId, "user_a@im.wechat", cat.id); + await setAssignment(db, botId, "user_b@im.wechat", cat.id); + await replaceMemories(db, botId, "user_a@im.wechat", cat.id, ["A 喜欢草莓"]); + await replaceMemories(db, botId, "user_b@im.wechat", cat.id, ["B 喜欢蓝莓"]); + + const memA = await listMemories(db, botId, "user_a@im.wechat", cat.id); + const memB = await listMemories(db, botId, "user_b@im.wechat", cat.id); + assert.equal(memA[0]?.content, "A 喜欢草莓"); + assert.equal(memB[0]?.content, "B 喜欢蓝莓"); + + const chat = new ChatService(db, asLlm(new FakeLlm("喵～你好")), { + allowUnapproved: false, + memoryExtractEveryN: 999, + // Keep isolation test free of second-pass filter coupling + replyFilterEnabled: false, + }); + const r1 = await chat.handleInbound({ + botAccountId: botId, + peerId: "user_a@im.wechat", + text: "嗨", + contextToken: "tok-a", + }); + assert.equal(r1.kind, "reply"); + await db.close(); + }); + + it("handleInbound uses reply filter second LLM pass", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const botId = `bot_filter_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u_test", + displayName: "test", + botToken: "test-token", + }); + await approvePeer(db, botId, "user_f@im.wechat"); + await setAssignment(db, botId, "user_f@im.wechat", cat.id); + + const fake = new FakeLlm([ + "好呀～想你了 下次见哦", + JSON.stringify({ messages: ["好呀～", "想你了", "下次见哦"] }), + ]); + const chat = new ChatService(db, asLlm(fake), { + allowUnapproved: false, + memoryExtractEveryN: 999, + replyFilterEnabled: true, + stickersEnabled: false, + }); + const r = await chat.handleInbound({ + botAccountId: botId, + peerId: "user_f@im.wechat", + text: "嗨", + contextToken: "tok-f", + }); + assert.equal(r.kind, "reply"); + assert.equal(fake.calls, 2, "primary + filter LLM"); + assert.ok(r.parts && r.parts.length >= 2, JSON.stringify(r.parts)); + assert.equal(r.bubblesFromJson, true); + await db.close(); + }); + + it("handleProactive skips without writing user message", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const botId = `bot_proactive_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u_test", + displayName: "test", + botToken: "test-token", + }); + await approvePeer(db, botId, "user_p@im.wechat"); + await setAssignment(db, botId, "user_p@im.wechat", cat.id); + await setPeerProactiveEnabled(db, botId, "user_p@im.wechat", true); + + const chat = new ChatService( + db, + asLlm(new FakeLlm('{"skip":true,"reason":"quiet"}')), + { + allowUnapproved: false, + memoryExtractEveryN: 999, + replyFilterEnabled: true, + }, + ); + const r = await chat.handleProactive({ + botAccountId: botId, + peerId: "user_p@im.wechat", + contextToken: "tok-p", + idleHours: 14, + }); + assert.equal(r.kind, "skip"); + const hist = await listRecentMessages(db, botId, "user_p@im.wechat", 20); + assert.equal(hist.length, 0); + await db.close(); + }); + + it("handleProactive stores assistant reply", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const botId = `bot_proactive2_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u_test", + displayName: "test", + botToken: "test-token", + }); + await approvePeer(db, botId, "user_q@im.wechat"); + await setAssignment(db, botId, "user_q@im.wechat", cat.id); + await setPeerProactiveEnabled(db, botId, "user_q@im.wechat", true); + + const fake = new FakeLlm([ + "想你啦 在干嘛喵", + JSON.stringify({ messages: ["想你啦", "在干嘛喵"] }), + ]); + const chat = new ChatService(db, asLlm(fake), { + allowUnapproved: false, + memoryExtractEveryN: 999, + replyFilterEnabled: true, + stickersEnabled: false, + }); + const r = await chat.handleProactive({ + botAccountId: botId, + peerId: "user_q@im.wechat", + contextToken: "tok-q", + idleHours: 15, + }); + assert.equal(r.kind, "reply"); + assert.equal(fake.calls, 2); + assert.ok(r.parts && r.parts.length >= 1); + const hist = await listRecentMessages(db, botId, "user_q@im.wechat", 20); + assert.equal(hist.length, 1); + assert.equal(hist[0]?.role, "assistant"); + await db.close(); + }); + + it("handleInbound single-pass parses primary multi-bubble JSON (filter off)", async (t) => { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + await seedPersonas(db); + const cat = (await getPersonaBySlug(db, "catgirl"))!; + const botId = `bot_single_${Date.now()}`; + await upsertBotAccount(db, { + id: botId, + ownerUserId: "u_test", + displayName: "test", + botToken: "test-token", + }); + await approvePeer(db, botId, "user_s@im.wechat"); + await setAssignment(db, botId, "user_s@im.wechat", cat.id); + + const fake = new FakeLlm( + JSON.stringify({ + messages: [ + "给你看～", + { type: "sticker", slug: "wave" }, + "喜欢吗", + ], + }), + ); + const chat = new ChatService(db, asLlm(fake), { + allowUnapproved: false, + memoryExtractEveryN: 999, + // default path: no second-pass filter + replyFilterEnabled: false, + stickersEnabled: false, + }); + const r = await chat.handleInbound({ + botAccountId: botId, + peerId: "user_s@im.wechat", + text: "嗨", + contextToken: "tok-s", + }); + assert.equal(r.kind, "reply"); + assert.equal(fake.calls, 1, "primary LLM only"); + assert.equal(r.bubblesFromJson, true); + // stickersEnabled false → no catalog; dropDisallowedStickers drops unknown stickers + assert.ok(r.parts && r.parts.length >= 2, JSON.stringify(r.parts)); + assert.ok( + r.parts!.every((p) => p.kind === "text"), + "without sticker catalog, stickers must be dropped", + ); + assert.ok( + !JSON.stringify(r.parts).includes('"type":"sticker"') || + r.parts!.some((p) => p.kind === "text" && p.text.includes("给你看")), + ); + await db.close(); + }); +}); diff --git a/packages/core/src/chat-service.ts b/packages/core/src/chat-service.ts new file mode 100644 index 0000000..4d405c3 --- /dev/null +++ b/packages/core/src/chat-service.ts @@ -0,0 +1,968 @@ +import { + type Db, + type StickerPromptEntry, + clearMemories, + ensurePeer, + getBotAccount, + getPublishedGraph, + getPublishedPrompt, + getUser, + insertMessage, + listStickersForOwnerPrompt, + listMemories, + listRecentMessages, + recordTokenUsage, + replaceMemories, + resolvePersonaForPeer, + resolvePersonaUpstream, + touchPeerActivity, + touchPeerActivityFrom, + writeAudit, +} from "@wechat-ai/db"; +import { + LlmClient, + WebSearchClient, + type BuiltinToolName, + type ChatCallOptions, + type LlmUpstream, +} from "@wechat-ai/llm"; +import { + buildChatMessages, + buildImageCaptionMessages, + buildMemoryExtractMessages, + buildProactiveMessages, + describeAttachments, + parseFactsJson, + parseProactiveSkip, + type PromptAttachment, +} from "./prompt.js"; +import { + parseMultiBubbleReply, + type ReplyPart, +} from "./reply-format.js"; +import { dropDisallowedStickers, ReplyFilter } from "./reply-filter.js"; +import { + normalizeFactList, + selectMemoriesForPrompt, +} from "./memory-retrieve.js"; +import { ChatflowEngine } from "./chatflow/engine.js"; + +export interface ChatServiceOptions { + shortHistoryLimit: number; + memoryExtractEveryN: number; + allowUnapproved: boolean; + unapprovedReply: string; + noPersonaReply: string; + /** Ask model to return multi-bubble JSON (default true; ignored when replyFilterEnabled) */ + multiBubbleJson?: boolean; + maxReplyBubbles?: number; + /** Soft max chars per WeChat bubble when re-splitting (default 72) */ + maxChunkChars?: number; + /** Max stickers in one reply (default 2) */ + maxStickersPerReply?: number; + /** Inject sticker catalog into system prompt (default true) */ + stickersEnabled?: boolean; + /** + * Second-pass AI filter: reformat primary reply into multi-bubble JSON before send. + * When true, primary model does not receive REPLY_FORMAT_INSTRUCTION. + * Default false (primary model must emit multi-bubble JSON itself). + */ + replyFilterEnabled?: boolean; + /** Max memories injected when total exceeds fullInjectMax (default 12) */ + memoryTopK?: number; + /** Inject all memories when count ≤ this (default 20) */ + memoryFullInjectMax?: number; + /** Hard cap stored facts per peer+persona (default 100) */ + memoryMaxItems?: number; + /** Allow model to call get_current_time (default true) */ + timeToolEnabled?: boolean; + /** Default IANA timezone for get_current_time (default Asia/Shanghai) */ + timeToolTimeZone?: string; + /** + * Global web search switch. Persona must also have web_search_enabled. + * Search always goes through tools gateway (never main-site DDG). + */ + webSearchEnabled?: boolean; + /** Default result count for the web_search tool (default 5) */ + webSearchMaxResults?: number; + toolsBaseUrl?: string; + toolsApiKey?: string; + /** Wall clock for one tools-gateway search call */ + toolsTimeoutMs?: number; + /** Decrypt user custom LLM keys; required for persona llm_provider_id */ + llmProviderSecret?: string; + /** Host allowlist for chatflow http nodes (plus tools host) */ + chatflowHttpAllowHosts?: string[]; + chatflowMaxSteps?: number; + chatflowMaxNodes?: number; + /** + * How an attached image reaches the conversation. + * + * `caption` (default): describe it with `visionLlm` and pass only that text on, + * so the roleplay model needs no multimodal support. `direct`: hand the image + * to the roleplay model itself, which must then be vision-capable. + */ + visionMode?: "caption" | "direct"; + /** + * Model id for the vision call. Required for either mode to do anything — + * without it there is nothing that can read an image. + */ + visionModel?: string; + /** Cap on caption length */ + visionCaptionMaxTokens?: number; +} + +export interface InboundChatRequest { + botAccountId: string; + peerId: string; + text: string; + contextToken: string; + /** + * Media the user attached. Entries with a `dataUri` are handed to the model as + * content parts; the rest only appear in the attachment notice so the persona + * can say it cannot see them instead of inventing contents. + */ + attachments?: PromptAttachment[]; +} + +export interface ProactiveChatRequest { + botAccountId: string; + peerId: string; + contextToken: string; + /** Approximate idle hours for prompt (from scheduler) */ + idleHours: number; +} + +export interface InboundChatResult { + kind: "reply" | "reject" | "skip"; + /** Joined plain text (for reject or single-shot) */ + text?: string; + /** Ordered WeChat bubbles (roleplay reply) — legacy text view */ + bubbles?: string[]; + /** Structured parts for mixed text/sticker send */ + parts?: ReplyPart[]; + /** Whether bubbles came from model JSON */ + bubblesFromJson?: boolean; + personaId?: string; + personaSlug?: string; + /** Owner of the bot — saves the worker a second getBotAccount per message */ + ownerUserId?: string; + /** Present when kind=skip (proactive model declined) */ + skipReason?: string; +} + +const DEFAULTS: ChatServiceOptions = { + shortHistoryLimit: 20, + memoryExtractEveryN: 8, + allowUnapproved: false, + unapprovedReply: + "账号尚未开通对话权限。请前往网页端批准对话权限！\n（此项目为公益免费项目！使用文档：应用网址/docs）", + noPersonaReply: "系统尚未配置默认人设，请管理员先创建并设置默认角色。", + multiBubbleJson: true, + maxReplyBubbles: 5, + maxChunkChars: 72, + maxStickersPerReply: 2, + stickersEnabled: true, + replyFilterEnabled: false, + memoryTopK: 12, + memoryFullInjectMax: 20, + memoryMaxItems: 100, + timeToolEnabled: true, + timeToolTimeZone: "Asia/Shanghai", + webSearchEnabled: false, + webSearchMaxResults: 5, + visionMode: "caption", + visionCaptionMaxTokens: 300, +}; + +export class ChatService { + private opts: ChatServiceOptions; + private replyFilter: ReplyFilter; + private webSearch: WebSearchClient | null = null; + /** Identity of the tools config the current webSearch client was built from. */ + private webSearchKey = ""; + private chatflow: ChatflowEngine; + + constructor( + private db: Db, + private llm: LlmClient, + opts: Partial = {}, + /** + * Vision endpoint for reading images. Separate client because the roleplay + * model is usually text-only — see ChatServiceOptions.visionMode. + */ + private visionLlm: LlmClient | null = null, + ) { + this.opts = { ...DEFAULTS, ...opts }; + this.replyFilter = new ReplyFilter(llm, { + enabled: this.opts.replyFilterEnabled === true, + }); + this.syncWebSearch(); + this.chatflow = new ChatflowEngine({ + platformLlm: llm, + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + toolsTimeoutMs: this.opts.toolsTimeoutMs, + webSearchEnabled: this.opts.webSearchEnabled === true, + webSearchMaxResults: this.opts.webSearchMaxResults, + maxSteps: this.opts.chatflowMaxSteps ?? 32, + maxNodes: this.opts.chatflowMaxNodes ?? 40, + httpAllowHosts: this.opts.chatflowHttpAllowHosts, + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }); + } + + /** + * Rebuild the search client only when the tools config actually changed. + * + * Built lazily rather than once in the constructor so that turning + * WEB_SEARCH on (or filling in TOOLS_BASE_URL) from the admin panel does not + * leave a permanently search-less process behind. + */ + private syncWebSearch(): void { + const key = + this.opts.webSearchEnabled && this.opts.toolsBaseUrl + ? [ + this.opts.toolsBaseUrl, + this.opts.toolsApiKey ?? "", + this.opts.toolsTimeoutMs ?? "", + ].join("|") + : ""; + if (key === this.webSearchKey) return; + this.webSearchKey = key; + this.webSearch = key + ? new WebSearchClient({ + toolsBaseUrl: this.opts.toolsBaseUrl!, + toolsApiKey: this.opts.toolsApiKey || "", + timeoutMs: this.opts.toolsTimeoutMs, + }) + : null; + } + + /** + * Apply admin-editable settings in place (runtime settings reload). + * Only the keys present in `patch` are touched. + */ + applyRuntimeOptions(patch: Partial): void { + Object.assign(this.opts, patch); + if ("replyFilterEnabled" in patch) { + this.replyFilter.setEnabled(this.opts.replyFilterEnabled === true); + } + this.syncWebSearch(); + this.chatflow.applyOptions({ + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + toolsTimeoutMs: this.opts.toolsTimeoutMs, + webSearchEnabled: this.opts.webSearchEnabled === true, + webSearchMaxResults: this.opts.webSearchMaxResults, + maxSteps: this.opts.chatflowMaxSteps ?? 32, + maxNodes: this.opts.chatflowMaxNodes ?? 40, + httpAllowHosts: this.opts.chatflowHttpAllowHosts, + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }); + } + + /** + * Resolve LLM call options for a persona: + * - platform LLM: direct via this.llm + * - user custom: LlmClient.forUserUpstream → HF tools only + */ + private async resolveChatClient(params: { + persona: { llm_provider_id?: string | null; web_search_enabled?: number }; + ownerUserId?: string | null; + }): Promise<{ + client: LlmClient; + callOpts: ChatCallOptions; + }> { + const tools: BuiltinToolName[] = []; + if (this.opts.timeToolEnabled !== false) { + tools.push("get_current_time"); + } + const wantSearch = + this.opts.webSearchEnabled === true && + Boolean(params.persona.web_search_enabled) && + this.webSearch; + if (wantSearch) { + tools.push("web_search"); + } + + const callOpts: ChatCallOptions = { + tools, + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }; + if (wantSearch && this.webSearch) { + const ws = this.webSearch; + const defaultMax = this.opts.webSearchMaxResults ?? 5; + callOpts.webSearch = (query, maxResults) => + ws.searchAsToolResult(query, maxResults ?? defaultMax); + } + + const secret = this.opts.llmProviderSecret || ""; + let upstream: LlmUpstream | null = null; + if (secret && params.persona.llm_provider_id) { + const resolved = await resolvePersonaUpstream(this.db, { + llmProviderId: params.persona.llm_provider_id, + ownerUserId: params.ownerUserId, + secret, + }); + if (resolved) { + upstream = { + baseUrl: resolved.baseUrl, + apiKey: resolved.apiKey, + model: resolved.model, + }; + } + } + + if (upstream) { + if (!this.opts.toolsBaseUrl || !this.opts.toolsApiKey) { + throw new Error( + "User custom LLM requires TOOLS_BASE_URL and TOOLS_API_KEY (HF tools gateway)", + ); + } + const client = LlmClient.forUserUpstream({ + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + upstream, + }); + return { client, callOpts }; + } + + return { client: this.llm, callOpts }; + } + + /** When filter is on, primary model should not emit send-plan JSON. */ + private primaryMultiBubbleJson(): boolean { + if (this.opts.replyFilterEnabled === true) return false; + return this.opts.multiBubbleJson !== false; + } + + /** + * Second-pass filter (or legacy parse) → structured parts for WeChat send. + * Records filter token usage when the filter LLM ran. + */ + private async finalizeReplyParts(params: { + rawLlmText: string; + stickers: StickerPromptEntry[]; + botAccountId: string; + ownerUserId?: string | null; + ownerUsername?: string; + botName?: string | null; + }): Promise<{ + parts: ReplyPart[]; + bubbles: string[]; + displayText: string; + bubblesFromJson: boolean; + }> { + const maxBubbles = this.opts.maxReplyBubbles ?? 5; + const maxChunkChars = this.opts.maxChunkChars ?? 72; + const maxStickers = this.opts.maxStickersPerReply ?? 2; + const raw = (params.rawLlmText ?? "").trim(); + + if (this.opts.replyFilterEnabled === true) { + const filtered = await this.replyFilter.filter({ + rawText: raw, + allowedStickerSlugs: params.stickers.map((s) => s.slug), + maxBubbles, + maxChunkChars, + maxStickers, + }); + if (filtered.promptTokens > 0 || filtered.completionTokens > 0) { + await recordTokenUsage(this.db, { + userId: params.ownerUserId ?? undefined, + botId: params.botAccountId, + promptTokens: filtered.promptTokens, + completionTokens: filtered.completionTokens, + username: params.ownerUsername, + botName: params.botName ?? undefined, + }); + } + const parts = + filtered.parts.length > 0 + ? filtered.parts + : raw + ? [{ kind: "text" as const, text: raw }] + : []; + const bubbles = + filtered.bubbles.length > 0 + ? filtered.bubbles + : parts.map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ); + const displayText = + filtered.displayText || bubbles.join("\n") || raw; + return { + parts, + bubbles, + displayText, + bubblesFromJson: filtered.fromFilterJson, + }; + } + + const parsed = parseMultiBubbleReply(raw, { + maxBubbles, + maxChunkChars, + maxStickers, + fallbackSplit: true, + expandLongBubbles: true, + }); + const allowedSlugs = params.stickers.map((s) => s.slug); + // `raw` is only a safe fallback when it was not a recognised JSON envelope. + // Otherwise "the reply reduced to nothing" would be answered by sending + // `{"messages":[…]}` to the user verbatim — say nothing instead. + const rawFallback: ReplyPart[] = + !parsed.fromJson && raw ? [{ kind: "text" as const, text: raw }] : []; + let parts = parsed.parts.length > 0 ? parsed.parts : rawFallback; + // Drop invented / unknown sticker slugs (same rule as ReplyFilter path) + parts = dropDisallowedStickers(parts, allowedSlugs); + if (!parts.length) { + parts = rawFallback; + } + const bubbles = + parts.length > 0 + ? parts.map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ) + : raw + ? [raw] + : []; + const displayText = + parts.length > 0 + ? parts + .map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ) + .join("\n") + : parsed.displayText || bubbles.join("\n"); + return { + parts, + bubbles, + displayText, + bubblesFromJson: parsed.fromJson, + }; + } + + /** Swap in a vision client at runtime (settings reload). */ + setVisionClient(client: LlmClient | null): void { + this.visionLlm = client; + } + + /** + * Caption mode: describe each attached image with the vision endpoint and + * hand the roleplay model text instead of bytes. + * + * The returned attachments carry `caption` and have `dataUri` cleared — the + * roleplay model must not receive image parts it cannot parse. Failures + * degrade to a notice-only attachment rather than failing the turn: a flaky + * captioner should cost the user a description, not their reply. + * + * Token cost is billed to the bot owner like any other call. + */ + private async captionAttachments(params: { + attachments: PromptAttachment[]; + userText: string; + botAccountId: string; + ownerUserId?: string | null; + ownerUsername?: string; + botName?: string | null; + }): Promise { + const vision = this.visionLlm; + const model = this.opts.visionModel?.trim(); + if (!vision || !model) { + // Nothing can read the image — drop the bytes so the prompt tells the + // persona it cannot see it, instead of shipping an unusable data URI. + return params.attachments.map((a) => + a.dataUri ? { ...a, dataUri: undefined } : a, + ); + } + + let promptTokens = 0; + let completionTokens = 0; + const out = await Promise.all( + params.attachments.map(async (a) => { + if (!a.dataUri) return a; + try { + const res = await vision.chatWithUsage( + buildImageCaptionMessages({ + dataUri: a.dataUri, + userText: params.userText, + }), + { + model, + tools: [], + maxToolRounds: 0, + maxTokens: this.opts.visionCaptionMaxTokens, + }, + ); + promptTokens += res.promptTokens; + completionTokens += res.completionTokens; + const caption = res.text.trim(); + return { + ...a, + dataUri: undefined, + caption: caption || undefined, + } satisfies PromptAttachment; + } catch (err) { + console.error( + `[vision] caption failed bot=${params.botAccountId}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + return { ...a, dataUri: undefined }; + } + }), + ); + + if (promptTokens > 0 || completionTokens > 0) { + await recordTokenUsage(this.db, { + userId: params.ownerUserId ?? undefined, + botId: params.botAccountId, + promptTokens, + completionTokens, + username: params.ownerUsername, + botName: params.botName ?? undefined, + }); + } + return out; + } + + async handleInbound(req: InboundChatRequest): Promise { + const peer = await ensurePeer(this.db, req.botAccountId, req.peerId); + // NOTE: no upsertContextToken here — BotWorkerManager.handleJob already + // wrote the identical key/value, and it does so *before* the rate-limit + // early-return and the P2P intercept. Writing it again here would be a + // duplicate RTT on every message; removing the worker's copy instead would + // leave throttled / P2P peers with a stale token (breaking proactive and + // P2P delivery), so the worker keeps ownership of this write. + + if (!peer.approved && !this.opts.allowUnapproved) { + return { kind: "reject", text: this.opts.unapprovedReply }; + } + + if (/^\s*\/角色/.test(req.text)) { + return { + kind: "reply", + text: "角色切换仅支持机器人主人在后台分配，暂不支持用户自助 /角色 命令。", + }; + } + + const persona = await resolvePersonaForPeer( + this.db, + req.botAccountId, + req.peerId, + ); + if (!persona) { + return { kind: "reject", text: this.opts.noPersonaReply }; + } + + // Parallelize independent Redis reads (critical on remote Redis / Upstash) + const [systemPrompt, bot, history, memories] = await Promise.all([ + getPublishedPrompt(this.db, persona.id), + getBotAccount(this.db, req.botAccountId), + listRecentMessages( + this.db, + req.botAccountId, + req.peerId, + this.opts.shortHistoryLimit, + persona.id, + ), + listMemories(this.db, req.botAccountId, req.peerId, persona.id), + ]); + if (!systemPrompt) { + return { kind: "reject", text: this.opts.noPersonaReply }; + } + + const botName = bot?.display_name?.trim() || "助手"; + const owner = bot?.owner_user_id + ? await getUser(this.db, bot.owner_user_id) + : undefined; + + // Caption mode turns the image into text here, before anything else reads + // the attachments — so history, memory retrieval and the roleplay prompt all + // see the description rather than an opaque placeholder. + let attachments = req.attachments ?? []; + if ( + this.opts.visionMode !== "direct" && + attachments.some((a) => a.dataUri) + ) { + attachments = await this.captionAttachments({ + attachments, + userText: req.text, + botAccountId: req.botAccountId, + ownerUserId: bot?.owner_user_id, + ownerUsername: owner?.username, + botName: bot?.display_name, + }); + } + // The media bytes are never persisted, so this described text is the only + // trace a later turn can see. + const historyText = describeAttachments(req.text, attachments); + + const [userMsg, , stickers] = await Promise.all([ + insertMessage(this.db, { + botAccountId: req.botAccountId, + peerId: req.peerId, + personaId: persona.id, + role: "user", + content: historyText, + contextToken: req.contextToken, + }), + // `peer` is already loaded — skip the read half of the read-modify-write + touchPeerActivityFrom(this.db, peer), + this.opts.stickersEnabled === false || !bot?.owner_user_id + ? Promise.resolve([]) + : listStickersForOwnerPrompt(this.db, bot.owner_user_id), + ]); + // INCR reply from the insert above — no extra GET needed to decide on + // memory extraction further down. + const userMsgCount = userMsg.user_count ?? 0; + + const queryText = [ + historyText, + ...history + .slice(-4) + .map((m) => m.content) + .filter(Boolean), + ].join("\n"); + const selectedMemories = selectMemoriesForPrompt(memories, queryText, { + topK: this.opts.memoryTopK, + fullInjectMax: this.opts.memoryFullInjectMax, + }); + + // `owner` was already resolved above the caption step (it bills the caption + // call), so there is no second lookup here. + + let rawLlmText: string; + let promptTokens = 0; + let completionTokens = 0; + + if (persona.mode === "chatflow") { + // Chatflow MVP: proactive path still uses prompt mode elsewhere; + // inbound uses graph. Disable proactive separately at scheduler. + const graph = await getPublishedGraph(this.db, persona.id); + const secret = this.opts.llmProviderSecret || ""; + let upstream: LlmUpstream | null = null; + if (secret && persona.llm_provider_id) { + const resolved = await resolvePersonaUpstream(this.db, { + llmProviderId: persona.llm_provider_id, + ownerUserId: bot?.owner_user_id, + secret, + }); + if (resolved) { + upstream = { + baseUrl: resolved.baseUrl, + apiKey: resolved.apiKey, + model: resolved.model, + }; + } + } + const cf = await this.chatflow.run(graph, { + // The graph engine has no multimodal node, so a chatflow persona sees + // the `[图片]` placeholder rather than the image itself. + userText: historyText, + botName, + systemPrompt, + history: history.map((m) => ({ role: m.role, content: m.content })), + memories: selectedMemories.map((m) => m.content), + webSearchEnabled: Boolean(persona.web_search_enabled), + upstream, + }); + rawLlmText = cf.text; + promptTokens = cf.promptTokens; + completionTokens = cf.completionTokens; + } else { + const messages = buildChatMessages({ + systemPrompt, + memories: selectedMemories, + history, + userText: req.text, + botName, + multiBubbleJson: this.primaryMultiBubbleJson(), + stickers, + timeToolEnabled: this.opts.timeToolEnabled !== false, + attachments, + }); + + const { client: chatClient, callOpts } = await this.resolveChatClient({ + persona, + ownerUserId: bot?.owner_user_id, + }); + // Route the turn to the vision model only when the model actually gets an + // image; text turns must keep using the persona's normal model. + const visionModel = this.opts.visionModel?.trim(); + if (visionModel && attachments.some((a) => a.dataUri)) { + callOpts.model = visionModel; + } + const usage = await chatClient.chatWithUsage(messages, callOpts); + rawLlmText = usage.text; + promptTokens = usage.promptTokens; + completionTokens = usage.completionTokens; + } + + const finalized = await this.finalizeReplyParts({ + rawLlmText, + stickers, + botAccountId: req.botAccountId, + ownerUserId: bot?.owner_user_id, + ownerUsername: owner?.username, + botName: bot?.display_name, + }); + const { parts, bubbles, displayText, bubblesFromJson } = finalized; + + // Everything left is independent bookkeeping — one wave, not four. + // This runs between "model produced text" and "first bubble sent", so each + // serialized round trip here is latency the user feels. Still awaited: + // the next message on this peer reads history to build its prompt, so a + // floating insertMessage would drop this turn out of context. + await Promise.all([ + recordTokenUsage(this.db, { + userId: bot?.owner_user_id, + botId: req.botAccountId, + promptTokens, + completionTokens, + username: owner?.username, + botName: bot?.display_name, + }), + // Store plain bubbles text in history (never raw JSON wrapper) + insertMessage(this.db, { + botAccountId: req.botAccountId, + peerId: req.peerId, + personaId: persona.id, + role: "assistant", + content: displayText, + contextToken: req.contextToken, + }), + // Re-read here, unlike the pre-LLM touch: `peer` was loaded before the + // model call, so writing it back whole could revert an approve / + // proactive toggle the owner made during those seconds. + touchPeerActivity(this.db, req.botAccountId, req.peerId), + ]); + + if ( + userMsgCount > 0 && + userMsgCount % this.opts.memoryExtractEveryN === 0 + ) { + void this.extractMemory(req.botAccountId, req.peerId, persona.id).catch( + (err) => { + console.error("[memory] extract failed", err); + }, + ); + } + + return { + kind: "reply", + text: displayText, + bubbles, + parts, + bubblesFromJson, + personaId: persona.id, + personaSlug: persona.slug, + ownerUserId: bot?.owner_user_id, + }; + } + + /** + * Generate a proactive message when the peer has been idle. + * Does not insert a user message. May return kind=skip if the model declines. + */ + async handleProactive(req: ProactiveChatRequest): Promise { + const peer = await ensurePeer(this.db, req.botAccountId, req.peerId); + if (!peer.approved && !this.opts.allowUnapproved) { + return { kind: "reject", text: this.opts.unapprovedReply }; + } + if (!peer.proactive_enabled) { + return { kind: "skip", skipReason: "peer_off" }; + } + + const persona = await resolvePersonaForPeer( + this.db, + req.botAccountId, + req.peerId, + ); + if (!persona) { + return { kind: "reject", text: this.opts.noPersonaReply }; + } + // Chatflow MVP: proactive outreach only supported for classic prompt mode + if (persona.mode === "chatflow") { + return { kind: "skip", skipReason: "chatflow_no_proactive" }; + } + + const [systemPrompt, bot, history, memories] = await Promise.all([ + getPublishedPrompt(this.db, persona.id), + getBotAccount(this.db, req.botAccountId), + listRecentMessages( + this.db, + req.botAccountId, + req.peerId, + this.opts.shortHistoryLimit, + persona.id, + ), + listMemories(this.db, req.botAccountId, req.peerId, persona.id), + ]); + if (!systemPrompt) { + return { kind: "reject", text: this.opts.noPersonaReply }; + } + + const botName = bot?.display_name?.trim() || "助手"; + const stickers = + this.opts.stickersEnabled === false || !bot?.owner_user_id + ? [] + : await listStickersForOwnerPrompt(this.db, bot.owner_user_id); + + const proactiveQuery = history + .slice(-6) + .map((m) => m.content) + .filter(Boolean) + .join("\n"); + const selectedMemories = selectMemoriesForPrompt( + memories, + proactiveQuery || "主动联系", + { + topK: this.opts.memoryTopK, + fullInjectMax: this.opts.memoryFullInjectMax, + }, + ); + + const messages = buildProactiveMessages({ + systemPrompt, + memories: selectedMemories, + history, + idleHours: req.idleHours, + botName, + multiBubbleJson: this.primaryMultiBubbleJson(), + stickers, + timeToolEnabled: this.opts.timeToolEnabled !== false, + }); + + const { client: chatClient, callOpts } = await this.resolveChatClient({ + persona, + ownerUserId: bot?.owner_user_id, + }); + // Proactive: keep tools minimal (no web search spam) + callOpts.tools = (callOpts.tools || []).filter((t) => t !== "web_search"); + callOpts.webSearch = undefined; + const usage = await chatClient.chatWithUsage(messages, callOpts); + + const owner = bot?.owner_user_id + ? await getUser(this.db, bot.owner_user_id) + : undefined; + await recordTokenUsage(this.db, { + userId: bot?.owner_user_id, + botId: req.botAccountId, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + username: owner?.username, + botName: bot?.display_name, + }); + + const skip = parseProactiveSkip(usage.text); + if (skip.skip) { + return { + kind: "skip", + skipReason: skip.reason ?? "model_skip", + personaId: persona.id, + personaSlug: persona.slug, + }; + } + + const finalized = await this.finalizeReplyParts({ + rawLlmText: usage.text, + stickers, + botAccountId: req.botAccountId, + ownerUserId: bot?.owner_user_id, + ownerUsername: owner?.username, + botName: bot?.display_name, + }); + const parts = finalized.parts; + const bubbles = finalized.bubbles; + const displayText = finalized.displayText.trim(); + if (!displayText) { + return { + kind: "skip", + skipReason: "empty_reply", + personaId: persona.id, + personaSlug: persona.slug, + }; + } + + await insertMessage(this.db, { + botAccountId: req.botAccountId, + peerId: req.peerId, + personaId: persona.id, + role: "assistant", + content: displayText, + contextToken: req.contextToken, + }); + await touchPeerActivity(this.db, req.botAccountId, req.peerId); + + return { + kind: "reply", + text: displayText, + bubbles, + parts, + bubblesFromJson: finalized.bubblesFromJson, + personaId: persona.id, + personaSlug: persona.slug, + }; + } + + async extractMemory( + botAccountId: string, + peerId: string, + personaId: string, + ): Promise { + const history = await listRecentMessages( + this.db, + botAccountId, + peerId, + 30, + personaId, + ); + const existing = await listMemories( + this.db, + botAccountId, + peerId, + personaId, + ); + const msgs = buildMemoryExtractMessages({ history, existing }); + const raw = await this.llm.chatWithUsage(msgs); + const parsed = parseFactsJson(raw.text); + // Merge with existing so extraction can keep prior facts the model omitted + const merged = normalizeFactList( + [...existing.map((m) => m.content), ...parsed], + this.opts.memoryMaxItems ?? 100, + ); + const bot = await getBotAccount(this.db, botAccountId); + await recordTokenUsage(this.db, { + userId: bot?.owner_user_id, + botId: botAccountId, + promptTokens: raw.promptTokens, + completionTokens: raw.completionTokens, + botName: bot?.display_name, + }); + if (merged.length) { + await replaceMemories(this.db, botAccountId, peerId, personaId, merged, { + maxItems: this.opts.memoryMaxItems ?? 100, + }); + await writeAudit(this.db, "memory_extracted", "system", { + botAccountId, + peerId, + personaId, + count: merged.length, + }); + } + } + + async resetMemory( + botAccountId: string, + peerId: string, + personaId?: string, + ): Promise { + await clearMemories(this.db, botAccountId, peerId, personaId); + await writeAudit(this.db, "memory_reset", "admin", { + botAccountId, + peerId, + personaId, + }); + } +} diff --git a/packages/core/src/chatflow/default-graph.ts b/packages/core/src/chatflow/default-graph.ts new file mode 100644 index 0000000..a54cd1d --- /dev/null +++ b/packages/core/src/chatflow/default-graph.ts @@ -0,0 +1,39 @@ +import type { ChatflowGraph } from "./types.js"; + +/** Default: start → llm → answer */ +export function createDefaultChatflowGraph(): ChatflowGraph { + return { + version: 1, + nodes: [ + { + id: "start", + type: "start", + label: "开始", + data: {}, + }, + { + id: "llm", + type: "llm", + label: "LLM", + data: { + system: "{{system_prompt}}", + prompt: + "对话历史：\n{{history}}\n\n相关记忆：\n{{memories}}\n\n用户：{{query}}", + temperature: 0.8, + }, + }, + { + id: "answer", + type: "answer", + label: "回复", + data: { + answer: "{{llm.text}}", + }, + }, + ], + edges: [ + { id: "e1", source: "start", target: "llm" }, + { id: "e2", source: "llm", target: "answer" }, + ], + }; +} diff --git a/packages/core/src/chatflow/engine-http.test.ts b/packages/core/src/chatflow/engine-http.test.ts new file mode 100644 index 0000000..a0a1732 --- /dev/null +++ b/packages/core/src/chatflow/engine-http.test.ts @@ -0,0 +1,317 @@ +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import type { LlmClient } from "@wechat-ai/llm"; +import { ChatflowEngine } from "./engine.js"; +import { ChatflowError, type ChatflowGraph } from "./types.js"; + +/** + * The http node never touches the LLM, so a bare stub is enough — constructing + * a real LlmClient would drag credentials into a unit test for no gain. + */ +const stubLlm = {} as LlmClient; + +function httpGraph(url: string, extra: Record = {}): ChatflowGraph { + return { + version: 1, + nodes: [ + { id: "start", type: "start", data: {} }, + { id: "call", type: "http", data: { url, method: "GET", ...extra } }, + { id: "answer", type: "answer", data: { answer: "{{call.text}}" } }, + ], + edges: [ + { id: "e1", source: "start", target: "call" }, + { id: "e2", source: "call", target: "answer" }, + ], + }; +} + +const RUN_INPUT = { + userText: "hi", + botName: "bot", + systemPrompt: "sys", + history: [], + memories: [], +}; + +/** Records what each request looked like so header handling can be asserted. */ +interface Recorder { + server: http.Server; + port: number; + seen: Array<{ url: string; auth: string | undefined }>; +} + +async function startServer( + handler: (req: http.IncomingMessage, res: http.ServerResponse, rec: Recorder) => void, +): Promise { + const rec: Recorder = { server: null as unknown as http.Server, port: 0, seen: [] }; + rec.server = http.createServer((req, res) => { + rec.seen.push({ url: req.url ?? "", auth: req.headers.authorization }); + handler(req, res, rec); + }); + await new Promise((resolve) => rec.server.listen(0, "127.0.0.1", resolve)); + rec.port = (rec.server.address() as AddressInfo).port; + return rec; +} + +async function stopServer(rec: Recorder | undefined): Promise { + if (!rec) return; + await new Promise((resolve) => rec.server.close(() => resolve())); +} + +describe("chatflow http node: allowlist gating", () => { + it("blocks everything when the allowlist and tools host are both empty", async () => { + const engine = new ChatflowEngine({ platformLlm: stubLlm, httpAllowHosts: [] }); + await assert.rejects( + () => engine.run(httpGraph("https://api.example.com/x"), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.equal(err.code, "http_blocked"); + assert.match(err.message, /no TOOLS_BASE_URL/); + return true; + }, + ); + }); + + it("still refuses a host that is not listed", async () => { + const engine = new ChatflowEngine({ + platformLlm: stubLlm, + httpAllowHosts: ["api.allowed.com"], + }); + await assert.rejects( + () => engine.run(httpGraph("https://api.evil.com/x"), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.match(err.message, /not allowlisted/); + return true; + }, + ); + }); + + it("`*` does NOT open up internal space", async () => { + const engine = new ChatflowEngine({ platformLlm: stubLlm, httpAllowHosts: ["*"] }); + for (const target of [ + "http://169.254.169.254/latest/meta-data/iam/security-credentials/", + "http://100.100.100.200/latest/meta-data/", + "http://127.0.0.1:8000/admin", + "http://10.0.0.5/", + "http://192.168.1.1/", + "http://[::1]/", + "http://2130706433/", + "http://metadata.google.internal/computeMetadata/v1/", + "http://redis/", + ]) { + await assert.rejects( + () => engine.run(httpGraph(target), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError, `${target} should be a ChatflowError`); + assert.equal(err.code, "http_blocked", `${target} should be http_blocked`); + assert.match(err.message, /private host blocked/, target); + return true; + }, + `expected ${target} to be refused`, + ); + } + }); + + it("`*` still rejects non-http schemes", async () => { + const engine = new ChatflowEngine({ platformLlm: stubLlm, httpAllowHosts: ["*"] }); + await assert.rejects( + () => engine.run(httpGraph("file:///etc/passwd"), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.match(err.message, /only allows http/); + return true; + }, + ); + }); +}); + +describe("chatflow http node: redirects", () => { + let target: Recorder | undefined; + + before(async () => { + target = await startServer((req, res) => { + if (req.url === "/ok") { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("payload-body"); + return; + } + if (req.url === "/to-metadata") { + res.writeHead(302, { Location: "http://169.254.169.254/latest/meta-data/" }); + res.end(); + return; + } + if (req.url === "/to-loopback") { + res.writeHead(302, { Location: "http://127.0.0.1:9/" }); + res.end(); + return; + } + if (req.url === "/loop") { + res.writeHead(302, { Location: "/loop" }); + res.end(); + return; + } + res.writeHead(404); + res.end("nope"); + }); + }); + + after(async () => { + await stopServer(target); + }); + + /** The tools host is exempt, which lets a loopback test server stand in for an allowed origin. */ + function engineForLocal(): ChatflowEngine { + return new ChatflowEngine({ + platformLlm: stubLlm, + toolsBaseUrl: `http://127.0.0.1:${target!.port}`, + httpAllowHosts: ["*"], + }); + } + + it("fetches an allowed host and exposes the body to the flow", async () => { + const out = await engineForLocal().run( + httpGraph(`http://127.0.0.1:${target!.port}/ok`), + RUN_INPUT, + ); + assert.equal(out.text, "payload-body"); + }); + + it("refuses a redirect into cloud metadata", async () => { + // The whole point: the guard sees only the first URL unless hops are + // re-checked, so without redirect: "manual" this would succeed. + await assert.rejects( + () => + engineForLocal().run( + httpGraph(`http://127.0.0.1:${target!.port}/to-metadata`), + RUN_INPUT, + ), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.equal(err.code, "http_blocked"); + assert.match(err.message, /169\.254\.169\.254/); + return true; + }, + ); + }); + + it("refuses a redirect to a loopback port that is not the tools host", async () => { + await assert.rejects( + () => + engineForLocal().run( + httpGraph(`http://127.0.0.1:${target!.port}/to-loopback`), + RUN_INPUT, + ), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.equal(err.code, "http_blocked"); + return true; + }, + ); + }); + + it("caps a redirect loop instead of spinning", async () => { + await assert.rejects( + () => + engineForLocal().run( + httpGraph(`http://127.0.0.1:${target!.port}/loop`), + RUN_INPUT, + ), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.match(err.message, /exceeded \d+ redirects/); + return true; + }, + ); + }); +}); + +describe("chatflow http node: credential handling", () => { + let toolsSrv: Recorder | undefined; + let otherSrv: Recorder | undefined; + + before(async () => { + otherSrv = await startServer((_req, res) => { + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("other-origin"); + }); + toolsSrv = await startServer((req, res) => { + if (req.url === "/hop") { + res.writeHead(302, { Location: `http://127.0.0.1:${otherSrv!.port}/landed` }); + res.end(); + return; + } + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("tools-origin"); + }); + }); + + after(async () => { + await stopServer(toolsSrv); + await stopServer(otherSrv); + }); + + it("sends the tools key to the tools host", async () => { + const engine = new ChatflowEngine({ + platformLlm: stubLlm, + toolsBaseUrl: `http://127.0.0.1:${toolsSrv!.port}`, + toolsApiKey: "secret-tools-key", + httpAllowHosts: ["*"], + }); + await engine.run(httpGraph(`http://127.0.0.1:${toolsSrv!.port}/direct`), RUN_INPUT); + assert.equal(toolsSrv!.seen.at(-1)?.auth, "Bearer secret-tools-key"); + }); + + it("cannot be redirected off the tools origin to leak the key", async () => { + // Only the exact tools host:port is exempt from the internal-space check, + // so a hop to any other loopback port dies before a socket is opened — + // which is also why the second server must see no request at all. + // + // (fetchHttpNode additionally strips Authorization whenever the origin + // changes. That path only matters for public->public redirects, which a + // unit test cannot exercise without real network egress; the assertion + // below is the reachable half of the same invariant.) + const engine = new ChatflowEngine({ + platformLlm: stubLlm, + toolsBaseUrl: `http://127.0.0.1:${toolsSrv!.port}`, + toolsApiKey: "secret-tools-key", + httpAllowHosts: ["*"], + }); + await assert.rejects( + () => engine.run(httpGraph(`http://127.0.0.1:${toolsSrv!.port}/hop`), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.equal(err.code, "http_blocked"); + return true; + }, + ); + assert.deepEqual( + otherSrv!.seen, + [], + "the redirect target must never have been contacted", + ); + }); + + it("refuses a sibling port on the tools machine", async () => { + // The regression this pins: the exemption used to match on hostname only, + // so with a loopback TOOLS_BASE_URL every other local port — including + // this service's own API — was reachable through an http node. + const engine = new ChatflowEngine({ + platformLlm: stubLlm, + toolsBaseUrl: `http://127.0.0.1:${toolsSrv!.port}`, + toolsApiKey: "secret-tools-key", + httpAllowHosts: ["*"], + }); + await assert.rejects( + () => engine.run(httpGraph(`http://127.0.0.1:${otherSrv!.port}/landed`), RUN_INPUT), + (err: unknown) => { + assert.ok(err instanceof ChatflowError); + assert.equal(err.code, "http_blocked"); + assert.match(err.message, /loopback/); + return true; + }, + ); + }); +}); diff --git a/packages/core/src/chatflow/engine.ts b/packages/core/src/chatflow/engine.ts new file mode 100644 index 0000000..95eeff5 --- /dev/null +++ b/packages/core/src/chatflow/engine.ts @@ -0,0 +1,574 @@ +import { + LlmClient, + WebSearchClient, + type LlmUpstream, +} from "@wechat-ai/llm"; +import { createDefaultChatflowGraph } from "./default-graph.js"; +import { + ChatflowError, + type ChatflowGraph, + type ChatflowNodeBase, + type ChatflowRunInput, + type ChatflowRunResult, +} from "./types.js"; +import { + evalCondition, + renderTemplate, + validateChatflowGraph, +} from "./validate.js"; +import { + ALLOW_ANY_HOST, + allowsAnyHost, + blockedHostReason, + blockedResolvedReason, + normalizeHost, +} from "./http-guard.js"; + +/** Wall clock for one http node. Tools gateway calls are the only sanctioned target. */ +const HTTP_NODE_TIMEOUT_MS = Number( + process.env.CHATFLOW_HTTP_TIMEOUT_MS ?? "15000", +); + +/** Redirect hops an http node may take. Every hop is re-checked by the guard. */ +const HTTP_NODE_MAX_REDIRECTS = 3; + +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); + +export interface ChatflowEngineOptions { + /** Platform LLM client (admin path; also used when no upstream). */ + platformLlm: LlmClient; + toolsBaseUrl?: string; + toolsApiKey?: string; + webSearchEnabled?: boolean; + webSearchMaxResults?: number; + /** Wall clock for one tools-gateway search call */ + toolsTimeoutMs?: number; + maxSteps?: number; + maxNodes?: number; + /** + * Hostnames allowed for http nodes (default: host of toolsBaseUrl only). + * Prevents bypass of AI gateway via arbitrary HTTP. + */ + httpAllowHosts?: string[]; + timeZone?: string; +} + +export class ChatflowEngine { + private opts: Required< + Pick< + ChatflowEngineOptions, + | "maxSteps" + | "maxNodes" + | "webSearchEnabled" + | "webSearchMaxResults" + | "timeZone" + > + > & + ChatflowEngineOptions; + + constructor(opts: ChatflowEngineOptions) { + this.opts = { + ...opts, + maxSteps: opts.maxSteps ?? 32, + maxNodes: opts.maxNodes ?? 40, + webSearchEnabled: opts.webSearchEnabled === true, + webSearchMaxResults: opts.webSearchMaxResults ?? 5, + timeZone: opts.timeZone || "Asia/Shanghai", + }; + } + + /** + * Merge admin-editable options in place (runtime settings reload). + * `platformLlm` is intentionally not settable here — swap it via the + * owning service so every holder is updated together. + */ + applyOptions(patch: Partial>): void { + if (patch.toolsBaseUrl !== undefined) this.opts.toolsBaseUrl = patch.toolsBaseUrl; + if (patch.toolsApiKey !== undefined) this.opts.toolsApiKey = patch.toolsApiKey; + if (patch.toolsTimeoutMs !== undefined) { + this.opts.toolsTimeoutMs = patch.toolsTimeoutMs; + } + if (patch.webSearchEnabled !== undefined) { + this.opts.webSearchEnabled = patch.webSearchEnabled === true; + } + if (patch.webSearchMaxResults !== undefined) { + this.opts.webSearchMaxResults = patch.webSearchMaxResults; + } + if (patch.maxSteps !== undefined) this.opts.maxSteps = patch.maxSteps; + if (patch.maxNodes !== undefined) this.opts.maxNodes = patch.maxNodes; + if (patch.httpAllowHosts !== undefined) { + this.opts.httpAllowHosts = patch.httpAllowHosts; + } + if (patch.timeZone !== undefined) { + this.opts.timeZone = patch.timeZone || "Asia/Shanghai"; + } + } + + async run( + graphRaw: unknown | null | undefined, + input: ChatflowRunInput, + ): Promise { + const graph = + graphRaw && typeof graphRaw === "object" + ? (graphRaw as ChatflowGraph) + : createDefaultChatflowGraph(); + const validated = validateChatflowGraph(graph, { + maxNodes: this.opts.maxNodes, + }); + const { graph: g, startId } = validated; + const byId = new Map(g.nodes.map((n) => [n.id, n])); + + const historyText = (input.history || []) + .map((m) => `${m.role}: ${m.content}`) + .join("\n") + .slice(0, 12_000); + const memoriesText = (input.memories || []).join("\n") || "(无)"; + + const vars: Record = { + query: input.userText, + user_input: input.userText, + bot_name: input.botName, + system_prompt: input.systemPrompt, + history: historyText, + memories: memoriesText, + }; + + let promptTokens = 0; + let completionTokens = 0; + const trace: string[] = []; + let steps = 0; + let current: string | null = startId; + let finalAnswer: string | null = null; + + const outgoing = (nodeId: string, handle?: string | null) => { + const edges = g.edges.filter((e) => e.source === nodeId); + if (handle) { + const hit = edges.find( + (e) => (e.sourceHandle || "true") === handle, + ); + if (hit) return hit.target; + } + const def = edges.find((e) => !e.sourceHandle || e.sourceHandle === "default"); + if (def) return def.target; + return edges[0]?.target ?? null; + }; + + while (current) { + steps += 1; + if (steps > this.opts.maxSteps) { + throw new ChatflowError( + "max_steps", + `chatflow exceeded max steps (${this.opts.maxSteps})`, + ); + } + const node = byId.get(current); + if (!node) { + throw new ChatflowError("node", `missing node ${current}`); + } + trace.push(`enter:${node.type}:${node.id}`); + + if (node.type === "start") { + current = outgoing(node.id); + continue; + } + + if (node.type === "memory") { + // Re-expose memories (already selected by caller); optional kind filter later + vars[node.id] = { text: memoriesText, items: input.memories }; + vars.memories = memoriesText; + current = outgoing(node.id); + continue; + } + + if (node.type === "if-else") { + const cond = String( + (node.data?.condition as string) || + (node.data?.expr as string) || + "", + ); + const ok = evalCondition(renderTemplate(cond, vars), vars); + vars[node.id] = { result: ok }; + current = outgoing(node.id, ok ? "true" : "false"); + if (!current) { + current = outgoing(node.id, ok ? "false" : "true"); + } + continue; + } + + if (node.type === "search") { + const q = renderTemplate( + String((node.data?.query as string) || "{{query}}"), + vars, + ); + if (!this.opts.webSearchEnabled || !input.webSearchEnabled) { + throw new ChatflowError( + "search_disabled", + "web search disabled (persona or WEB_SEARCH_ENABLED)", + ); + } + if (!this.opts.toolsBaseUrl) { + throw new ChatflowError( + "search_disabled", + "TOOLS_BASE_URL required for search node", + ); + } + const client = new WebSearchClient({ + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey || "", + timeoutMs: this.opts.toolsTimeoutMs, + }); + const maxR = Number( + node.data?.max_results ?? this.opts.webSearchMaxResults, + ); + const hits = await client.search(q, maxR); + const text = hits + .map( + (h, i) => + `${i + 1}. ${h.title}\n${h.url}\n${h.snippet}`, + ) + .join("\n\n"); + vars[node.id] = { text, hits, query: q }; + vars.search = text; + current = outgoing(node.id); + continue; + } + + if (node.type === "http") { + const urlTpl = String((node.data?.url as string) || ""); + const url = renderTemplate(urlTpl, vars).trim(); + await this.assertHttpAllowed(url); + const method = String( + (node.data?.method as string) || "POST", + ).toUpperCase(); + const headersRaw = (node.data?.headers as Record) || {}; + const headers: Record = { + "Content-Type": "application/json", + "User-Agent": "WeChat-AI-Chatflow/1.0", + }; + for (const [k, v] of Object.entries(headersRaw)) { + headers[k] = renderTemplate(String(v), vars); + } + if (this.opts.toolsApiKey && !headers.Authorization) { + // Only inject when targeting the tools gateway itself — host AND + // port, so a different service on the same machine never sees the key. + try { + const host = new URL(url).host.toLowerCase(); + const toolsHostPort = this.toolsHostPort(); + if (toolsHostPort && host === toolsHostPort) { + headers.Authorization = `Bearer ${this.opts.toolsApiKey}`; + } + } catch { + /* ignore */ + } + } + let body: string | undefined; + if (method !== "GET" && method !== "HEAD") { + const bodyTpl = + (node.data?.body as string) || + JSON.stringify({ query: "{{query}}" }); + body = renderTemplate(bodyTpl, vars); + } + // Bounded: an unresponsive endpoint here would otherwise hold the + // reply-consumer slot and the bot:peer chain open indefinitely. + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), HTTP_NODE_TIMEOUT_MS); + let resp: Response; + let respText: string; + try { + resp = await this.fetchHttpNode( + url, + { method, headers, body }, + ctrl.signal, + ); + respText = await resp.text(); + } catch (err: unknown) { + // A guard rejection — a redirect into internal space, say — is a hard + // stop. Letting it fall through to http_error would turn a blocked + // SSRF attempt into a branch the flow can quietly carry on from. + if (err instanceof ChatflowError) throw err; + const message = ctrl.signal.aborted + ? `timeout after ${HTTP_NODE_TIMEOUT_MS}ms` + : err instanceof Error + ? err.message + : String(err); + vars[node.id] = { status: 0, text: `http_error: ${message}`, json: null }; + trace.push(`http_error:${message}`); + current = outgoing(node.id); + continue; + } finally { + clearTimeout(timer); + } + let json: unknown = null; + try { + json = JSON.parse(respText); + } catch { + json = null; + } + vars[node.id] = { + status: resp.status, + text: respText.slice(0, 50_000), + json, + }; + if (!resp.ok) { + trace.push(`http_error:${resp.status}`); + } + current = outgoing(node.id); + continue; + } + + if (node.type === "llm") { + const system = renderTemplate( + String( + (node.data?.system as string) || + (node.data?.system_prompt as string) || + "{{system_prompt}}", + ), + vars, + ); + const prompt = renderTemplate( + String( + (node.data?.prompt as string) || + (node.data?.user as string) || + "{{query}}", + ), + vars, + ); + const temperature = + typeof node.data?.temperature === "number" + ? node.data.temperature + : 0.8; + const client = this.resolveLlmClient(input.upstream ?? null); + const messages = [ + { role: "system" as const, content: system }, + { role: "user" as const, content: prompt }, + ]; + const usage = await client.chatWithUsage(messages, { + tools: [], + timeZone: this.opts.timeZone, + // force temperature via... LlmClient uses constructor temp; ok for MVP + }); + // Note: LlmClient temperature is fixed at construct; forUserUpstream uses defaults. + void temperature; + promptTokens += usage.promptTokens; + completionTokens += usage.completionTokens; + const text = usage.text.trim(); + vars[node.id] = { text, model: usage.model }; + vars.llm_text = text; + current = outgoing(node.id); + continue; + } + + if (node.type === "answer") { + const ans = renderTemplate( + String( + (node.data?.answer as string) || + (node.data?.text as string) || + "{{llm_text}}", + ), + vars, + ).trim(); + finalAnswer = ans || finalAnswer; + vars[node.id] = { text: ans }; + // Prefer first non-empty answer; stop graph + if (ans) break; + current = outgoing(node.id); + continue; + } + + throw new ChatflowError("node", `unsupported node type ${node.type}`); + } + + if (!finalAnswer?.trim()) { + throw new ChatflowError("no_answer", "chatflow produced no answer"); + } + + return { + text: finalAnswer.trim(), + trace, + steps, + promptTokens, + completionTokens, + }; + } + + private resolveLlmClient(upstream: LlmUpstream | null): LlmClient { + if (upstream) { + if (!this.opts.toolsBaseUrl || !this.opts.toolsApiKey) { + throw new ChatflowError( + "node", + "User custom LLM requires TOOLS_BASE_URL and TOOLS_API_KEY", + ); + } + return LlmClient.forUserUpstream({ + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + upstream, + }); + } + return this.opts.platformLlm; + } + + private toolsHost(): string | null { + const base = (this.opts.toolsBaseUrl || "").trim(); + if (!base) return null; + try { + return new URL(base).hostname.toLowerCase(); + } catch { + return null; + } + } + + /** + * Tools gateway host *including port*, for the checks where the port is the + * whole point. + * + * The internal-space exemption and the Authorization injection must both + * match on host+port, not hostname. A local tools container is normally + * TOOLS_BASE_URL=http://127.0.0.1:7860; matching on hostname alone would + * exempt every other port on loopback too — this service's own API among + * them — and would hand the tools key to whatever is listening there. + */ + private toolsHostPort(): string | null { + const base = (this.opts.toolsBaseUrl || "").trim(); + if (!base) return null; + try { + return new URL(base).host.toLowerCase(); + } catch { + return null; + } + } + + private async assertHttpAllowed(url: string): Promise { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new ChatflowError("http_blocked", "invalid http node URL"); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new ChatflowError("http_blocked", "http node only allows http(s)"); + } + // parsed.hostname already drops userinfo and normalises IP encodings, so + // http://allowed@169.254.169.254/ is judged on the address, not the label. + const host = normalizeHost(parsed.hostname); + const entries = (this.opts.httpAllowHosts || []) + .map((h) => h.trim().toLowerCase()) + .filter(Boolean); + const anyHost = allowsAnyHost(entries); + const allow = new Set( + entries.filter((h) => h !== ALLOW_ANY_HOST).map((h) => normalizeHost(h)), + ); + const toolsHost = this.toolsHost(); + if (toolsHost) allow.add(toolsHost); + if (!anyHost) { + // Always allow loopback tools during local dev if tools points there + if (!allow.size) { + throw new ChatflowError( + "http_blocked", + "http node blocked: no TOOLS_BASE_URL / CHATFLOW_HTTP_ALLOWLIST", + ); + } + if (!allow.has(host)) { + throw new ChatflowError( + "http_blocked", + `http node host not allowlisted: ${host}`, + ); + } + } + // The tools container is a deliberate config, loopback or not — but only + // that exact host:port, not every port on the same machine. + const toolsHostPort = this.toolsHostPort(); + if (toolsHostPort && parsed.host.toLowerCase() === toolsHostPort) return; + // Internal space stays blocked on both paths. With `*` this is the only + // barrier left, so it is a range check rather than a list of spellings, + // followed by a look at what the name actually resolves to. + const reason = + blockedHostReason(host) ?? (await blockedResolvedReason(host)); + if (reason) { + throw new ChatflowError( + "http_blocked", + `http node private host blocked: ${host} (${reason})`, + ); + } + } + + /** + * Fetch an http node's URL, re-checking every redirect hop. + * + * `fetch` defaults to redirect: "follow", which would let an allowlisted + * host bounce the request to 169.254.169.254 — the guard only ever sees the + * first URL. So hops are taken manually and each one goes back through + * assertHttpAllowed. + */ + private async fetchHttpNode( + startUrl: string, + init: { method: string; headers: Record; body?: string }, + signal: AbortSignal, + ): Promise { + let url = startUrl; + let method = init.method; + let body = init.body; + let headers = { ...init.headers }; + + for (let hop = 0; ; hop++) { + await this.assertHttpAllowed(url); + const resp = await fetch(url, { + method, + headers, + body, + signal, + redirect: "manual", + }); + if (!REDIRECT_STATUSES.has(resp.status)) return resp; + + const location = resp.headers.get("location"); + if (!location) return resp; + if (hop >= HTTP_NODE_MAX_REDIRECTS) { + throw new ChatflowError( + "http_blocked", + `http node exceeded ${HTTP_NODE_MAX_REDIRECTS} redirects`, + ); + } + + let next: URL; + try { + next = new URL(location, url); + } catch { + throw new ChatflowError( + "http_blocked", + `http node redirect to invalid URL: ${location}`, + ); + } + // Never carry credentials across an origin change — the Authorization + // header is only ever injected for the tools host. + if (next.host !== new URL(url).host) delete headers.Authorization; + // Mirror fetch's own method rewriting: 303 always downgrades to GET, + // and 301/302 do so for anything that had a body. + if (resp.status === 303 || (body !== undefined && resp.status !== 307 && resp.status !== 308)) { + method = "GET"; + body = undefined; + const { "Content-Type": _ct, ...rest } = headers; + headers = rest; + } + url = next.toString(); + } + } +} + +export function isChatflowGraph(raw: unknown): raw is ChatflowGraph { + try { + validateChatflowGraph(raw); + return true; + } catch { + return false; + } +} + +export function summarizeGraph(graph: ChatflowGraph | null | undefined): { + nodeCount: number; + types: string[]; +} { + if (!graph?.nodes?.length) return { nodeCount: 0, types: [] }; + const types = [...new Set(graph.nodes.map((n: ChatflowNodeBase) => n.type))]; + return { nodeCount: graph.nodes.length, types }; +} diff --git a/packages/core/src/chatflow/http-guard.test.ts b/packages/core/src/chatflow/http-guard.test.ts new file mode 100644 index 0000000..da75468 --- /dev/null +++ b/packages/core/src/chatflow/http-guard.test.ts @@ -0,0 +1,289 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + ALLOW_ANY_HOST, + allowsAnyHost, + blockedHostReason, + blockedResolvedReason, + ipReason, + normalizeHost, +} from "./http-guard.js"; + +/** Reads better than `!== null` at every call site. */ +function blocked(host: string): boolean { + return blockedHostReason(host) !== null; +} + +describe("chatflow http guard: public hosts", () => { + it("allows ordinary public names and addresses", () => { + for (const host of [ + "api.openai.com", + "example.com", + "sub.domain.example.co.uk", + "1.1.1.1", + "8.8.8.8", + "93.184.216.34", + "wechat.smnet-ai.asia", + "2c2ch1u11-share-api-0.hf.space", + // 172.x outside 16-31 is public + "172.15.0.1", + "172.32.0.1", + // 100.x outside the CGNAT block is public + "100.63.255.255", + "100.128.0.1", + // octal-looking but normalises to 8.0.0.1 + "010.0.0.1", + "[2606:4700::1111]", + ]) { + assert.equal(blockedHostReason(host), null, `expected allowed: ${host}`); + } + }); +}); + +describe("chatflow http guard: loopback and private ranges", () => { + it("blocks loopback in every encoding the URL parser emits", () => { + // The old string check only knew the literal "127.0.0.1". + for (const host of [ + "127.0.0.1", + "127.0.0.2", + "127.1.2.3", + "2130706433", + "0x7f.0.0.1", + "0177.0.0.1", + "127.1", + "localhost", + "LOCALHOST", + "[::1]", + "[::ffff:127.0.0.1]", + "[::ffff:7f00:1]", + "[64:ff9b::7f00:1]", + ]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("blocks RFC1918 space", () => { + for (const host of [ + "10.0.0.1", + "10.255.255.255", + "172.16.0.1", + "172.31.255.255", + "192.168.0.1", + "192.168.1.1", + ]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("blocks the unspecified address", () => { + for (const host of ["0.0.0.0", "0", "[::]"]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); +}); + +describe("chatflow http guard: cloud metadata", () => { + it("blocks the link-local metadata endpoints", () => { + // AWS / Azure IMDS, and GCP addressed by IP. None of these is a string the + // old blacklist matched. + for (const host of ["169.254.169.254", "169.254.170.2", "169.254.0.1"]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + assert.match(blockedHostReason("169.254.169.254")!, /metadata|link-local/); + }); + + it("blocks Alibaba Cloud metadata inside the CGNAT block", () => { + assert.ok(blocked("100.100.100.200")); + assert.ok(blocked("100.64.0.1")); + assert.ok(blocked("100.127.255.255")); + }); + + it("blocks metadata hostnames including the GCP one", () => { + for (const host of [ + "metadata", + "metadata.google.internal", + "anything.internal", + ]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); +}); + +describe("chatflow http guard: intranet names", () => { + it("blocks single-label hosts such as docker service names", () => { + for (const host of ["wechat-ai-tools", "redis", "db", "kubelet"]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("blocks internal suffixes, trailing dot and case included", () => { + for (const host of [ + "printer.local", + "EVIL.LOCAL.", + "host.localhost", + "svc.cluster.internal", + "nas.home.arpa", + "fileserver.lan", + "wiki.corp", + ]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("blocks IPv6 unique-local and link-local", () => { + for (const host of ["[fd00::1]", "[fc00::1]", "[fe80::1]", "[ff02::1]"]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("blocks hosts it cannot parse rather than passing them through", () => { + for (const host of ["", " ", "a b", "[not:an:ip", "%%%"]) { + assert.ok(blocked(host), `expected blocked: ${JSON.stringify(host)}`); + } + }); +}); + +describe("chatflow http guard: gaps found by the SSRF audit", () => { + it("blocks all of fe00::/8, not just fe80::/10", () => { + // Regression: the mask `(b[1] & 0xc0) === 0x80` matched fe80-febf only, so + // site-local fec0::/10 and the unassigned fe00-fe7f block passed through. + for (const host of [ + "[fe00::1]", + "[fe40::1]", + "[fe7f::1]", + "[fec0::1]", + "[fedf::1]", + "[feff:ffff::1]", + "[fe80::1]", + ]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + }); + + it("default-denies IPv6 outside global unicast 2000::/3", () => { + for (const host of ["[100::1]", "[0100::1]", "[4000::1]", "[1000::1]"]) { + assert.ok(blocked(host), `expected blocked: ${host}`); + } + // ...while real global unicast stays reachable. + for (const host of ["[2606:4700::1111]", "[2001:4860:4860::8888]", "[3fff::1]"]) { + assert.equal(blockedHostReason(host), null, `expected allowed: ${host}`); + } + }); + + it("blocks cloud metadata endpoints that use a public-looking name", () => { + // metadata.tencentyun.com is neither an internal suffix nor an IP range, + // and it serves CVM role credentials without a token step. + assert.ok(blocked("metadata.tencentyun.com")); + assert.ok(blocked("metadata.goog")); + assert.ok(blocked("instance-data")); + }); + + it("blocks Kubernetes .svc names", () => { + assert.ok(blocked("kubernetes.default.svc")); + assert.ok(blocked("redis.default.svc")); + }); + + it("ipReason judges v4 and v6 literals identically to the host check", () => { + assert.match(ipReason("127.0.0.1")!, /loopback/); + assert.match(ipReason("169.254.169.254")!, /metadata|link-local/); + assert.match(ipReason("100.100.100.200")!, /metadata|NAT/); + assert.match(ipReason("fec0::1")!, /fe00::\/8/); + assert.equal(ipReason("8.8.8.8"), null); + assert.equal(ipReason("2606:4700::1111"), null); + assert.equal(ipReason("not-an-ip"), null); + }); +}); + +describe("chatflow http guard: DNS resolution", () => { + /** Fixed answers — a real resolver would make these tests measure the network. */ + const fakeDns = (map: Record) => async (host: string) => { + const hit = map[host]; + if (!hit) throw new Error("ENOTFOUND"); + return hit; + }; + + it("blocks a public name that resolves into internal space", async () => { + // Lexically these are ordinary .io names; only the answer gives them away. + // nip.io-style wildcard resolvers need no attacker infrastructure at all. + const dns = fakeDns({ + "169-254-169-254.nip.io": ["169.254.169.254"], + "100-100-100-200.nip.io": ["100.100.100.200"], + "evil.example.com": ["10.0.0.5"], + "wechat-ai-tools.wechat-ai_default": ["172.17.0.3"], + }); + + const imds = await blockedResolvedReason("169-254-169-254.nip.io", dns); + assert.match(imds!, /169\.254\.169\.254/); + assert.match(imds!, /link-local|metadata/); + + assert.match( + (await blockedResolvedReason("100-100-100-200.nip.io", dns))!, + /100\.100\.100\.200/, + ); + assert.match( + (await blockedResolvedReason("evil.example.com", dns))!, + /10\.0\.0\.5/, + ); + // Dotted Docker service name — the single-label rule cannot see this one. + assert.match( + (await blockedResolvedReason("wechat-ai-tools.wechat-ai_default", dns))!, + /172\.17\.0\.3/, + ); + }); + + it("blocks when ANY answer is internal, not just the first", async () => { + const dns = fakeDns({ "split.example.com": ["93.184.216.34", "10.1.2.3"] }); + assert.match( + (await blockedResolvedReason("split.example.com", dns))!, + /10\.1\.2\.3/, + ); + }); + + it("blocks an IPv6 answer in internal space", async () => { + const dns = fakeDns({ "v6.example.com": ["fec0::1"] }); + assert.match((await blockedResolvedReason("v6.example.com", dns))!, /fe00::\/8/); + }); + + it("allows a name that resolves to public addresses", async () => { + const dns = fakeDns({ "api.example.com": ["93.184.216.34", "2606:4700::1111"] }); + assert.equal(await blockedResolvedReason("api.example.com", dns), null); + }); + + it("does not turn an unresolvable name into a block", async () => { + // Nothing can egress to a name that will not resolve, and a DNS blip must + // not abort the whole flow with a hard http_blocked. + const dns = fakeDns({}); + assert.equal(await blockedResolvedReason("nope.example.com", dns), null); + }); + + it("skips resolution for literals, which were already judged", async () => { + let called = false; + const dns = async (_h: string) => { + called = true; + return ["10.0.0.1"]; + }; + assert.equal(await blockedResolvedReason("8.8.8.8", dns), null); + assert.equal(await blockedResolvedReason("127.0.0.1", dns), null); + assert.equal(await blockedResolvedReason("[::1]", dns), null); + assert.equal(called, false, "literals must not hit the resolver"); + }); +}); + +describe("chatflow http guard: helpers", () => { + it("normalizeHost unbrackets, lowercases and drops the trailing dot", () => { + assert.equal(normalizeHost(" EXAMPLE.COM. "), "example.com"); + assert.equal(normalizeHost("[::1]"), "::1"); + assert.equal(normalizeHost("evil.local..."), "evil.local"); + }); + + it("allowsAnyHost only fires on the sentinel", () => { + assert.equal(allowsAnyHost([ALLOW_ANY_HOST]), true); + assert.equal(allowsAnyHost([" * "]), true); + assert.equal(allowsAnyHost(["api.example.com", "*"]), true); + assert.equal(allowsAnyHost(["api.example.com"]), false); + assert.equal(allowsAnyHost([]), false); + assert.equal(allowsAnyHost(undefined), false); + // Not a wildcard matcher — only the bare sentinel opens things up. + assert.equal(allowsAnyHost(["*.example.com"]), false); + }); +}); diff --git a/packages/core/src/chatflow/http-guard.ts b/packages/core/src/chatflow/http-guard.ts new file mode 100644 index 0000000..d25ea9a --- /dev/null +++ b/packages/core/src/chatflow/http-guard.ts @@ -0,0 +1,306 @@ +import { lookup as dnsLookup } from "node:dns/promises"; + +/** + * Egress guard for chatflow `http` nodes. + * + * Split out of engine.ts because this is the only thing standing between a + * user-authored URL and a server-side fetch: chatflow graphs are written by + * persona owners (any registered user — see the PUT graph route in + * apps/api/src/routes.ts), and the http node's response body is fed back into + * the flow's vars, so an unguarded fetch is a *readable* SSRF primitive, not a + * blind one. Worth unit-testing on its own. + * + * The old guard compared the hostname against a handful of literal strings + * ("localhost", "127.0.0.1", "0.0.0.0", "*.local"). That is sound only while an + * exact-match allowlist does the real work. Once CHATFLOW_HTTP_ALLOWLIST is + * opened up with `*`, this file becomes the barrier, so it checks address + * ranges instead of spellings — 169.254.169.254 and 100.100.100.200 are the + * addresses that actually matter and neither is a string the old list caught. + */ + +/** + * Allowlist entry meaning "any public host". Cannot collide with a real + * hostname: `*` is not a legal DNS label, and WHATWG URL parsing rejects it in + * an authority, so no reachable host can ever equal this sentinel. + */ +export const ALLOW_ANY_HOST = "*"; + +/** + * Suffixes that never resolve to anything public. `.internal` covers GCP's + * metadata.google.internal; `.local` is mDNS; the rest are conventional + * intranet suffixes that are not delegated TLDs. + */ +const INTERNAL_SUFFIXES = [ + ".local", + ".localhost", + ".internal", + ".home.arpa", + ".lan", + ".intranet", + ".corp", + ".private", + // Kubernetes service DNS (kubernetes.default.svc, svc.cluster.local). + ".svc", +]; + +/** + * Metadata endpoints that answer on a *public-looking* name, so neither the IP + * ranges nor the internal suffixes catch them. Tencent's is the one that + * matters here — this deployment is China-facing, and metadata.tencentyun.com + * hands out CVM role credentials with no token step. + */ +const METADATA_HOSTS = new Set([ + "metadata.tencentyun.com", + "metadata.google.internal", + "metadata.goog", + "instance-data", +]); + +/** + * Lowercase, unbracket, and drop the trailing dot. + * + * All three matter: WHATWG returns IPv6 hosts wrapped in brackets + * (`new URL("http://[::1]/").hostname === "[::1]"`), and "evil.local." is the + * same name to DNS but not to endsWith(".local"). + */ +export function normalizeHost(raw: string): string { + let h = (raw ?? "").trim().toLowerCase(); + if (h.startsWith("[") && h.endsWith("]")) h = h.slice(1, -1); + while (h.endsWith(".")) h = h.slice(0, -1); + return h; +} + +/** + * Re-run a bare hostname through WHATWG URL parsing. + * + * This is what collapses the classic alternate IPv4 encodings — 2130706433, + * 0x7f.0.0.1, 0177.1 all normalise to dotted-quad 127.x — so the range checks + * below only ever have to understand one form. Reusing the platform parser + * beats hand-rolling inet_aton. Returns null when the host is unparseable, + * which callers must treat as "block". + */ +function canonicalizeHost(host: string): string | null { + if (!host) return null; + try { + const probe = new URL(`http://${host.includes(":") && !host.startsWith("[") ? `[${host}]` : host}/`); + return normalizeHost(probe.hostname); + } catch { + return null; + } +} + +/** Dotted-quad only — the sole IPv4 form WHATWG URL emits. */ +function parseIpv4(host: string): number[] | null { + const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host); + if (!m) return null; + const parts = [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])]; + return parts.some((n) => n > 255) ? null : parts; +} + +/** Returns 16 bytes, expanding `::` and any embedded IPv4 tail. */ +function parseIpv6(host: string): number[] | null { + if (!host.includes(":")) return null; + // A zone id has no business in a URL; refuse rather than guess. + if (host.includes("%")) return null; + + const halves = host.split("::"); + if (halves.length > 2) return null; + + const toGroups = (s: string): number[] | null => { + if (!s) return []; + const out: number[] = []; + for (const part of s.split(":")) { + if (part === "") return null; + if (part.includes(".")) { + const v4 = parseIpv4(part); + if (!v4) return null; + out.push((v4[0]! << 8) | v4[1]!, (v4[2]! << 8) | v4[3]!); + continue; + } + if (!/^[0-9a-f]{1,4}$/.test(part)) return null; + out.push(parseInt(part, 16)); + } + return out; + }; + + const head = toGroups(halves[0] ?? ""); + const tail = halves.length === 2 ? toGroups(halves[1] ?? "") : []; + if (!head || !tail) return null; + + let groups: number[]; + if (halves.length === 2) { + const fill = 8 - head.length - tail.length; + if (fill < 1) return null; + groups = [...head, ...new Array(fill).fill(0), ...tail]; + } else { + groups = head; + } + if (groups.length !== 8) return null; + + const bytes: number[] = []; + for (const g of groups) bytes.push((g >> 8) & 0xff, g & 0xff); + return bytes; +} + +/** + * Non-public IPv4 space. Ordered roughly by how likely each is to be the thing + * an attacker actually wants. + */ +function ipv4Reason(b: number[]): string | null { + const [a, c] = [b[0]!, b[1]!]; + if (a === 127) return "loopback"; + if (a === 10) return "private range 10/8"; + if (a === 172 && c >= 16 && c <= 31) return "private range 172.16/12"; + if (a === 192 && c === 168) return "private range 192.168/16"; + // 169.254.169.254 (AWS/Azure IMDS, GCP by IP) lives here. + if (a === 169 && c === 254) return "link-local / cloud metadata"; + // 100.64/10 is CGNAT and also holds Alibaba Cloud's 100.100.100.200. + if (a === 100 && c >= 64 && c <= 127) return "carrier-grade NAT / cloud metadata"; + if (a === 0) return "unspecified range 0/8"; + if (a === 192 && c === 0) return "IETF reserved 192.0/16"; + if (a === 198 && (c === 18 || c === 19)) return "benchmarking range"; + if (a >= 224) return "multicast / reserved"; + return null; +} + +function ipv6Reason(b: number[]): string | null { + const zeros = (upTo: number) => b.slice(0, upTo).every((x) => x === 0); + + // ::ffff:a.b.c.d — judge the embedded IPv4, or ::ffff:7f00:1 slips through. + if (zeros(10) && b[10] === 0xff && b[11] === 0xff) { + return ipv4Reason(b.slice(12)) ?? null; + } + // NAT64 well-known prefix 64:ff9b::/96 also embeds IPv4. + if ( + b[0] === 0x00 && b[1] === 0x64 && b[2] === 0xff && b[3] === 0x9b && + b.slice(4, 12).every((x) => x === 0) + ) { + return ipv4Reason(b.slice(12)) ?? null; + } + if (zeros(12)) { + const last = b.slice(12); + if (last.every((x) => x === 0)) return "unspecified address"; + if (last[0] === 0 && last[1] === 0 && last[2] === 0 && last[3] === 1) { + return "loopback"; + } + // IPv4-compatible ::a.b.c.d + return ipv4Reason(last) ?? null; + } + if ((b[0]! & 0xfe) === 0xfc) return "unique-local fc00::/7"; + // All of fe00::/8, not just fe80::/10: the old `(b[1] & 0xc0) === 0x80` mask + // matched fe80-febf only, letting site-local fec0::/10 and the unassigned + // fe00-fe7f block straight through. None of fe00::/8 is globally routable. + if (b[0] === 0xfe) return "link-local / site-local fe00::/8"; + if (b[0] === 0xff) return "multicast ff00::/8"; + // Default-deny anything outside global unicast 2000::/3. Every publicly + // routable IPv6 address lives there, so this closes the whole long tail of + // reserved and special-purpose prefixes without enumerating them. + if ((b[0]! & 0xe0) !== 0x20) return "outside global unicast 2000::/3"; + return null; +} + +/** + * Range verdict for one already-literal address, v4 or v6. + * + * Shared by the lexical check and the post-resolution check so both judge + * addresses by exactly the same rules. + */ +export function ipReason(addr: string): string | null { + const host = normalizeHost(addr); + const v4 = parseIpv4(host); + if (v4) return ipv4Reason(v4); + const v6 = parseIpv6(host); + if (v6) return ipv6Reason(v6); + return null; +} + +/** + * Why this host must not be fetched, or null when it looks publicly routable. + * + * Lexical only, and deliberately so: it stays synchronous and cheap, and it + * refuses every *direct* attempt at loopback, RFC1918 and the metadata + * endpoints in each encoding the URL parser can produce. Names that merely + * resolve somewhere internal are not its job — pair it with + * blockedResolvedReason, which does the looking. + */ +export function blockedHostReason(rawHost: string): string | null { + const host = canonicalizeHost(normalizeHost(rawHost)); + if (!host) return "unparseable host"; + + const v4 = parseIpv4(host); + if (v4) return ipv4Reason(v4); + + const v6 = parseIpv6(host); + if (v6) return ipv6Reason(v6); + + if (host === "localhost") return "loopback"; + if (METADATA_HOSTS.has(host)) return "cloud metadata hostname"; + // A public name needs a dot. Single labels are Docker service names, bare + // intranet hostnames, and "metadata" — never anything routable. + if (!host.includes(".")) return "single-label / intranet hostname"; + for (const suffix of INTERNAL_SUFFIXES) { + if (host.endsWith(suffix)) return `internal suffix ${suffix}`; + } + return null; +} + +/** + * Resolve the name and judge what it actually points at. + * + * The lexical pass above cannot see through DNS, and that is where the real + * evasions live: `169-254-169-254.nip.io` is a perfectly ordinary `.io` name + * that resolves to the IMDS address, an attacker's own A record can point at + * 10.0.0.5, and Docker's embedded DNS answers `service.network` with a bridge + * address. All three are lexically indistinguishable from a public host, so the + * only way to refuse them is to look. + * + * A resolution failure is deliberately NOT an error: if the name does not + * resolve here it will not resolve for fetch either, so nothing can egress, and + * turning transient DNS trouble into a hard `http_blocked` would abort the whole + * flow over a blip. + * + * Residual gap: this resolves, then fetch resolves again, so an attacker who + * controls authoritative DNS with a very short TTL can still rebind between the + * two lookups. Closing that needs the socket pinned to the address that was + * actually validated — an undici Agent with a custom `connect.lookup`, which + * means taking on undici as a dependency. Documented in docs/chatflow.md. + */ +export async function blockedResolvedReason( + rawHost: string, + resolve: HostResolver = defaultResolver, +): Promise { + const host = canonicalizeHost(normalizeHost(rawHost)); + if (!host) return "unparseable host"; + // Literals were already judged directly; resolving them adds nothing. + if (parseIpv4(host) || parseIpv6(host)) return null; + + let addresses: string[]; + try { + addresses = await resolve(host); + } catch { + return null; + } + + for (const address of addresses) { + const reason = ipReason(address); + if (reason) return `${host} resolves to ${address} (${reason})`; + } + return null; +} + +/** + * Injectable so the range logic can be tested without a network. Real DNS in a + * test would measure the resolver, not the code — some sandboxes answer every + * query with a placeholder address. + */ +export type HostResolver = (host: string) => Promise; + +const defaultResolver: HostResolver = async (host) => { + const found = await dnsLookup(host, { all: true, verbatim: true }); + return found.map((f) => f.address); +}; + +/** True when the allowlist entries opt into "any public host". */ +export function allowsAnyHost(entries: readonly string[] | undefined): boolean { + return (entries ?? []).some((e) => e.trim() === ALLOW_ANY_HOST); +} diff --git a/packages/core/src/chatflow/types.ts b/packages/core/src/chatflow/types.ts new file mode 100644 index 0000000..bc916f2 --- /dev/null +++ b/packages/core/src/chatflow/types.ts @@ -0,0 +1,79 @@ +/** Chatflow graph types (Dify-like MVP). */ + +export type ChatflowNodeType = + | "start" + | "llm" + | "answer" + | "if-else" + | "http" + | "memory" + | "search"; + +export interface ChatflowNodeBase { + id: string; + type: ChatflowNodeType; + /** Display label in UI */ + label?: string; + data?: Record; +} + +export interface ChatflowEdge { + id: string; + source: string; + target: string; + /** For if-else: "true" | "false" | omit for default */ + sourceHandle?: string | null; +} + +export interface ChatflowGraph { + version: 1; + nodes: ChatflowNodeBase[]; + edges: ChatflowEdge[]; +} + +export interface ChatflowRunInput { + userText: string; + botName: string; + systemPrompt: string; + /** Recent history as role/content pairs */ + history: Array<{ role: string; content: string }>; + /** Memory facts already selected for this turn */ + memories: string[]; + /** + * When true, persona allows search nodes / search tools. + * Still requires global webSearch + tools gateway on engine deps. + */ + webSearchEnabled?: boolean; + /** Persona llm_provider_id resolved upstream (via tools only). */ + upstream?: { + baseUrl: string; + apiKey: string; + model: string; + } | null; +} + +export interface ChatflowRunResult { + text: string; + /** Ordered intermediate logs for debugging (not sent to WeChat) */ + trace: string[]; + steps: number; + promptTokens: number; + completionTokens: number; +} + +export class ChatflowError extends Error { + constructor( + public code: + | "invalid_graph" + | "max_steps" + | "no_answer" + | "http_blocked" + | "search_disabled" + | "node" + | "timeout", + message: string, + ) { + super(message); + this.name = "ChatflowError"; + } +} diff --git a/packages/core/src/chatflow/validate.test.ts b/packages/core/src/chatflow/validate.test.ts new file mode 100644 index 0000000..96f97e0 --- /dev/null +++ b/packages/core/src/chatflow/validate.test.ts @@ -0,0 +1,65 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { createDefaultChatflowGraph } from "./default-graph.js"; +import { + evalCondition, + renderTemplate, + validateChatflowGraph, +} from "./validate.js"; +import { ChatflowError } from "./types.js"; + +describe("chatflow validate", () => { + it("accepts default graph", () => { + const r = validateChatflowGraph(createDefaultChatflowGraph()); + assert.equal(r.ok, true); + assert.equal(r.startId, "start"); + assert.ok(r.answerIds.includes("answer")); + }); + + it("rejects missing answer", () => { + assert.throws( + () => + validateChatflowGraph({ + version: 1, + nodes: [{ id: "start", type: "start" }], + edges: [], + }), + (e: unknown) => e instanceof ChatflowError && e.code === "invalid_graph", + ); + }); + + it("rejects duplicate start", () => { + assert.throws( + () => + validateChatflowGraph({ + version: 1, + nodes: [ + { id: "s1", type: "start" }, + { id: "s2", type: "start" }, + { id: "a", type: "answer" }, + ], + edges: [], + }), + ChatflowError, + ); + }); +}); + +describe("chatflow templates", () => { + it("renders nested keys", () => { + const out = renderTemplate("hi {{user.name}} / {{query}}", { + query: "Q", + user: { name: "Ada" }, + }); + assert.equal(out, "hi Ada / Q"); + }); + + it("evalCondition equality and contains", () => { + const vars = { x: "hello", n: 1 }; + assert.equal(evalCondition('x == "hello"', vars), true); + assert.equal(evalCondition('x != "hello"', vars), false); + assert.equal(evalCondition("x contains ell", vars), true); + assert.equal(evalCondition("empty missing", vars), true); + assert.equal(evalCondition("not empty x", vars), true); + }); +}); diff --git a/packages/core/src/chatflow/validate.ts b/packages/core/src/chatflow/validate.ts new file mode 100644 index 0000000..516b7c0 --- /dev/null +++ b/packages/core/src/chatflow/validate.ts @@ -0,0 +1,202 @@ +import type { ChatflowEdge, ChatflowGraph, ChatflowNodeBase } from "./types.js"; +import { ChatflowError } from "./types.js"; + +const NODE_TYPES = new Set([ + "start", + "llm", + "answer", + "if-else", + "http", + "memory", + "search", +]); + +export interface ValidateGraphOptions { + maxNodes?: number; +} + +export interface ValidateGraphResult { + ok: true; + graph: ChatflowGraph; + startId: string; + answerIds: string[]; +} + +/** + * Structural validation for chatflow graphs. + * Throws ChatflowError on failure. + */ +export function validateChatflowGraph( + raw: unknown, + opts: ValidateGraphOptions = {}, +): ValidateGraphResult { + const maxNodes = Math.max(3, Math.min(200, opts.maxNodes ?? 40)); + if (!raw || typeof raw !== "object") { + throw new ChatflowError("invalid_graph", "graph must be an object"); + } + const g = raw as ChatflowGraph; + if (g.version !== 1) { + throw new ChatflowError("invalid_graph", "graph.version must be 1"); + } + if (!Array.isArray(g.nodes) || !Array.isArray(g.edges)) { + throw new ChatflowError("invalid_graph", "nodes and edges required"); + } + if (g.nodes.length > maxNodes) { + throw new ChatflowError( + "invalid_graph", + `too many nodes (max ${maxNodes})`, + ); + } + if (!g.nodes.length) { + throw new ChatflowError("invalid_graph", "graph has no nodes"); + } + + const ids = new Set(); + for (const n of g.nodes) { + if (!n || typeof n !== "object") { + throw new ChatflowError("invalid_graph", "invalid node"); + } + const id = String((n as ChatflowNodeBase).id || "").trim(); + const type = (n as ChatflowNodeBase).type; + if (!id) throw new ChatflowError("invalid_graph", "node id required"); + if (ids.has(id)) { + throw new ChatflowError("invalid_graph", `duplicate node id: ${id}`); + } + if (!NODE_TYPES.has(type)) { + throw new ChatflowError("invalid_graph", `unknown node type: ${type}`); + } + ids.add(id); + } + + const starts = g.nodes.filter((n) => n.type === "start"); + if (starts.length !== 1) { + throw new ChatflowError("invalid_graph", "exactly one start node required"); + } + const answers = g.nodes.filter((n) => n.type === "answer"); + if (!answers.length) { + throw new ChatflowError("invalid_graph", "at least one answer node required"); + } + + for (const e of g.edges) { + if (!e || typeof e !== "object") { + throw new ChatflowError("invalid_graph", "invalid edge"); + } + const edge = e as ChatflowEdge; + if (!ids.has(edge.source) || !ids.has(edge.target)) { + throw new ChatflowError( + "invalid_graph", + `edge ${edge.id || "?"} references missing node`, + ); + } + } + + return { + ok: true, + graph: { + version: 1, + nodes: g.nodes.map((n) => ({ + id: String(n.id).trim(), + type: n.type, + label: n.label, + data: n.data && typeof n.data === "object" ? n.data : {}, + })), + edges: g.edges.map((e, i) => ({ + id: String(e.id || `e${i}`), + source: e.source, + target: e.target, + sourceHandle: e.sourceHandle ?? null, + })), + }, + startId: starts[0]!.id, + answerIds: answers.map((a) => a.id), + }; +} + +/** Simple mustache-like {{path}} with dotted keys against a flat+nested vars bag. */ +export function renderTemplate( + template: string, + vars: Record, +): string { + if (!template) return ""; + return template.replace(/\{\{\s*([^}]+?)\s*\}\}/g, (_, rawKey: string) => { + const key = String(rawKey).trim(); + if (!key) return ""; + const val = lookupVar(vars, key); + if (val === undefined || val === null) return ""; + if (typeof val === "string") return val; + if (typeof val === "number" || typeof val === "boolean") return String(val); + try { + return JSON.stringify(val); + } catch { + return String(val); + } + }); +} + +function lookupVar(vars: Record, path: string): unknown { + if (Object.prototype.hasOwnProperty.call(vars, path)) { + return vars[path]; + } + const parts = path.split("."); + let cur: unknown = vars; + for (const p of parts) { + if (cur == null || typeof cur !== "object") return undefined; + cur = (cur as Record)[p]; + } + return cur; +} + +/** + * Evaluate a simple condition expression against vars. + * Supports: var, var == "x", var != "x", var contains "x", empty var, not empty var + */ +export function evalCondition( + expr: string, + vars: Record, +): boolean { + const s = (expr || "").trim(); + if (!s) return false; + const emptyM = /^empty\s+(.+)$/i.exec(s); + if (emptyM) { + const v = lookupVar(vars, emptyM[1]!.trim()); + return v === undefined || v === null || v === ""; + } + const notEmptyM = /^(?:not\s+empty|!empty)\s+(.+)$/i.exec(s); + if (notEmptyM) { + const v = lookupVar(vars, notEmptyM[1]!.trim()); + return !(v === undefined || v === null || v === ""); + } + const containsM = /^(.+?)\s+contains\s+(.+)$/i.exec(s); + if (containsM) { + const left = String(lookupVar(vars, containsM[1]!.trim()) ?? ""); + const right = unquote(containsM[2]!.trim(), vars); + return left.includes(right); + } + const eqM = /^(.+?)\s*(==|!=)\s*(.+)$/.exec(s); + if (eqM) { + const left = String(lookupVar(vars, eqM[1]!.trim()) ?? ""); + const right = unquote(eqM[3]!.trim(), vars); + return eqM[2] === "==" ? left === right : left !== right; + } + // bare truthiness + const v = lookupVar(vars, s); + if (typeof v === "boolean") return v; + if (typeof v === "number") return v !== 0; + if (typeof v === "string") return v.length > 0; + return Boolean(v); +} + +function unquote(token: string, vars: Record): string { + if ( + (token.startsWith('"') && token.endsWith('"')) || + (token.startsWith("'") && token.endsWith("'")) + ) { + return token.slice(1, -1); + } + if (token.startsWith("{{") && token.endsWith("}}")) { + return String(lookupVar(vars, token.slice(2, -2).trim()) ?? ""); + } + const v = lookupVar(vars, token); + if (v !== undefined) return String(v ?? ""); + return token; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts new file mode 100644 index 0000000..70549e9 --- /dev/null +++ b/packages/core/src/index.ts @@ -0,0 +1,113 @@ +export { + ChatService, + type ChatServiceOptions, + type InboundChatRequest, + type InboundChatResult, + type ProactiveChatRequest, +} from "./chat-service.js"; +export { + applyPromptTemplate, + buildAttachmentBlock, + buildBotIdentityBlock, + buildChatMessages, + buildImageCaptionMessages, + buildMemoryExtractMessages, + buildProactiveInstruction, + buildProactiveMessages, + buildUserContent, + describeAttachments, + BOT_NAME_VARS, + parseFactsJson, + parseProactiveSkip, + type AttachmentKind, + type PromptAttachment, +} from "./prompt.js"; +export { + hourInTimeZone, + isInQuietHours, + isProactiveEligible, + mergeBotProactiveConfig, + parseQuietHours, + resolveActivityMs, + type ProactiveEligibilityInput, + type ProactiveEligibilityResult, + type ProactivePolicy, + type ProactiveSkipReason, +} from "./proactive.js"; +export { + splitReplyIntoBubbles, + humanDelayMs, + type SplitReplyOptions, + type HumanDelayOptions, +} from "./split-reply.js"; +export { + REPLY_FORMAT_INSTRUCTION, + REPLY_FORMAT_INSTRUCTION_TEXT_ONLY, + hasStickerTextToken, + parseMultiBubbleReply, + partsToDisplayText, + partsToLegacyBubbles, + renderAssistantHistoryForModel, + sanitizePartsStripStickerJson, + stripAllStickerJson, + type ParsedBubbles, + type ReplyPart, +} from "./reply-format.js"; +export { + ReplyFilter, + REPLY_FILTER_SYSTEM_PROMPT, + buildReplyFilterMessages, + dropDisallowedStickers, + type ReplyFilterInput, + type ReplyFilterResult, + type ReplyFilterOptions, +} from "./reply-filter.js"; +export { buildStickerCatalogBlock } from "./prompt.js"; +export { + selectMemoriesForPrompt, + normalizeFactList, + extractTerms, + scoreMemoryAgainstQuery, + type MemoryRetrieveOptions, +} from "./memory-retrieve.js"; +export { + P2PService, + parseAtUsername, + isP2PCommand, + type P2PServiceOptions, + type P2PInboundRequest, + type P2PHandleResult, + type P2PRemoteSend, +} from "./p2p-service.js"; +export { + TryChatService, + TryChatError, + type TryChatServiceOptions, + type StartTrySessionInput, + type StartTrySessionResult, + type SendTryMessageInput, + type SendTryMessageResult, +} from "./try-chat-service.js"; +export { + ChatflowEngine, + isChatflowGraph, + summarizeGraph, + type ChatflowEngineOptions, +} from "./chatflow/engine.js"; +export { + createDefaultChatflowGraph, +} from "./chatflow/default-graph.js"; +export { + validateChatflowGraph, + renderTemplate, + evalCondition, +} from "./chatflow/validate.js"; +export { + ChatflowError, + type ChatflowGraph, + type ChatflowNodeType, + type ChatflowNodeBase, + type ChatflowEdge, + type ChatflowRunInput, + type ChatflowRunResult, +} from "./chatflow/types.js"; diff --git a/packages/core/src/memory-retrieve.test.ts b/packages/core/src/memory-retrieve.test.ts new file mode 100644 index 0000000..a380878 --- /dev/null +++ b/packages/core/src/memory-retrieve.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import type { MemoryRow } from "@wechat-ai/db"; +import { + extractTerms, + normalizeFactList, + selectMemoriesForPrompt, +} from "./memory-retrieve.js"; + +function mem(content: string, id = content): MemoryRow { + return { + id, + bot_account_id: "b", + peer_id: "p", + persona_id: "per", + kind: "fact", + content, + }; +} + +describe("memory-retrieve", () => { + it("extractTerms finds CJK bigrams and latin words", () => { + const t = extractTerms("用户叫小明 likes coffee"); + assert.ok(t.includes("coffee")); + assert.ok(t.some((x) => x.includes("小明") || x === "小明")); + }); + + it("selectMemoriesForPrompt injects all when under fullInjectMax", () => { + const list = [mem("A"), mem("B"), mem("C")]; + const out = selectMemoriesForPrompt(list, "无关", { + topK: 2, + fullInjectMax: 20, + }); + assert.equal(out.length, 3); + }); + + it("selectMemoriesForPrompt picks relevant top-K when over fullInjectMax", () => { + const list = Array.from({ length: 25 }, (_, i) => + mem(`无关事实编号${i}`, `id${i}`), + ); + list.push(mem("用户的名字是小明", "name")); + list.push(mem("用户喜欢草莓", "fruit")); + const out = selectMemoriesForPrompt(list, "我叫什么名字？小明", { + topK: 5, + fullInjectMax: 10, + }); + assert.ok(out.length <= 5); + assert.ok(out.some((m) => m.content.includes("小明"))); + }); + + it("normalizeFactList dedupes and drops substrings", () => { + const out = normalizeFactList( + ["用户喜欢咖啡", "用户喜欢咖啡", "喜欢咖啡", "用户叫小红", " "], + 100, + ); + assert.ok(out.some((f) => f.includes("喜欢咖啡"))); + assert.equal(out.filter((f) => f.includes("咖啡")).length, 1); + assert.ok(out.some((f) => f.includes("小红"))); + }); + + it("normalizeFactList respects maxItems", () => { + const facts = Array.from({ length: 30 }, (_, i) => `事实${i}`); + const out = normalizeFactList(facts, 10); + assert.equal(out.length, 10); + }); +}); diff --git a/packages/core/src/memory-retrieve.ts b/packages/core/src/memory-retrieve.ts new file mode 100644 index 0000000..8bbb94b --- /dev/null +++ b/packages/core/src/memory-retrieve.ts @@ -0,0 +1,165 @@ +import type { MemoryRow } from "@wechat-ai/db"; + +export interface MemoryRetrieveOptions { + /** Inject at most this many memories when count exceeds fullInjectMax (default 12) */ + topK?: number; + /** If total memories ≤ this, inject all without scoring (default 20) */ + fullInjectMax?: number; +} + +/** + * Normalize Chinese/English text for crude token overlap scoring. + * No external tokenizer / embedding API. + */ +export function normalizeForScore(text: string): string { + return (text ?? "") + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]/gu, " ") + .replace(/\s+/g, " ") + .trim(); +} + +/** Extract overlapping terms: CJK bigrams + latin words (≥2 chars). */ +export function extractTerms(text: string): string[] { + const n = normalizeForScore(text); + if (!n) return []; + const terms = new Set(); + // latin / digit words + for (const m of n.match(/[a-z0-9]{2,}/g) ?? []) { + terms.add(m); + } + // CJK runs → bigrams + unigrams for short runs + const cjkRuns = n.match(/[㐀-鿿豈-﫿]+/g) ?? []; + for (const run of cjkRuns) { + if (run.length === 1) { + terms.add(run); + continue; + } + for (let i = 0; i < run.length - 1; i++) { + terms.add(run.slice(i, i + 2)); + } + // also keep full short phrases (≤6) for exact-ish match boost + if (run.length <= 6) terms.add(run); + } + return [...terms]; +} + +export function scoreMemoryAgainstQuery( + memoryContent: string, + queryTerms: Set, + queryNorm: string, +): number { + const content = memoryContent ?? ""; + if (!content.trim() || !queryTerms.size) return 0; + const memNorm = normalizeForScore(content); + if (!memNorm) return 0; + + let score = 0; + // substring boost when query chunk appears in memory or vice versa + if (queryNorm.length >= 2 && memNorm.includes(queryNorm)) { + score += 8; + } else if (memNorm.length >= 2 && queryNorm.includes(memNorm)) { + score += 5; + } + + const memTerms = extractTerms(content); + for (const t of memTerms) { + if (queryTerms.has(t)) { + // longer terms (bigrams / words) weigh more + score += t.length >= 2 ? 2 : 1; + } + } + return score; +} + +/** + * Select memories for prompt injection. + * - ≤ fullInjectMax → all (stable order) + * - else → top-K by text overlap with query; ties keep original order (newer-ish list order) + */ +export function selectMemoriesForPrompt( + memories: MemoryRow[], + queryText: string, + opts: MemoryRetrieveOptions = {}, +): MemoryRow[] { + const topK = Math.max(1, opts.topK ?? 12); + const fullInjectMax = Math.max(0, opts.fullInjectMax ?? 20); + if (!memories.length) return []; + if (memories.length <= fullInjectMax) return memories; + + const queryNorm = normalizeForScore(queryText); + const queryTerms = new Set(extractTerms(queryText)); + + // If query empty, keep most recent-looking tail (list is typically chronological push) + if (!queryTerms.size) { + return memories.slice(-topK); + } + + const ranked = memories + .map((m, index) => ({ + m, + index, + score: scoreMemoryAgainstQuery(m.content, queryTerms, queryNorm), + })) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + // prefer later entries on tie (often newer) + return b.index - a.index; + }); + + // Always include at least some high-score ones; if all zero, take tail + const best = ranked[0]?.score ?? 0; + if (best <= 0) { + return memories.slice(-topK); + } + return ranked.slice(0, topK).map((r) => r.m); +} + +/** + * Merge / dedupe fact strings for storage. + * - trim, drop empty + * - drop exact duplicates (case-insensitive) + * - drop facts that are strict substrings of a longer fact + * - cap at maxItems (keep last / longer preferred order of input) + */ +export function normalizeFactList( + facts: string[], + maxItems = 100, +): string[] { + const cleaned = facts + .map((f) => (typeof f === "string" ? f.trim() : "")) + .filter(Boolean); + + // Prefer longer facts first when checking containment + const byLen = [...cleaned].sort((a, b) => b.length - a.length); + const kept: string[] = []; + const lowerKept: string[] = []; + + for (const f of byLen) { + const low = f.toLowerCase(); + if (lowerKept.some((k) => k === low)) continue; + // drop if this is substring of an already kept longer fact + if (lowerKept.some((k) => k.includes(low) && k !== low)) continue; + // if a shorter kept fact is substring of this, remove the shorter one + for (let i = kept.length - 1; i >= 0; i--) { + if (low.includes(lowerKept[i]) && low !== lowerKept[i]) { + kept.splice(i, 1); + lowerKept.splice(i, 1); + } + } + kept.push(f); + lowerKept.push(low); + } + + // Restore roughly input order among survivors + const order = new Map(cleaned.map((f, i) => [f.toLowerCase(), i])); + kept.sort( + (a, b) => + (order.get(a.toLowerCase()) ?? 0) - (order.get(b.toLowerCase()) ?? 0), + ); + + const max = Math.max(1, maxItems); + if (kept.length <= max) return kept; + // keep the most recent-ish (end of list) when over cap + return kept.slice(-max); +} diff --git a/packages/core/src/p2p-service.test.ts b/packages/core/src/p2p-service.test.ts new file mode 100644 index 0000000..0f0975f --- /dev/null +++ b/packages/core/src/p2p-service.test.ts @@ -0,0 +1,314 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + createBindCode, + openDatabase, + saveBotCredentials, + setPrimaryBind, + upsertBotAccount, + upsertContextToken, + upsertUser, + nowIso, +} from "@wechat-ai/db"; +import { P2PService, isP2PCommand, parseAtUsername } from "./p2p-service.js"; + +const redisUrl = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; + +describe("P2P parse helpers", () => { + it("parses whole-message @username only", () => { + assert.equal(parseAtUsername("@alice"), "alice"); + assert.equal(parseAtUsername(" @Bob_1 "), "Bob_1"); + assert.equal(parseAtUsername("hello @alice"), null); + assert.equal(parseAtUsername("@alice hi"), null); + }); + + it("detects p2p commands", () => { + assert.equal(isP2PCommand("/绑定 ABC123"), true); + assert.equal(isP2PCommand("/同意"), true); + assert.equal(isP2PCommand("@user"), true); + assert.equal(isP2PCommand("普通聊天"), false); + }); +}); + +describe("P2PService (Redis)", () => { + async function withDb( + t: { skip: (msg?: string) => void }, + fn: (db: ReturnType) => Promise, + ) { + let db; + try { + db = openDatabase(redisUrl); + await Promise.race([ + db.ping(), + new Promise((_, rej) => + setTimeout(() => rej(new Error("timeout")), 2500), + ), + ]); + } catch { + try { + await db?.close(); + } catch { + /* ignore */ + } + t.skip("Redis not available"); + return; + } + try { + await fn(db); + } finally { + await db.close(); + } + } + + it("bind code once + whoami + unbind", async (t) => { + await withDb(t, async (db) => { + const suffix = Date.now().toString(36); + const userId = `u_p2p_${suffix}`; + const botId = `bot_p2p_${suffix}`; + const peerId = `peer_a@im.wechat_${suffix}`; + + await upsertUser( + db, + { id: userId, username: `alice_${suffix}` }, + new Set(), + ); + await upsertBotAccount(db, { + id: botId, + ownerUserId: userId, + displayName: "p2p-bot", + botToken: "tok", + }); + await saveBotCredentials(db, { + botId, + botToken: "tok-secret", + savedAt: nowIso(), + }); + + const codeRec = await createBindCode(db, userId, `alice_${suffix}`, 600); + const svc = new P2PService(db); + + const bad = await svc.handleInbound({ + botId, + peerId, + text: "/绑定 WRONG1", + }); + assert.equal(bad.handled, true); + assert.match(bad.localReplies[0]!, /无效|过期/); + + const ok = await svc.handleInbound({ + botId, + peerId, + text: `/绑定 ${codeRec.code}`, + }); + assert.equal(ok.handled, true); + assert.match(ok.localReplies[0]!, /已绑定/); + + const again = await svc.handleInbound({ + botId, + peerId, + text: `/绑定 ${codeRec.code}`, + }); + assert.match(again.localReplies[0]!, /无效|过期/); + + const who = await svc.handleInbound({ + botId, + peerId, + text: "/我的身份", + }); + assert.match(who.localReplies[0]!, new RegExp(`alice_${suffix}`)); + + const un = await svc.handleInbound({ + botId, + peerId, + text: "/解绑", + }); + assert.match(un.localReplies[0]!, /解除/); + }); + }); + + it("request → accept → relay prefix → disconnect", async (t) => { + await withDb(t, async (db) => { + const suffix = Date.now().toString(36); + const aliceId = `u_a_${suffix}`; + const bobId = `u_b_${suffix}`; + const botA = `bot_a_${suffix}`; + const botB = `bot_b_${suffix}`; + const peerA = `peer_a_${suffix}@im.wechat`; + const peerB = `peer_b_${suffix}@im.wechat`; + + await upsertUser(db, { id: aliceId, username: `Alice_${suffix}` }, new Set()); + await upsertUser(db, { id: bobId, username: `Bob_${suffix}` }, new Set()); + + for (const [botId, owner] of [ + [botA, aliceId], + [botB, bobId], + ] as const) { + await upsertBotAccount(db, { + id: botId, + ownerUserId: owner, + displayName: botId, + botToken: "t", + }); + await saveBotCredentials(db, { + botId, + botToken: `token-${botId}`, + savedAt: nowIso(), + }); + } + + await setPrimaryBind(db, { + userId: aliceId, + username: `Alice_${suffix}`, + botId: botA, + peerId: peerA, + boundAt: nowIso(), + }); + await setPrimaryBind(db, { + userId: bobId, + username: `Bob_${suffix}`, + botId: botB, + peerId: peerB, + boundAt: nowIso(), + }); + await upsertContextToken(db, botB, peerB, "ctx-bob"); + await upsertContextToken(db, botA, peerA, "ctx-alice"); + + const svc = new P2PService(db, { maxRequestsPerDay: 50 }); + + // fallthrough when unbound style message + const fall = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: "你好呀", + }); + assert.equal(fall.handled, false); + + // hello @user is NOT connect + const notAt = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: `hello @Bob_${suffix}`, + }); + assert.equal(notAt.handled, false); + + const req = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: `@Bob_${suffix}`, + }); + assert.equal(req.handled, true); + assert.match(req.localReplies[0]!, /对话请求/); + assert.equal(req.remoteSends.length, 1); + assert.equal(req.remoteSends[0]!.botId, botB); + assert.match(req.remoteSends[0]!.text, /对话请求/); + + const accept = await svc.handleInbound({ + botId: botB, + peerId: peerB, + text: "/同意", + }); + assert.equal(accept.handled, true); + assert.match(accept.localReplies[0]!, /建立对话/); + assert.equal(accept.remoteSends.length, 1); + assert.equal(accept.remoteSends[0]!.botId, botA); + + const relay = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: "在吗朋友", + }); + assert.equal(relay.handled, true); + assert.equal(relay.localReplies.length, 0); + assert.equal(relay.remoteSends.length, 1); + assert.equal( + relay.remoteSends[0]!.text, + `[Alice_${suffix}] 在吗朋友`, + ); + assert.equal(relay.remoteSends[0]!.botId, botB); + + const disc = await svc.handleInbound({ + botId: botB, + peerId: peerB, + text: "/断开", + }); + assert.equal(disc.handled, true); + assert.match(disc.localReplies[0]!, /断开/); + assert.equal(disc.remoteSends[0]!.botId, botA); + + const after = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: "又回到角色扮演", + }); + assert.equal(after.handled, false); + }); + }); + + it("rejects @ self and unreachable without context", async (t) => { + await withDb(t, async (db) => { + const suffix = Date.now().toString(36); + const aliceId = `u_a2_${suffix}`; + const bobId = `u_b2_${suffix}`; + const botA = `bot_a2_${suffix}`; + const botB = `bot_b2_${suffix}`; + const peerA = `peer_a2_${suffix}`; + const peerB = `peer_b2_${suffix}`; + + await upsertUser(db, { id: aliceId, username: `aa_${suffix}` }, new Set()); + await upsertUser(db, { id: bobId, username: `bb_${suffix}` }, new Set()); + await upsertBotAccount(db, { + id: botA, + ownerUserId: aliceId, + displayName: "a", + botToken: "t", + }); + await upsertBotAccount(db, { + id: botB, + ownerUserId: bobId, + displayName: "b", + botToken: "t", + }); + await saveBotCredentials(db, { + botId: botA, + botToken: "tokA", + savedAt: nowIso(), + }); + await saveBotCredentials(db, { + botId: botB, + botToken: "tokB", + savedAt: nowIso(), + }); + await setPrimaryBind(db, { + userId: aliceId, + username: `aa_${suffix}`, + botId: botA, + peerId: peerA, + boundAt: nowIso(), + }); + await setPrimaryBind(db, { + userId: bobId, + username: `bb_${suffix}`, + botId: botB, + peerId: peerB, + boundAt: nowIso(), + }); + // no context_token for bob → unreachable + await upsertContextToken(db, botA, peerA, "ctx-a"); + + const svc = new P2PService(db); + const self = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: `@aa_${suffix}`, + }); + assert.match(self.localReplies[0]!, /自己/); + + const unreach = await svc.handleInbound({ + botId: botA, + peerId: peerA, + text: `@bb_${suffix}`, + }); + assert.match(unreach.localReplies[0]!, /不可达/); + }); + }); +}); diff --git a/packages/core/src/p2p-service.ts b/packages/core/src/p2p-service.ts new file mode 100644 index 0000000..b5d8087 --- /dev/null +++ b/packages/core/src/p2p-service.ts @@ -0,0 +1,613 @@ +import { + type Db, + acceptConnectRequest, + blockUser, + clearPrimaryBind, + consumeBindCode, + createConnectRequest, + deleteConnectRequest, + deleteP2PSession, + getBindByPeer, + getBindByUser, + getInboundRequest, + getOutboundRequest, + getP2PRequestDayCount, + getP2PSessionForPeer, + type P2PSession, + getUser, + getUserByUsername, + getUsersByIds, + incrP2PRequestDay, + isBlockedEitherWay, + isPeerReachable, + listBlockedUserIds, + otherParty, + selfParty, + setPrimaryBind, + touchP2PSession, + unblockUser, + type PeerEndpoint, + type PeerIdentity, + type UserWechatBind, + nowIso, +} from "@wechat-ai/db"; + +export interface P2PServiceOptions { + bindCodeTtlSec: number; + requestTtlSec: number; + sessionIdleSec: number; + relayMaxChars: number; + maxRequestsPerDay: number; +} + +export interface P2PInboundRequest { + botId: string; + peerId: string; + text: string; + mediaOnly?: boolean; +} + +export interface P2PRemoteSend { + botId: string; + peerId: string; + text: string; +} + +export interface P2PHandleResult { + /** true = worker must NOT call ChatService / LLM */ + handled: boolean; + localReplies: string[]; + remoteSends: P2PRemoteSend[]; +} + +const DEFAULTS: P2PServiceOptions = { + bindCodeTtlSec: 600, + requestTtlSec: 300, + sessionIdleSec: 1800, + relayMaxChars: 500, + maxRequestsPerDay: 20, +}; + +/** Whole-message @username (optional leading/trailing whitespace). */ +const AT_USER_RE = /^\s*@([A-Za-z0-9_.\-]{1,64})\s*$/; + +const BIND_RE = /^\s*\/绑定\s+([A-Za-z0-9]{4,12})\s*$/i; +const UNBIND_RE = /^\s*\/解绑\s*$/; +const WHOAMI_RE = /^\s*\/我的身份\s*$/; +const ACCEPT_RE = /^\s*\/同意\s*$/; +const REJECT_RE = /^\s*\/拒绝\s*$/; +const DISCONNECT_RE = /^\s*\/断开\s*$/; +const CANCEL_RE = /^\s*\/取消请求\s*$/; +const BLOCK_RE = /^\s*\/拉黑\s+@?([A-Za-z0-9_.\-]{1,64})\s*$/; +const UNBLOCK_RE = /^\s*\/取消拉黑\s+@?([A-Za-z0-9_.\-]{1,64})\s*$/; +const BLOCKLIST_RE = /^\s*\/黑名单\s*$/; + +function localOnly(text: string): P2PHandleResult { + return { handled: true, localReplies: [text], remoteSends: [] }; +} + +function fallthrough(): P2PHandleResult { + return { handled: false, localReplies: [], remoteSends: [] }; +} + +function fmtMin(sec: number): number { + return Math.max(1, Math.round(sec / 60)); +} + +export class P2PService { + private opts: P2PServiceOptions; + + constructor( + private db: Db, + opts: Partial = {}, + ) { + this.opts = { ...DEFAULTS, ...opts }; + } + + /** Apply admin-editable settings in place (runtime settings reload). */ + applyRuntimeOptions(patch: Partial): void { + Object.assign(this.opts, patch); + } + + async handleInbound(req: P2PInboundRequest): Promise { + const text = (req.text ?? "").trim(); + const mediaOnly = Boolean(req.mediaOnly) || !text; + + // Command matching is pure regex and already takes precedence over relay, + // so run it BEFORE loading the session. Every ordinary roleplay message + // used to pay a Redis GET here just to find no session. + if (!mediaOnly) { + const bindMatch = text.match(BIND_RE); + if (bindMatch) return this.handleBind(req, bindMatch[1]!); + + if (UNBIND_RE.test(text)) return this.handleUnbind(req); + if (WHOAMI_RE.test(text)) return this.handleWhoami(req); + if (ACCEPT_RE.test(text)) return this.handleAccept(req); + if (REJECT_RE.test(text)) return this.handleReject(req); + if (DISCONNECT_RE.test(text)) return this.handleDisconnect(req); + if (CANCEL_RE.test(text)) return this.handleCancel(req); + + const blockMatch = text.match(BLOCK_RE); + if (blockMatch) return this.handleBlock(req, blockMatch[1]!); + const unblockMatch = text.match(UNBLOCK_RE); + if (unblockMatch) return this.handleUnblock(req, unblockMatch[1]!); + if (BLOCKLIST_RE.test(text)) return this.handleBlockList(req); + + const atMatch = text.match(AT_USER_RE); + if (atMatch) return this.handleAt(req, atMatch[1]!); + } + + const session = await getP2PSessionForPeer( + this.db, + req.botId, + req.peerId, + ); + if (session && mediaOnly) { + return localOnly("会话中暂不支持图片/语音，请发送文字，或发送 /断开 结束对话。"); + } + + // Active session → relay (pass the row we just loaded; microseconds old) + if (session && !mediaOnly) { + return this.handleRelay(req, session); + } + + return fallthrough(); + } + + // ── Bind ───────────────────────────────────────────── + + private async handleBind( + req: P2PInboundRequest, + code: string, + ): Promise { + const rec = await consumeBindCode(this.db, code); + if (!rec) { + return localOnly("绑定码无效或已过期。请到用户中心重新生成绑定码后再试。"); + } + + const user = await getUser(this.db, rec.userId); + if (!user) { + return localOnly("绑定失败：平台账号不存在。请重新登录用户中心后再生成绑定码。"); + } + + const bind: UserWechatBind = { + userId: user.id, + username: user.username, + botId: req.botId, + peerId: req.peerId, + boundAt: nowIso(), + }; + await setPrimaryBind(this.db, bind); + + return localOnly( + `已绑定 LINUX DO 账号 @${user.username}。现在可以发送 @对方用户名 发起对话。`, + ); + } + + private async handleUnbind( + req: P2PInboundRequest, + ): Promise { + const bind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!bind) { + return localOnly("当前微信尚未绑定 LINUX DO 账号。"); + } + await clearPrimaryBind(this.db, bind.userId); + return localOnly(`已解除与 @${bind.username} 的绑定。`); + } + + private async handleWhoami( + req: P2PInboundRequest, + ): Promise { + const bind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!bind) { + return localOnly( + "当前微信尚未绑定。请到用户中心生成绑定码，然后发送 /绑定 验证码。", + ); + } + const session = await getP2PSessionForPeer( + this.db, + req.botId, + req.peerId, + ); + const out = await getOutboundRequest(this.db, req.botId, req.peerId); + const inn = await getInboundRequest(this.db, req.botId, req.peerId); + let state = "空闲"; + if (session) { + const other = otherParty(session, req.botId, req.peerId); + state = other ? `对话中（与 @${other.username}）` : "对话中"; + } else if (out) { + state = `等待 @${out.to.username} 同意对话请求`; + } else if (inn) { + state = `收到 @${inn.from.username} 的对话请求（/同意 或 /拒绝）`; + } + return localOnly( + `身份：@${bind.username}\n状态：${state}\n命令：@用户名 /同意 /拒绝 /断开 /取消请求 /拉黑 用户名 /解绑`, + ); + } + + private async handleBlock( + req: P2PInboundRequest, + rawUsername: string, + ): Promise { + const selfBind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!selfBind) { + return localOnly("请先绑定 LINUX DO 账号后再使用拉黑。"); + } + const target = await getUserByUsername(this.db, rawUsername); + if (!target) { + return localOnly(`找不到用户 @${rawUsername}。`); + } + if (target.id === selfBind.userId) { + return localOnly("不能拉黑自己。"); + } + const r = await blockUser(this.db, selfBind.userId, target.id); + if (!r.ok && r.reason === "already") { + return localOnly(`@${target.username} 已在你的黑名单中。`); + } + return localOnly( + `已拉黑 @${target.username}。对方无法再向你发起对话；进行中的会话已结束。可在用户中心管理黑名单，或发送 /取消拉黑 ${target.username}。`, + ); + } + + private async handleUnblock( + req: P2PInboundRequest, + rawUsername: string, + ): Promise { + const selfBind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!selfBind) { + return localOnly("请先绑定 LINUX DO 账号。"); + } + const target = await getUserByUsername(this.db, rawUsername); + if (!target) { + return localOnly(`找不到用户 @${rawUsername}。`); + } + const ok = await unblockUser(this.db, selfBind.userId, target.id); + if (!ok) { + return localOnly(`@${target.username} 不在你的黑名单中。`); + } + return localOnly(`已将 @${target.username} 移出黑名单。`); + } + + private async handleBlockList( + req: P2PInboundRequest, + ): Promise { + const selfBind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!selfBind) { + return localOnly("请先绑定 LINUX DO 账号。"); + } + const ids = await listBlockedUserIds(this.db, selfBind.userId); + if (!ids.length) { + return localOnly("黑名单为空。可发送 /拉黑 用户名 或在用户中心管理。"); + } + const map = await getUsersByIds(this.db, ids); + const lines = ids.map((id) => { + const u = map.get(id); + return u ? `· @${u.username}` : `· (id:${id})`; + }); + return localOnly(`黑名单（${ids.length}）：\n${lines.join("\n")}`); + } + + // ── Connect ────────────────────────────────────────── + + private async handleAt( + req: P2PInboundRequest, + rawUsername: string, + ): Promise { + const selfBind = await getBindByPeer(this.db, req.botId, req.peerId); + if (!selfBind) { + return localOnly( + "请先绑定 LINUX DO 账号后再使用 @。打开用户中心生成绑定码，然后发送 /绑定 验证码。", + ); + } + + // Busy checks + const existingSess = await getP2PSessionForPeer( + this.db, + req.botId, + req.peerId, + ); + if (existingSess) { + const other = otherParty(existingSess, req.botId, req.peerId); + return localOnly( + `你正在与 @${other?.username ?? "对方"} 对话中。请先发送 /断开 再发起新请求。`, + ); + } + const existingOut = await getOutboundRequest( + this.db, + req.botId, + req.peerId, + ); + if (existingOut) { + return localOnly( + `已有等待中的请求（@${existingOut.to.username}）。发送 /取消请求 可取消。`, + ); + } + const existingIn = await getInboundRequest( + this.db, + req.botId, + req.peerId, + ); + if (existingIn) { + return localOnly( + `你有来自 @${existingIn.from.username} 的待处理请求。请先 /同意 或 /拒绝。`, + ); + } + + const targetUser = await getUserByUsername(this.db, rawUsername); + if (!targetUser) { + return localOnly(`找不到用户 @${rawUsername}。请确认对方已用 LINUX DO 登录过本平台。`); + } + if (targetUser.id === selfBind.userId) { + return localOnly("不能与自己建立对话。"); + } + + if (await isBlockedEitherWay(this.db, selfBind.userId, targetUser.id)) { + return localOnly( + `无法与 @${targetUser.username} 建立对话（黑名单限制）。可在用户中心查看/管理黑名单。`, + ); + } + + const targetBind = await getBindByUser(this.db, targetUser.id); + if (!targetBind) { + return localOnly( + `@${targetUser.username} 尚未绑定微信，暂时无法联系。请对方先在用户中心完成绑定。`, + ); + } + + const reachable = await isPeerReachable( + this.db, + targetBind.botId, + targetBind.peerId, + ); + if (!reachable) { + return localOnly( + `@${targetUser.username} 当前不可达（对方需先与机器人聊过至少一次）。`, + ); + } + + // Daily rate + const dayCount = await getP2PRequestDayCount( + this.db, + req.botId, + req.peerId, + ); + if (dayCount >= this.opts.maxRequestsPerDay) { + return localOnly( + `今日发起对话请求次数已达上限（${this.opts.maxRequestsPerDay}）。请明天再试。`, + ); + } + + const from: PeerIdentity = { + botId: selfBind.botId, + peerId: selfBind.peerId, + userId: selfBind.userId, + username: selfBind.username, + }; + // Prefer live username + const liveSelf = await getUser(this.db, selfBind.userId); + if (liveSelf?.username) from.username = liveSelf.username; + + const to: PeerIdentity = { + botId: targetBind.botId, + peerId: targetBind.peerId, + userId: targetBind.userId, + username: targetUser.username, + }; + + const created = await createConnectRequest( + this.db, + from, + to, + this.opts.requestTtlSec, + ); + if (!created.ok) { + if (created.reason === "to_busy") { + return localOnly(`@${targetUser.username} 正忙（已有请求或对话），请稍后再试。`); + } + if (created.reason === "from_busy") { + return localOnly("你当前有进行中的请求或对话，请先处理后再发起。"); + } + return localOnly("发起请求失败，请稍后再试。"); + } + + await incrP2PRequestDay(this.db, req.botId, req.peerId); + + const mins = fmtMin(this.opts.requestTtlSec); + return { + handled: true, + localReplies: [ + `已向 @${to.username} 发送对话请求，等待对方同意…（${mins}分钟内有效）`, + ], + remoteSends: [ + { + botId: to.botId, + peerId: to.peerId, + text: `【对话请求】@${from.username} 想通过机器人与你对话。回复 /同意 开始，/拒绝 忽略。（${mins}分钟内有效）`, + }, + ], + }; + } + + private async handleAccept( + req: P2PInboundRequest, + ): Promise { + const inbound = await getInboundRequest(this.db, req.botId, req.peerId); + if (inbound) { + const selfBind = await getBindByPeer(this.db, req.botId, req.peerId); + if ( + selfBind && + (await isBlockedEitherWay( + this.db, + selfBind.userId, + inbound.from.userId, + )) + ) { + await deleteConnectRequest(this.db, inbound); + return localOnly( + `无法同意：与 @${inbound.from.username} 存在黑名单限制，请求已关闭。`, + ); + } + } + + const result = await acceptConnectRequest( + this.db, + req.botId, + req.peerId, + this.opts.sessionIdleSec, + ); + if (!result.ok) { + return localOnly("没有待处理的对话请求。"); + } + const { request } = result; + const idleMin = fmtMin(this.opts.sessionIdleSec); + return { + handled: true, + localReplies: [ + `已与 @${request.from.username} 建立对话。直接发文字即可，发送 /断开 结束。（空闲 ${idleMin} 分钟自动结束）`, + ], + remoteSends: [ + { + botId: request.from.botId, + peerId: request.from.peerId, + text: `@${request.to.username} 已同意对话。现在可以直接发消息了。发送 /断开 结束。`, + }, + ], + }; + } + + private async handleReject( + req: P2PInboundRequest, + ): Promise { + const request = await getInboundRequest(this.db, req.botId, req.peerId); + if (!request) { + return localOnly("没有待处理的对话请求。"); + } + await deleteConnectRequest(this.db, request); + return { + handled: true, + localReplies: [`已拒绝 @${request.from.username} 的对话请求。`], + remoteSends: [ + { + botId: request.from.botId, + peerId: request.from.peerId, + text: `@${request.to.username} 拒绝了你的对话请求。`, + }, + ], + }; + } + + private async handleCancel( + req: P2PInboundRequest, + ): Promise { + const request = await getOutboundRequest(this.db, req.botId, req.peerId); + if (!request) { + return localOnly("没有可取消的对话请求。"); + } + await deleteConnectRequest(this.db, request); + return { + handled: true, + localReplies: [`已取消向 @${request.to.username} 的对话请求。`], + remoteSends: [ + { + botId: request.to.botId, + peerId: request.to.peerId, + text: `@${request.from.username} 取消了对话请求。`, + }, + ], + }; + } + + private async handleDisconnect( + req: P2PInboundRequest, + ): Promise { + const session = await getP2PSessionForPeer( + this.db, + req.botId, + req.peerId, + ); + if (!session) { + return localOnly("当前没有进行中的对话。"); + } + const me = selfParty(session, req.botId, req.peerId); + const other = otherParty(session, req.botId, req.peerId); + await deleteP2PSession(this.db, session); + const local = "已断开对话，恢复与机器人的角色扮演。"; + if (!other || !me) { + return localOnly(local); + } + return { + handled: true, + localReplies: [local], + remoteSends: [ + { + botId: other.botId, + peerId: other.peerId, + text: `@${me.username} 已断开对话。`, + }, + ], + }; + } + + // ── Relay ──────────────────────────────────────────── + + private async handleRelay( + req: P2PInboundRequest, + session: P2PSession, + ): Promise { + // Session was loaded by the caller microseconds ago — re-reading it here + // was a second GET on every relayed message for no added safety. + const me = selfParty(session, req.botId, req.peerId); + const other = otherParty(session, req.botId, req.peerId); + if (!me || !other) { + await deleteP2PSession(this.db, session); + return localOnly("会话状态异常，已清理。请重新 @用户名 发起。"); + } + + let body = (req.text ?? "").trim(); + if (body.length > this.opts.relayMaxChars) { + body = body.slice(0, this.opts.relayMaxChars); + } + if (!body) { + return localOnly("消息为空，未发送。"); + } + + await touchP2PSession(this.db, session, this.opts.sessionIdleSec); + + return { + handled: true, + localReplies: [], // silent ack on sender side + remoteSends: [ + { + botId: other.botId, + peerId: other.peerId, + text: `[${me.username}] ${body}`, + }, + ], + }; + } +} + +/** Pure helpers exported for tests */ +export function parseAtUsername(text: string): string | null { + const m = text.match(AT_USER_RE); + return m?.[1] ?? null; +} + +export function isP2PCommand(text: string): boolean { + const t = text.trim(); + return ( + BIND_RE.test(t) || + UNBIND_RE.test(t) || + WHOAMI_RE.test(t) || + ACCEPT_RE.test(t) || + REJECT_RE.test(t) || + DISCONNECT_RE.test(t) || + CANCEL_RE.test(t) || + BLOCK_RE.test(t) || + UNBLOCK_RE.test(t) || + BLOCKLIST_RE.test(t) || + AT_USER_RE.test(t) + ); +} + +export type { PeerEndpoint, PeerIdentity }; diff --git a/packages/core/src/proactive.test.ts b/packages/core/src/proactive.test.ts new file mode 100644 index 0000000..fc741af --- /dev/null +++ b/packages/core/src/proactive.test.ts @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + hourInTimeZone, + isInQuietHours, + isProactiveEligible, + mergeBotProactiveConfig, + parseQuietHours, +} from "./proactive.js"; +import { parseProactiveSkip, buildProactiveInstruction } from "./prompt.js"; + +describe("parseQuietHours / isInQuietHours", () => { + it("parses daytime window", () => { + assert.deepEqual(parseQuietHours("0-8"), { start: 0, end: 8 }); + assert.equal(isInQuietHours(3, { start: 0, end: 8 }), true); + assert.equal(isInQuietHours(8, { start: 0, end: 8 }), false); + assert.equal(isInQuietHours(12, { start: 0, end: 8 }), false); + }); + + it("parses overnight window", () => { + assert.deepEqual(parseQuietHours("22-6"), { start: 22, end: 6 }); + assert.equal(isInQuietHours(23, { start: 22, end: 6 }), true); + assert.equal(isInQuietHours(3, { start: 22, end: 6 }), true); + assert.equal(isInQuietHours(10, { start: 22, end: 6 }), false); + }); + + it("treats empty as disabled", () => { + assert.equal(parseQuietHours(""), null); + assert.equal(parseQuietHours(null), null); + assert.equal(parseQuietHours("0-0"), null); + }); +}); + +describe("isProactiveEligible", () => { + const fixedNow = new Date("2026-07-20T12:00:00+08:00"); + const base = { + botStatus: "active", + botProactiveEnabled: 1, + peerApproved: 1, + peerProactiveEnabled: 1, + hasContextToken: true, + // 20h idle relative to fixedNow + lastActivityAt: new Date("2026-07-19T16:00:00+08:00").toISOString(), + dayCount: 0, + idleHours: 12, + minIntervalHours: 24, + maxPerDay: 1, + quietHours: "", + now: fixedNow, + quietTimeZone: "Asia/Shanghai", + attemptCooldownHours: 1, + }; + + it("allows idle approved peer", () => { + const r = isProactiveEligible(base); + assert.equal(r.ok, true); + assert.ok((r.idleHoursActual ?? 0) >= 12); + }); + + it("rejects when peer off", () => { + const r = isProactiveEligible({ ...base, peerProactiveEnabled: 0 }); + assert.equal(r.ok, false); + assert.equal(r.reason, "peer_off"); + }); + + it("rejects when not idle", () => { + const r = isProactiveEligible({ + ...base, + lastActivityAt: new Date(Date.now() - 1 * 3600 * 1000).toISOString(), + now: new Date(), + }); + assert.equal(r.ok, false); + assert.equal(r.reason, "not_idle"); + }); + + it("rejects day cap", () => { + const r = isProactiveEligible({ ...base, dayCount: 1, maxPerDay: 1 }); + assert.equal(r.ok, false); + assert.equal(r.reason, "day_cap"); + }); + + it("allows unlimited day when maxPerDay is 0", () => { + const r = isProactiveEligible({ ...base, dayCount: 99, maxPerDay: 0 }); + assert.equal(r.ok, true); + }); + + it("allows zero min interval", () => { + const r = isProactiveEligible({ + ...base, + minIntervalHours: 0, + lastProactiveAt: new Date(Date.now() - 60 * 1000).toISOString(), + now: new Date(), + attemptCooldownHours: 0, + }); + assert.equal(r.ok, true); + }); + + it("rejects quiet hours", () => { + // 3am Shanghai + const r = isProactiveEligible({ + ...base, + quietHours: "0-8", + now: new Date("2026-07-20T03:00:00+08:00"), + }); + assert.equal(r.ok, false); + assert.equal(r.reason, "quiet_hours"); + }); + + it("rejects min interval after last proactive", () => { + const r = isProactiveEligible({ + ...base, + lastProactiveAt: new Date(Date.now() - 2 * 3600 * 1000).toISOString(), + now: new Date(), + minIntervalHours: 24, + }); + assert.equal(r.ok, false); + assert.equal(r.reason, "min_interval"); + }); + + it("rejects no context token", () => { + const r = isProactiveEligible({ ...base, hasContextToken: false }); + assert.equal(r.ok, false); + assert.equal(r.reason, "no_context_token"); + }); +}); + +describe("mergeBotProactiveConfig", () => { + it("falls back to defaults", () => { + const m = mergeBotProactiveConfig( + {}, + { + idleHours: 12, + minIntervalHours: 24, + maxPerDay: 1, + quietHours: "0-8", + }, + ); + assert.equal(m.enabled, false); + assert.equal(m.idleHours, 12); + assert.equal(m.quietHours, "0-8"); + }); + + it("uses bot overrides", () => { + const m = mergeBotProactiveConfig( + { + proactive_enabled: 1, + proactive_idle_hours: 6, + proactive_quiet_hours: "", + }, + { + idleHours: 12, + minIntervalHours: 24, + maxPerDay: 1, + quietHours: "0-8", + }, + ); + assert.equal(m.enabled, true); + assert.equal(m.idleHours, 6); + assert.equal(m.quietHours, ""); + }); + + it("null quiet hours means disabled (not default)", () => { + const m = mergeBotProactiveConfig( + { proactive_quiet_hours: null }, + { + idleHours: 12, + minIntervalHours: 24, + maxPerDay: 1, + quietHours: "0-8", + }, + ); + assert.equal(m.quietHours, ""); + }); +}); + +describe("parseProactiveSkip", () => { + it("detects skip json", () => { + const r = parseProactiveSkip('{"skip":true,"reason":"休息"}'); + assert.equal(r.skip, true); + assert.equal(r.reason, "休息"); + }); + + it("does not skip normal messages json", () => { + const r = parseProactiveSkip('{"messages":["嗨～"]}'); + assert.equal(r.skip, false); + }); +}); + +describe("buildProactiveInstruction", () => { + it("includes idle hours", () => { + const s = buildProactiveInstruction(12.3); + assert.match(s, /12\.3/); + assert.match(s, /主动发起对话/); + }); +}); + +describe("hourInTimeZone", () => { + it("returns a valid hour", () => { + const h = hourInTimeZone(new Date("2026-07-20T04:00:00Z"), "UTC"); + assert.equal(h, 4); + }); +}); diff --git a/packages/core/src/proactive.ts b/packages/core/src/proactive.ts new file mode 100644 index 0000000..0882b04 --- /dev/null +++ b/packages/core/src/proactive.ts @@ -0,0 +1,277 @@ +/** + * Idle-based proactive outreach eligibility (pure helpers + types). + * Scheduling / I/O lives in the worker; this package only decides "should we?". + */ + +export interface ProactivePolicy { + /** Global hard off (env PROACTIVE_ENABLED) */ + globalEnabled: boolean; + /** Default idle hours when bot does not override */ + defaultIdleHours: number; + defaultMinIntervalHours: number; + defaultMaxPerDay: number; + /** Default quiet hours "H-H" or empty to disable */ + defaultQuietHours: string; + /** Cooldown after a skip/attempt without a full min-interval write (hours) */ + attemptCooldownHours: number; +} + +export interface ProactiveEligibilityInput { + botStatus: string; + botProactiveEnabled: boolean | number | undefined; + peerApproved: boolean | number | undefined; + peerProactiveEnabled: boolean | number | undefined; + hasContextToken: boolean; + /** ISO last activity; falls back to createdAt if missing */ + lastActivityAt?: string | null; + peerCreatedAt?: string | null; + lastProactiveAt?: string | null; + lastProactiveAttemptAt?: string | null; + dayCount: number; + idleHours: number; + minIntervalHours: number; + maxPerDay: number; + quietHours?: string | null; + /** Override "now" for tests */ + now?: Date; + /** Timezone for quiet hours (default Asia/Shanghai) */ + quietTimeZone?: string; + attemptCooldownHours?: number; +} + +export type ProactiveSkipReason = + | "global_off" + | "bot_inactive" + | "bot_off" + | "peer_unapproved" + | "peer_off" + | "no_context_token" + | "not_idle" + | "min_interval" + | "day_cap" + | "quiet_hours" + | "attempt_cooldown"; + +export interface ProactiveEligibilityResult { + ok: boolean; + reason?: ProactiveSkipReason; + /** Hours since last activity (for prompts/logs) */ + idleHoursActual?: number; +} + +function isTruthyFlag(v: boolean | number | undefined | null): boolean { + if (v === true || v === 1) return true; + if (typeof v === "string" && (v === "1" || v === "true")) return true; + return false; +} + +function parseIsoMs(iso: string | null | undefined): number | null { + if (!iso) return null; + const t = Date.parse(iso); + return Number.isFinite(t) ? t : null; +} + +/** + * Parse "0-8" or "22-6" (overnight wrap) quiet window. + * Returns null if disabled / invalid. + */ +export function parseQuietHours( + raw: string | null | undefined, +): { start: number; end: number } | null { + if (raw == null) return null; + const s = String(raw).trim(); + if (!s) return null; + const m = s.match(/^(\d{1,2})-(\d{1,2})$/); + if (!m) return null; + const start = Number(m[1]); + const end = Number(m[2]); + if ( + !Number.isInteger(start) || + !Number.isInteger(end) || + start < 0 || + start > 23 || + end < 0 || + end > 23 + ) { + return null; + } + if (start === end) return null; // empty window + return { start, end }; +} + +/** Hour-of-day 0–23 in the given IANA timezone. */ +export function hourInTimeZone( + date: Date, + timeZone = "Asia/Shanghai", +): number { + try { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone, + hour: "numeric", + hour12: false, + }).formatToParts(date); + const h = parts.find((p) => p.type === "hour")?.value; + const n = h != null ? Number(h) : date.getHours(); + // Some engines emit "24" for midnight + if (n === 24) return 0; + return Number.isFinite(n) ? n : date.getHours(); + } catch { + return date.getHours(); + } +} + +/** True if `hour` is inside [start, end) with overnight wrap. */ +export function isInQuietHours( + hour: number, + window: { start: number; end: number }, +): boolean { + const { start, end } = window; + if (start < end) { + return hour >= start && hour < end; + } + // overnight e.g. 22-6 → 22,23,0,1,2,3,4,5 + return hour >= start || hour < end; +} + +export function resolveActivityMs(input: { + lastActivityAt?: string | null; + peerCreatedAt?: string | null; +}): number | null { + return ( + parseIsoMs(input.lastActivityAt) ?? parseIsoMs(input.peerCreatedAt) ?? null + ); +} + +/** + * Pure eligibility check for one bot+peer pair. + * Does not acquire locks or touch Redis. + */ +export function isProactiveEligible( + input: ProactiveEligibilityInput, +): ProactiveEligibilityResult { + const now = input.now ?? new Date(); + const nowMs = now.getTime(); + + if (input.botStatus && input.botStatus !== "active") { + return { ok: false, reason: "bot_inactive" }; + } + if (!isTruthyFlag(input.botProactiveEnabled)) { + return { ok: false, reason: "bot_off" }; + } + if (!isTruthyFlag(input.peerApproved)) { + return { ok: false, reason: "peer_unapproved" }; + } + if (!isTruthyFlag(input.peerProactiveEnabled)) { + return { ok: false, reason: "peer_off" }; + } + if (!input.hasContextToken) { + return { ok: false, reason: "no_context_token" }; + } + + const quiet = parseQuietHours(input.quietHours); + if (quiet) { + const hour = hourInTimeZone(now, input.quietTimeZone ?? "Asia/Shanghai"); + if (isInQuietHours(hour, quiet)) { + return { ok: false, reason: "quiet_hours" }; + } + } + + // maxPerDay <= 0 means unlimited + const maxPerDay = Number(input.maxPerDay); + if (Number.isFinite(maxPerDay) && maxPerDay > 0 && input.dayCount >= maxPerDay) { + return { ok: false, reason: "day_cap" }; + } + + const activityMs = resolveActivityMs({ + lastActivityAt: input.lastActivityAt, + peerCreatedAt: input.peerCreatedAt, + }); + if (activityMs == null) { + return { ok: false, reason: "not_idle" }; + } + + const idleMs = nowMs - activityMs; + const idleHoursActual = idleMs / (3600 * 1000); + if (idleHoursActual < input.idleHours) { + return { ok: false, reason: "not_idle", idleHoursActual }; + } + + const lastProactiveMs = parseIsoMs(input.lastProactiveAt); + const minInterval = Math.max(0, Number(input.minIntervalHours) || 0); + if (lastProactiveMs != null && minInterval > 0) { + const sinceProactiveH = (nowMs - lastProactiveMs) / (3600 * 1000); + if (sinceProactiveH < minInterval) { + return { ok: false, reason: "min_interval", idleHoursActual }; + } + } + + const attemptCooldown = + input.attemptCooldownHours != null && input.attemptCooldownHours > 0 + ? input.attemptCooldownHours + : Math.min(1, input.minIntervalHours); + const lastAttemptMs = parseIsoMs(input.lastProactiveAttemptAt); + // Only apply attempt cooldown when last attempt was not a successful send + // (if last_proactive_at equals last_proactive_attempt_at, min_interval already covers it) + if ( + lastAttemptMs != null && + (lastProactiveMs == null || lastAttemptMs > lastProactiveMs) + ) { + const sinceAttemptH = (nowMs - lastAttemptMs) / (3600 * 1000); + if (sinceAttemptH < attemptCooldown) { + return { ok: false, reason: "attempt_cooldown", idleHoursActual }; + } + } + + return { ok: true, idleHoursActual }; +} + +export function mergeBotProactiveConfig( + bot: { + proactive_enabled?: number; + proactive_idle_hours?: number; + proactive_min_interval_hours?: number; + proactive_max_per_day?: number; + /** undefined = use default; null or "" = explicitly disabled */ + proactive_quiet_hours?: string | null; + }, + defaults: { + idleHours: number; + minIntervalHours: number; + maxPerDay: number; + quietHours: string; + }, +): { + enabled: boolean; + idleHours: number; + minIntervalHours: number; + maxPerDay: number; + quietHours: string; +} { + let quietHours = defaults.quietHours; + if (bot.proactive_quiet_hours === null) { + quietHours = ""; + } else if (bot.proactive_quiet_hours !== undefined) { + quietHours = String(bot.proactive_quiet_hours); + } + + return { + enabled: isTruthyFlag(bot.proactive_enabled), + idleHours: + bot.proactive_idle_hours != null && bot.proactive_idle_hours > 0 + ? bot.proactive_idle_hours + : defaults.idleHours, + // Allow explicit 0 (= no min interval) + minIntervalHours: + bot.proactive_min_interval_hours != null && + Number.isFinite(bot.proactive_min_interval_hours) + ? Math.max(0, bot.proactive_min_interval_hours) + : defaults.minIntervalHours, + // Allow explicit 0 (= unlimited daily) + maxPerDay: + bot.proactive_max_per_day != null && + Number.isFinite(bot.proactive_max_per_day) + ? Math.max(0, bot.proactive_max_per_day) + : defaults.maxPerDay, + quietHours, + }; +} diff --git a/packages/core/src/prompt-vars.test.ts b/packages/core/src/prompt-vars.test.ts new file mode 100644 index 0000000..5874357 --- /dev/null +++ b/packages/core/src/prompt-vars.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { flattenChatContent } from "@wechat-ai/llm"; +import { + applyPromptTemplate, + buildBotIdentityBlock, + buildChatMessages, +} from "./prompt.js"; + +/** ChatMessage.content is string | ChatContentPart[] since vision landed. */ +const text = (m: { content: Parameters[0] }): string => + flattenChatContent(m.content); + +describe("applyPromptTemplate", () => { + it("replaces bot name variables", () => { + const t = applyPromptTemplate( + "我是{{bot_name}}，也叫{{机器人名字}}。", + { botName: "小铃" }, + ); + assert.equal(t, "我是小铃，也叫小铃。"); + }); +}); + +describe("buildChatMessages bot identity", () => { + it("injects bot name into system", () => { + const msgs = buildChatMessages({ + systemPrompt: "你是猫娘{{bot_name}}。", + memories: [], + history: [], + userText: "hi", + botName: "小铃", + multiBubbleJson: false, + }); + assert.match(text(msgs[0]!), /小铃/); + assert.match(text(msgs[0]!), /智能体身份/); + assert.doesNotMatch(text(msgs[0]!), /\{\{bot_name\}\}/); + }); +}); + +describe("buildBotIdentityBlock", () => { + it("uses fallback name", () => { + assert.match(buildBotIdentityBlock(" "), /助手/); + }); +}); diff --git a/packages/core/src/prompt.test.ts b/packages/core/src/prompt.test.ts new file mode 100644 index 0000000..a923e95 --- /dev/null +++ b/packages/core/src/prompt.test.ts @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { parseFactsJson } from "./prompt.js"; + +describe("parseFactsJson", () => { + it("parses plain array", () => { + assert.deepEqual(parseFactsJson('["a","b"]'), ["a", "b"]); + }); + + it("parses fenced noise", () => { + assert.deepEqual( + parseFactsJson('Here:\n```json\n["喜欢猫"]\n```'), + ["喜欢猫"], + ); + }); +}); diff --git a/packages/core/src/prompt.ts b/packages/core/src/prompt.ts new file mode 100644 index 0000000..62703ac --- /dev/null +++ b/packages/core/src/prompt.ts @@ -0,0 +1,520 @@ +import type { MemoryRow, MessageRow, StickerPromptEntry } from "@wechat-ai/db"; +import type { ChatContentPart, ChatMessage } from "@wechat-ai/llm"; +import { + REPLY_FORMAT_INSTRUCTION, + REPLY_FORMAT_INSTRUCTION_TEXT_ONLY, + renderAssistantHistoryForModel, +} from "./reply-format.js"; + +/** + * Attachment kinds. Declared as a plain union rather than importing from + * @wechat-ai/ilink so core stays free of protocol coupling — the worker + * translates iLink media refs into these. + */ +export type AttachmentKind = "image" | "voice" | "video" | "file"; + +/** One attachment on the message currently being answered. */ +export interface PromptAttachment { + kind: AttachmentKind; + /** + * `data:;base64,...` — present only when the model can actually read + * the bytes (a vision-capable model plus a supported image mime). Absent + * means "tell the persona it exists but it cannot see/hear it". + */ + dataUri?: string; + mime?: string | null; + fileName?: string; + /** + * Text description produced by the vision endpoint (caption mode). Present + * means the roleplay model can "see" this attachment through words, even + * though it never receives the bytes. + */ + caption?: string; +} + +const ATTACHMENT_LABEL: Record = { + image: "图片", + voice: "语音", + video: "视频", + file: "文件", +}; + +function countByKind( + attachments: PromptAttachment[], +): Array<{ + kind: AttachmentKind; + total: number; + /** Bytes attached to this turn (direct vision mode) */ + readable: number; + /** Described in words by a vision endpoint (caption mode) */ + captioned: number; +}> { + const order: AttachmentKind[] = ["image", "voice", "video", "file"]; + return order + .map((kind) => { + const of = attachments.filter((a) => a.kind === kind); + return { + kind, + total: of.length, + readable: of.filter((a) => Boolean(a.dataUri)).length, + captioned: of.filter((a) => !a.dataUri && Boolean(a.caption?.trim())) + .length, + }; + }) + .filter((x) => x.total > 0); +} + +/** + * Tell the model what is attached and, crucially, what it cannot perceive — + * without this a persona happily hallucinates the contents of a video it never + * received. + * + * Three states per kind: bytes attached (direct mode), described in words + * (caption mode), or nothing at all. + */ +export function buildAttachmentBlock( + attachments: PromptAttachment[] | undefined, +): string { + if (!attachments?.length) return ""; + const lines: string[] = ["## 本条消息的附件"]; + for (const { kind, total, readable, captioned } of countByKind(attachments)) { + const label = ATTACHMENT_LABEL[kind]; + if (readable > 0) { + lines.push( + `- ${label} ×${total}：其中 ${readable} 个已随本条消息发给你，请**据实描述你真正看到的内容**。`, + ); + } + if (captioned > 0) { + lines.push( + `- ${label} ×${captioned}：你看不到原图，但**已由识图模型转成文字描述**，写在用户消息的方括号里。` + + `请把这段描述当作你亲眼所见来回应，但只依据描述里写到的内容，不要往外扩写细节。`, + ); + } + const blind = total - readable - captioned; + if (blind > 0) { + lines.push( + `- ${label} ×${blind}：内容**没有**发给你，你无法查看/收听。不要猜测或编造里面是什么；用人设语气说明看不了，并邀请对方用文字描述。`, + ); + } + } + return lines.join("\n"); +} + +/** + * The user turn for this message: plain text when nothing is readable, content + * parts when a vision model can see an attached image. + */ +export function buildUserContent( + userText: string, + attachments: PromptAttachment[] | undefined, +): string | ChatContentPart[] { + const text = userText.trim(); + const readable = (attachments ?? []).filter((a) => a.dataUri); + if (!readable.length) { + // Always go through describeAttachments, never bare `text`: in caption mode + // the description lives on the attachment, and returning just the user's + // words would silently drop it. It also keeps this turn identical to what + // the *next* turn will read back out of history. + return describeAttachments(userText, attachments); + } + const parts: ChatContentPart[] = []; + // Tag only what the model does NOT receive — a `[图片]` next to the actual + // image adds nothing, while a `[视频]` beside it is the only hint it exists. + const unsent = (attachments ?? []).filter((a) => !a.dataUri); + const lead = + describeAttachments(userText, unsent) || + describeAttachments(userText, attachments) || + text; + if (lead) parts.push({ type: "text", text: lead }); + for (const a of readable) { + parts.push({ type: "image_url", image_url: { url: a.dataUri! } }); + } + return parts; +} + +/** + * Readable one-liner for conversation history. The bytes are never persisted, + * so this is what later turns see — without it a follow-up like "所以呢？" loses + * all trace that an image was sent. + * + * When a caption is available it goes in too, which is the real payoff of + * caption mode: the *content* of the image survives in history rather than an + * opaque `[图片]`, so the model can still discuss it three turns later. + */ +export function describeAttachments( + userText: string, + attachments: PromptAttachment[] | undefined, +): string { + const text = (userText ?? "").trim(); + if (!attachments?.length) return text; + + const tags: string[] = []; + const captioned = new Set(); + for (const a of attachments) { + const caption = a.caption?.trim(); + if (!caption) continue; + captioned.add(a); + tags.push(`[${ATTACHMENT_LABEL[a.kind]}：${caption}]`); + } + const rest = attachments.filter((a) => !captioned.has(a)); + for (const { kind, total } of countByKind(rest)) { + tags.push( + total > 1 + ? `[${ATTACHMENT_LABEL[kind]}×${total}]` + : `[${ATTACHMENT_LABEL[kind]}]`, + ); + } + const joined = tags.join(""); + return text ? `${text}\n${joined}` : joined; +} + +/** + * Ask a vision model to describe an image. + * + * Deliberately persona-free and roleplay-free: this is a perception step whose + * output is fed to the roleplay model as plain text, so it must report what is + * actually in the frame and nothing else. Any character voice belongs to the + * model that reads this, not the one that writes it. + */ +export function buildImageCaptionMessages(params: { + dataUri: string; + /** The user's own caption / question, when they sent one — helps focus it */ + userText?: string; +}): ChatMessage[] { + const ask = (params.userText ?? "").trim(); + const focus = ask + ? `\n用户随图说了：「${ask.slice(0, 200)}」。若与图片相关，描述时请覆盖这一点。` + : ""; + return [ + { + role: "system", + content: [ + "你是图像描述器。用中文客观描述图片内容，供另一个对话模型参考。", + "规则：", + "- 只描述你确实看到的：主体、动作、场景、显著文字、表情与氛围", + "- 不要扮演角色、不要与用户对话、不要评论、不要加称呼或语气词", + "- 看不清就说看不清；绝对不要猜测或编造", + "- 控制在 120 字以内，一段话", + ].join("\n"), + }, + { + role: "user", + content: [ + { type: "text", text: `请描述这张图片。${focus}` }, + { type: "image_url", image_url: { url: params.dataUri } }, + ], + }, + ]; +} + +/** Placeholder tokens users can insert in persona editor */ +export const BOT_NAME_VARS = [ + "{{bot_name}}", + "{{BOT_NAME}}", + "{{机器人名字}}", + "{{机器人名称}}", +] as const; + +/** + * Replace persona template variables with runtime values. + * Unknown placeholders are left as-is. + */ +export function applyPromptTemplate( + template: string, + vars: { botName: string }, +): string { + const name = (vars.botName || "助手").trim() || "助手"; + let out = template; + for (const key of BOT_NAME_VARS) { + out = out.split(key).join(name); + } + return out; +} + +/** Always injected so the agent knows its display name even without variables. */ +export function buildBotIdentityBlock(botName: string): string { + const name = (botName || "助手").trim() || "助手"; + return [ + "## 智能体身份", + `你的名字是「${name}」。在对话中以该名字自称，并接受用户这样称呼你。`, + "若人设正文与名字冲突，以本段名字为准，其余性格设定仍以人设为准。", + ].join("\n"); +} + +export function buildStickerCatalogBlock( + stickers: StickerPromptEntry[] | undefined, +): string { + if (!stickers?.length) return ""; + const lines = stickers.map((s) => { + const tags = s.tags.length ? ` tags=[${s.tags.join(",")}]` : ""; + const desc = s.description ? ` — ${s.description}` : ""; + return `- slug=\`${s.slug}\` 名称=${s.display_name}${tags}${desc}`; + }); + return [ + "## 可用表情包（仅可使用下列 slug，禁止编造）", + "微信规则：**图片必须单独一条消息**，不能和文字写在同一条里。", + "发图时在 messages 数组里放**单独的对象元素**（前后可以是文字元素，但是相邻的另一条消息）：", + '正确：{"messages":["给你看～",{"type":"sticker","slug":"xxx"},"喜欢吗"]} → 三条消息：字 / 图 / 字', + "错误：字符串里塞 JSON、或试图在一条里又字又图、或编造未列出的 slug。", + "合适时才发，不要刷屏；每条回复最多 2 个 sticker；sticker 对象内禁止带文字。", + ...lines, + ].join("\n"); +} + +function buildMemoryBlock(memories: MemoryRow[]): string { + if (!memories.length) return ""; + return [ + "## 关于该用户的长期记忆（仅限此用户，勿与他人混淆）", + ...memories.map((m) => `- ${m.content}`), + ].join("\n"); +} + +function buildTimeToolBlock(enabled: boolean | undefined): string { + if (enabled === false) return ""; + return [ + "## 时间工具", + "当你需要准确的当前日期、星期或时刻时，调用工具 get_current_time。", + "不要凭空编造「现在几点/今天周几」；拿到工具结果后再用人设语气回复。", + ].join("\n"); +} + +function buildFormatBlock(opts: { + multiBubbleJson?: boolean; + stickers?: StickerPromptEntry[]; +}): string { + if (opts.multiBubbleJson === false) return ""; + const hasStickers = (opts.stickers?.length ?? 0) > 0; + return hasStickers + ? REPLY_FORMAT_INSTRUCTION + : REPLY_FORMAT_INSTRUCTION_TEXT_ONLY; +} + +/** + * Replay one stored turn back to the model. + * + * History holds display text, and for assistant turns that includes the + * `[表情:slug]` rendering of a sticker. Fed back raw it is a worked example of + * a format the model is explicitly forbidden to produce — and it copies what it + * sees over what it is told, which is how literal `[表情:…]` bubbles end up in + * WeChat. Normalize assistant turns to the sanctioned sticker JSON on the way + * in, so the transcript and the instruction agree. + */ +function historyMessage( + role: "user" | "assistant", + content: string, +): ChatMessage { + return { + role, + content: + role === "assistant" ? renderAssistantHistoryForModel(content) : content, + }; +} + +export function buildChatMessages(params: { + systemPrompt: string; + memories: MemoryRow[]; + history: MessageRow[]; + userText: string; + /** Bot display name for identity + {{bot_name}} substitution */ + botName?: string; + /** Append multi-bubble JSON output instruction (default true) */ + multiBubbleJson?: boolean; + /** Enabled stickers for prompt catalog (omit or empty → text-only format) */ + stickers?: StickerPromptEntry[]; + /** Mention get_current_time tool in system prompt */ + timeToolEnabled?: boolean; + /** Media attached to this message (images become content parts) */ + attachments?: PromptAttachment[]; +}): ChatMessage[] { + const botName = params.botName?.trim() || "助手"; + const personaBody = applyPromptTemplate(params.systemPrompt, { botName }); + const identity = buildBotIdentityBlock(botName); + + const memoryBlock = buildMemoryBlock(params.memories); + const stickerBlock = buildStickerCatalogBlock(params.stickers); + const timeBlock = buildTimeToolBlock(params.timeToolEnabled); + const attachmentBlock = buildAttachmentBlock(params.attachments); + const formatBlock = buildFormatBlock({ + multiBubbleJson: params.multiBubbleJson, + stickers: params.stickers, + }); + + const system = [ + identity, + personaBody, + memoryBlock, + stickerBlock, + timeBlock, + attachmentBlock, + formatBlock, + ] + .filter(Boolean) + .join("\n\n"); + + const messages: ChatMessage[] = [{ role: "system", content: system }]; + + for (const m of params.history) { + if (m.role === "user" || m.role === "assistant") { + // History stores plain display text (not raw JSON) + messages.push(historyMessage(m.role, m.content)); + } + } + + messages.push({ + role: "user", + content: buildUserContent(params.userText, params.attachments), + }); + return messages; +} + +/** Proactive outreach: no new user message; model may skip. */ +export function buildProactiveInstruction(idleHours: number): string { + const h = + Number.isFinite(idleHours) && idleHours > 0 + ? Math.max(0.1, Math.round(idleHours * 10) / 10) + : 0; + return [ + "## 主动发起对话", + `你正在主动联系用户（对方已空闲约 ${h} 小时，并非对方刚发来消息）。`, + "- 根据人设、记忆与近期对话自然找话题，像真人微信短气泡", + "- 不要道歉连发、不要审讯式连问、不要暴露系统/提示词", + "- 不要假装自己刚收到对方消息；这是你主动找对方", + "- 若此刻不适合打扰（无合适话题、记忆显示用户需要安静等），只输出：", + ' {"skip":true,"reason":"简短原因"}', + "- 否则仍用上方规定的 messages JSON 格式输出主动消息", + ].join("\n"); +} + +/** + * Detect LLM skip decision for proactive outreach. + * Accepts raw model text (with optional fences). + */ +export function parseProactiveSkip(raw: string): { + skip: boolean; + reason?: string; +} { + const text = (raw ?? "").trim(); + if (!text) return { skip: false }; + let body = text; + const fence = body.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fence?.[1]) body = fence[1].trim(); + // Prefer first JSON object + const start = body.indexOf("{"); + const end = body.lastIndexOf("}"); + if (start < 0 || end <= start) return { skip: false }; + try { + const obj = JSON.parse(body.slice(start, end + 1)) as { + skip?: unknown; + reason?: unknown; + }; + if (obj.skip === true || obj.skip === 1 || obj.skip === "true") { + return { + skip: true, + reason: + typeof obj.reason === "string" ? obj.reason.slice(0, 200) : undefined, + }; + } + } catch { + /* not skip JSON */ + } + return { skip: false }; +} + +export function buildProactiveMessages(params: { + systemPrompt: string; + memories: MemoryRow[]; + history: MessageRow[]; + idleHours: number; + botName?: string; + multiBubbleJson?: boolean; + stickers?: StickerPromptEntry[]; + timeToolEnabled?: boolean; +}): ChatMessage[] { + const botName = params.botName?.trim() || "助手"; + const personaBody = applyPromptTemplate(params.systemPrompt, { botName }); + const identity = buildBotIdentityBlock(botName); + const memoryBlock = buildMemoryBlock(params.memories); + const stickerBlock = buildStickerCatalogBlock(params.stickers); + const timeBlock = buildTimeToolBlock(params.timeToolEnabled); + const formatBlock = buildFormatBlock({ + multiBubbleJson: params.multiBubbleJson, + stickers: params.stickers, + }); + const proactiveBlock = buildProactiveInstruction(params.idleHours); + + const system = [ + identity, + personaBody, + memoryBlock, + stickerBlock, + timeBlock, + formatBlock, + proactiveBlock, + ] + .filter(Boolean) + .join("\n\n"); + + const messages: ChatMessage[] = [{ role: "system", content: system }]; + + for (const m of params.history) { + if (m.role === "user" || m.role === "assistant") { + messages.push(historyMessage(m.role, m.content)); + } + } + + messages.push({ + role: "user", + content: + "（系统）对方已空闲一段时间。请生成这次主动找对方聊天的消息；若不适合打扰则输出 skip JSON。", + }); + return messages; +} + +export function buildMemoryExtractMessages(params: { + history: MessageRow[]; + existing: MemoryRow[]; +}): ChatMessage[] { + const existing = + params.existing.length > 0 + ? params.existing.map((m) => `- ${m.content}`).join("\n") + : "(无)"; + const transcript = params.history + .map((m) => `${m.role}: ${m.content}`) + .join("\n"); + + return [ + { + role: "system", + content: `你是记忆整理助手。根据对话提取应长期记住的事实（昵称、偏好、关系约定）。 +规则： +- 只输出 JSON 数组，元素为中文字符串，例如 ["用户喜欢咖啡","用户叫我小铃"] +- 最多 12 条；合并重复；不要编造 +- 已有记忆可保留有用的并更新过时的`, + }, + { + role: "user", + content: `已有记忆：\n${existing}\n\n对话：\n${transcript}\n\n请输出 JSON 数组：`, + }, + ]; +} + +export function parseFactsJson(raw: string): string[] { + const text = (raw ?? "").trim(); + if (!text) return []; + let body = text; + const fence = body.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fence?.[1]) body = fence[1].trim(); + const start = body.indexOf("["); + const end = body.lastIndexOf("]"); + if (start >= 0 && end > start) body = body.slice(start, end + 1); + try { + const data = JSON.parse(body) as unknown; + if (!Array.isArray(data)) return []; + return data + .map((x) => (typeof x === "string" ? x.trim() : "")) + .filter(Boolean) + .slice(0, 12); + } catch { + return []; + } +} diff --git a/packages/core/src/reply-filter.test.ts b/packages/core/src/reply-filter.test.ts new file mode 100644 index 0000000..23809c9 --- /dev/null +++ b/packages/core/src/reply-filter.test.ts @@ -0,0 +1,220 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { flattenChatContent, type LlmClient } from "@wechat-ai/llm"; +import { + ReplyFilter, + buildReplyFilterMessages, + dropDisallowedStickers, +} from "./reply-filter.js"; + +/** ChatMessage.content is string | ChatContentPart[] since vision landed. */ +const text = (m: { content: Parameters[0] }): string => + flattenChatContent(m.content); + +class SeqFakeLlm implements Pick { + calls = 0; + constructor(private replies: string[] | string) {} + private next(): string { + this.calls++; + if (typeof this.replies === "string") return this.replies; + const i = Math.min(this.calls - 1, this.replies.length - 1); + return this.replies[i] ?? ""; + } + async chat(): Promise { + return this.next(); + } + async chatWithUsage() { + const text = this.next(); + return { + text, + promptTokens: 3, + completionTokens: 7, + totalTokens: 10, + model: "fake", + }; + } +} + +class FailingLlm implements Pick { + async chat(): Promise { + throw new Error("llm down"); + } + async chatWithUsage(): Promise { + throw new Error("llm down"); + } +} + +function asLlm(fake: Pick): LlmClient { + return fake as unknown as LlmClient; +} + +describe("buildReplyFilterMessages", () => { + it("includes caps and allowed slugs", () => { + const msgs = buildReplyFilterMessages({ + rawText: "你好呀～想你了", + allowedStickerSlugs: ["Happy-Cat", "wave"], + maxBubbles: 4, + maxChunkChars: 40, + maxStickers: 2, + }); + assert.equal(msgs.length, 2); + assert.equal(msgs[0]?.role, "system"); + assert.match(text(msgs[0]!), /最多 4 个元素/); + assert.match(text(msgs[0]!), /`happy-cat`/); + assert.match(text(msgs[0]!), /`wave`/); + assert.match(text(msgs[1]!), /你好呀/); + }); + + it("forbids stickers when maxStickers is 0", () => { + const msgs = buildReplyFilterMessages({ + rawText: "hi", + allowedStickerSlugs: ["wave"], + maxStickers: 0, + }); + assert.match(text(msgs[0]!), /禁止.*sticker/); + }); +}); + +describe("dropDisallowedStickers", () => { + it("keeps only allow-listed slugs", () => { + const parts = dropDisallowedStickers( + [ + { kind: "text", text: "a" }, + { kind: "sticker", slug: "wave" }, + { kind: "sticker", slug: "nope" }, + { kind: "text", text: "b" }, + ], + ["wave"], + ); + assert.deepEqual(parts, [ + { kind: "text", text: "a" }, + { kind: "sticker", slug: "wave" }, + { kind: "text", text: "b" }, + ]); + }); + + it("drops all stickers when allow list empty", () => { + const parts = dropDisallowedStickers( + [ + { kind: "text", text: "hi" }, + { kind: "sticker", slug: "wave" }, + ], + [], + ); + assert.deepEqual(parts, [{ kind: "text", text: "hi" }]); + }); +}); + +describe("ReplyFilter", () => { + it("parses filter JSON into ordered parts", async () => { + const llm = new SeqFakeLlm( + JSON.stringify({ + messages: [ + "好呀～", + { type: "sticker", slug: "wave" }, + "下次见", + ], + }), + ); + const filter = new ReplyFilter(asLlm(llm)); + const r = await filter.filter({ + rawText: "好呀～ [wave表情] 下次见", + allowedStickerSlugs: ["wave"], + maxStickers: 2, + }); + assert.equal(r.usedFallback, false); + assert.equal(r.fromFilterJson, true); + assert.equal(llm.calls, 1); + assert.deepEqual(r.parts, [ + { kind: "text", text: "好呀～" }, + { kind: "sticker", slug: "wave" }, + { kind: "text", text: "下次见" }, + ]); + assert.equal(r.promptTokens, 3); + assert.equal(r.completionTokens, 7); + }); + + it("falls back to rule parse when filter returns garbage", async () => { + const llm = new SeqFakeLlm("这不是json也没有结构只是一段很长的话。真的。"); + const filter = new ReplyFilter(asLlm(llm)); + const r = await filter.filter({ + rawText: "原文明天见！加油！", + maxStickers: 0, + }); + // Non-JSON filter output → parse primary raw instead + assert.ok(r.parts.length >= 1); + assert.ok(r.displayText.includes("明天见") || r.displayText.includes("加油")); + assert.equal(r.fromFilterJson, false); + assert.equal(r.usedFallback, true); + }); + + it("falls back to primary raw when LLM throws", async () => { + const filter = new ReplyFilter(asLlm(new FailingLlm())); + const r = await filter.filter({ + rawText: '{"messages":["你好","在吗"]}', + maxStickers: 0, + }); + assert.equal(r.usedFallback, true); + assert.equal(r.promptTokens, 0); + assert.deepEqual(r.bubbles, ["你好", "在吗"]); + }); + + it("drops hallucinated sticker slugs", async () => { + const llm = new SeqFakeLlm( + JSON.stringify({ + messages: [ + "看", + { type: "sticker", slug: "invented-slug" }, + { type: "sticker", slug: "wave" }, + ], + }), + ); + const filter = new ReplyFilter(asLlm(llm)); + const r = await filter.filter({ + rawText: "看", + allowedStickerSlugs: ["wave"], + maxStickers: 2, + }); + const slugs = r.parts + .filter((p) => p.kind === "sticker") + .map((p) => (p.kind === "sticker" ? p.slug : "")); + assert.deepEqual(slugs, ["wave"]); + }); + + it("converts stickers to text placeholders when maxStickers=0", async () => { + const llm = new SeqFakeLlm( + JSON.stringify({ + messages: ["hi", { type: "sticker", slug: "wave" }], + }), + ); + const filter = new ReplyFilter(asLlm(llm)); + const r = await filter.filter({ + rawText: "hi", + maxStickers: 0, + allowedStickerSlugs: ["wave"], + }); + assert.ok(r.parts.every((p) => p.kind === "text")); + assert.ok(r.displayText.includes("[表情:wave]") || r.parts.some((p) => p.kind === "text" && p.text.includes("表情"))); + }); + + it("does not call LLM when disabled", async () => { + const llm = new SeqFakeLlm("should-not-be-used"); + const filter = new ReplyFilter(asLlm(llm), { enabled: false }); + const r = await filter.filter({ + rawText: '{"messages":["a","b"]}', + maxStickers: 0, + }); + assert.equal(llm.calls, 0); + assert.equal(r.usedFallback, true); + assert.deepEqual(r.bubbles, ["a", "b"]); + assert.equal(r.promptTokens, 0); + }); + + it("returns empty for blank input", async () => { + const llm = new SeqFakeLlm("x"); + const filter = new ReplyFilter(asLlm(llm)); + const r = await filter.filter({ rawText: " " }); + assert.equal(llm.calls, 0); + assert.equal(r.parts.length, 0); + }); +}); diff --git a/packages/core/src/reply-filter.ts b/packages/core/src/reply-filter.ts new file mode 100644 index 0000000..f21c8c2 --- /dev/null +++ b/packages/core/src/reply-filter.ts @@ -0,0 +1,332 @@ +import type { ChatMessage, LlmClient } from "@wechat-ai/llm"; +import { + parseMultiBubbleReply, + type ReplyPart, +} from "./reply-format.js"; + +/** Core rules for the second-pass send-plan formatter. */ +export const REPLY_FILTER_SYSTEM_PROMPT = ` +## 角色 +你是微信消息**发送格式化器**，不是角色扮演者。 + +输入是角色已经写好的回复（可能是自然语言，也可能是杂乱 JSON）。你的任务： +- **只拆条、抽表情、去格式噪声** +- **禁止**新增剧情、禁止大幅改写口吻、禁止编造原文没有的内容 +- 若原文已是 JSON，先理解语义再规范化；**不要**把 JSON 字面量当作用户可见文字发出 + +## 输出（系统强制） +你必须**只**输出一个 JSON 对象，不要 markdown 代码块、不要前后解释： + +{"messages":["给你看～",{"type":"sticker","slug":"示例slug"},"喜欢吗"]} + +硬性规则： +1. messages 是数组；**每一个元素 = 微信里单独发出的一条消息** +2. **微信不能「图文同条」**：一条消息只能是「纯文字」或「纯图片」 +3. 元素只能是： + - 字符串 → 只发文字 + - 对象 {"type":"sticker","slug":"..."} → 只发图片（不能带任何文字） +4. **禁止**把 sticker 的 JSON 写进字符串里；禁止在 sticker 对象里塞 text/caption +5. 正常闲聊拆成 2～4 条；极短附和可用 1 条；每条文字尽量短 +6. 除该 JSON 外不要输出任何字符 +`.trim(); + +export interface ReplyFilterInput { + /** Primary model raw output (trimmed by caller or here) */ + rawText: string; + /** Allowed sticker slugs; empty / omit → no sticker objects */ + allowedStickerSlugs?: string[]; + maxBubbles?: number; + /** Soft max chars per bubble (hint for the filter model) */ + maxChunkChars?: number; + /** Cap stickers per reply (try-chat should pass 0) */ + maxStickers?: number; +} + +export interface ReplyFilterResult { + parts: ReplyPart[]; + bubbles: string[]; + displayText: string; + /** Whether filter LLM produced parseable multi-bubble JSON */ + fromFilterJson: boolean; + /** True when rule-based fallback on primary raw was used */ + usedFallback: boolean; + promptTokens: number; + completionTokens: number; +} + +export interface ReplyFilterOptions { + /** When false, skip LLM and only run parseMultiBubbleReply (default true) */ + enabled?: boolean; +} + +function emptyResult(): ReplyFilterResult { + return { + parts: [], + bubbles: [], + displayText: "", + fromFilterJson: false, + usedFallback: false, + promptTokens: 0, + completionTokens: 0, + }; +} + +function toAllowedSet( + slugs: string[] | Set | undefined, +): Set { + if (!slugs) return new Set(); + if (slugs instanceof Set) { + return new Set([...slugs].map((s) => s.trim().toLowerCase()).filter(Boolean)); + } + return new Set( + slugs.map((s) => s.trim().toLowerCase()).filter(Boolean), + ); +} + +/** + * Drop sticker parts whose slug is not in the allow-list. + * When allow-list is empty, all stickers are dropped. + */ +export function dropDisallowedStickers( + parts: ReplyPart[], + allowed: Set | string[], +): ReplyPart[] { + const set = toAllowedSet(allowed); + const out: ReplyPart[] = []; + for (const p of parts) { + if (p.kind === "text") { + if (p.text.trim()) out.push({ kind: "text", text: p.text.trim() }); + continue; + } + const slug = p.slug.trim().toLowerCase(); + if (slug && set.has(slug)) { + out.push({ kind: "sticker", slug }); + } + } + return out; +} + +/** Build chat messages for the filter LLM (pure, testable). */ +export function buildReplyFilterMessages( + input: ReplyFilterInput, +): ChatMessage[] { + const maxBubbles = input.maxBubbles ?? 5; + const maxChunkChars = input.maxChunkChars ?? 72; + const maxStickers = input.maxStickers ?? 2; + const allowed = (input.allowedStickerSlugs ?? []) + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + const uniqueAllowed = [...new Set(allowed)]; + + const constraints: string[] = [ + REPLY_FILTER_SYSTEM_PROMPT, + "", + "## 本轮约束", + `- messages 最多 ${maxBubbles} 个元素`, + `- 每条文字尽量 ≤${Math.min(40, maxChunkChars)} 字（软限制，总长可再拆）`, + `- sticker 最多 ${maxStickers} 个`, + ]; + + if (maxStickers <= 0 || uniqueAllowed.length === 0) { + constraints.push( + "- **禁止**输出任何 sticker 对象；messages 只能是字符串数组", + ); + } else { + constraints.push( + `- 可用 sticker slug（禁止编造）：${uniqueAllowed.map((s) => `\`${s}\``).join(", ")}`, + ); + } + + const system = constraints.join("\n"); + const raw = (input.rawText ?? "").trim(); + const user = [ + "## 待格式化的角色回复原文", + "请转换为规定的 messages JSON：", + "", + raw, + ].join("\n"); + + return [ + { role: "system", content: system }, + { role: "user", content: user }, + ]; +} + +function parseToResult( + raw: string, + opts: { + maxBubbles: number; + maxChunkChars: number; + maxStickers: number; + allowed: Set; + fromFilterJson: boolean; + usedFallback: boolean; + promptTokens: number; + completionTokens: number; + }, +): ReplyFilterResult { + // When maxStickers is 0 we still want to detect sticker objects and turn them + // into text placeholders rather than silently dropping them. + const parseMaxStickers = + opts.maxStickers <= 0 ? 2 : opts.maxStickers; + + const parsed = parseMultiBubbleReply(raw, { + maxBubbles: opts.maxBubbles, + maxChunkChars: opts.maxChunkChars, + maxStickers: parseMaxStickers, + fallbackSplit: true, + expandLongBubbles: true, + }); + + let parts = + parsed.parts.length > 0 + ? parsed.parts + : raw.trim() + ? [{ kind: "text" as const, text: raw.trim() }] + : []; + + if (opts.maxStickers <= 0) { + parts = parts.map((p) => + p.kind === "sticker" + ? ({ kind: "text" as const, text: `[表情:${p.slug}]` }) + : p, + ); + } else { + parts = dropDisallowedStickers(parts, opts.allowed); + } + + if (!parts.length && raw.trim()) { + const fallback = parseMultiBubbleReply(raw, { + maxBubbles: opts.maxBubbles, + maxChunkChars: opts.maxChunkChars, + maxStickers: 0, + fallbackSplit: true, + expandLongBubbles: true, + }); + parts = fallback.parts.length + ? fallback.parts + : [{ kind: "text", text: raw.trim() }]; + } + + const bubbles = parts.map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ); + const displayText = bubbles.join("\n"); + + return { + parts, + bubbles, + displayText, + fromFilterJson: opts.fromFilterJson && parsed.fromJson, + usedFallback: opts.usedFallback, + promptTokens: opts.promptTokens, + completionTokens: opts.completionTokens, + }; +} + +/** + * Second-pass AI filter: convert primary roleplay text into a WeChat send plan. + * On LLM failure or empty parseable output, falls back to rule-based parse of primary raw. + */ +export class ReplyFilter { + private enabled: boolean; + + constructor( + private llm: LlmClient, + opts: ReplyFilterOptions = {}, + ) { + this.enabled = opts.enabled !== false; + } + + /** Runtime settings reload. */ + setEnabled(enabled: boolean): void { + this.enabled = enabled; + } + + async filter(input: ReplyFilterInput): Promise { + const rawText = (input.rawText ?? "").trim(); + if (!rawText) return emptyResult(); + + const maxBubbles = input.maxBubbles ?? 5; + const maxChunkChars = input.maxChunkChars ?? 72; + const maxStickers = input.maxStickers ?? 2; + const allowed = toAllowedSet(input.allowedStickerSlugs); + + if (!this.enabled) { + return parseToResult(rawText, { + maxBubbles, + maxChunkChars, + maxStickers, + allowed, + fromFilterJson: false, + usedFallback: true, + promptTokens: 0, + completionTokens: 0, + }); + } + + try { + const messages = buildReplyFilterMessages({ + ...input, + rawText, + maxBubbles, + maxChunkChars, + maxStickers, + allowedStickerSlugs: [...allowed], + }); + // No tools — format only + const usage = await this.llm.chatWithUsage(messages); + const filteredRaw = (usage.text ?? "").trim(); + if (!filteredRaw) { + return parseToResult(rawText, { + maxBubbles, + maxChunkChars, + maxStickers, + allowed, + fromFilterJson: false, + usedFallback: true, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + }); + } + + const result = parseToResult(filteredRaw, { + maxBubbles, + maxChunkChars, + maxStickers, + allowed, + fromFilterJson: true, + usedFallback: false, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + }); + + // Filter must return parseable multi-bubble JSON; otherwise use primary raw + if (!result.fromFilterJson || !result.parts.length || !result.displayText.trim()) { + return parseToResult(rawText, { + maxBubbles, + maxChunkChars, + maxStickers, + allowed, + fromFilterJson: false, + usedFallback: true, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + }); + } + + return result; + } catch { + return parseToResult(rawText, { + maxBubbles, + maxChunkChars, + maxStickers, + allowed, + fromFilterJson: false, + usedFallback: true, + promptTokens: 0, + completionTokens: 0, + }); + } + } +} diff --git a/packages/core/src/reply-format.test.ts b/packages/core/src/reply-format.test.ts new file mode 100644 index 0000000..9e7205c --- /dev/null +++ b/packages/core/src/reply-format.test.ts @@ -0,0 +1,302 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + hasStickerTextToken, + parseMultiBubbleReply, + renderAssistantHistoryForModel, + stripAllStickerJson, +} from "./reply-format.js"; + +describe("parseMultiBubbleReply", () => { + it("parses {messages:[...]}", () => { + const r = parseMultiBubbleReply( + `{"messages":["你好呀～","今天过得怎么样？"]}`, + ); + assert.equal(r.fromJson, true); + assert.deepEqual(r.bubbles, ["你好呀～", "今天过得怎么样？"]); + assert.equal(r.displayText, "你好呀～\n今天过得怎么样？"); + assert.equal(r.parts.length, 2); + assert.equal(r.parts[0]?.kind, "text"); + }); + + it("collapses consecutive identical text bubbles", () => { + // Model sometimes emits the same string twice — shipping both is the + // byte-identical double bubble the user screenshots as a streaming bug. + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + "嗯", + "在呢\n刚改完一份排版眼睛有点花\n你说", + "在呢\n刚改完一份排版眼睛有点花\n你说", + "你说呗", + ], + }), + // Keep the long middle string as one part so the collapse is visible. + { maxBubbles: 5, expandLongBubbles: false, fallbackSplit: false }, + ); + assert.deepEqual( + r.parts.filter((p) => p.kind === "text").map((p) => (p as { text: string }).text), + [ + "嗯", + "在呢\n刚改完一份排版眼睛有点花\n你说", + "你说呗", + ], + ); + }); + + it("splits a single long string on newlines into multiple bubbles", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: ["在呢\n刚改完排版眼睛快瞎了\n你说你的"], + }), + { maxBubbles: 5, maxChunkChars: 80, expandLongBubbles: true }, + ); + assert.ok(r.parts.length >= 2, JSON.stringify(r.parts)); + assert.ok( + r.parts.every((p) => p.kind === "text" && !p.text.includes("\n")), + JSON.stringify(r.parts), + ); + }); + + it("parses fenced json", () => { + const r = parseMultiBubbleReply( + "```json\n{\"bubbles\":[\"喵\",\"在的\"]}\n```", + ); + assert.equal(r.fromJson, true); + assert.deepEqual(r.bubbles, ["喵", "在的"]); + }); + + it("parses bare array", () => { + const r = parseMultiBubbleReply(`["嗨","想你了"]`); + assert.equal(r.fromJson, true); + assert.deepEqual(r.bubbles, ["嗨", "想你了"]); + }); + + it("parses mixed text + sticker objects", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + "好呀～", + { type: "sticker", slug: "Happy-Cat" }, + "下次见", + ], + }), + ); + assert.equal(r.fromJson, true); + assert.deepEqual(r.parts, [ + { kind: "text", text: "好呀～" }, + { kind: "sticker", slug: "happy-cat" }, + { kind: "text", text: "下次见" }, + ]); + assert.equal(r.displayText, "好呀～\n[表情:happy-cat]\n下次见"); + }); + + it("accepts shorthand {sticker:slug}", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ messages: [{ sticker: "wave" }] }), + ); + assert.equal(r.parts[0]?.kind, "sticker"); + if (r.parts[0]?.kind === "sticker") { + assert.equal(r.parts[0].slug, "wave"); + } + }); + + it("extracts sticker JSON embedded inside a text string", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + '(/•/ω•/) {"type":"sticker","slug":"s-1001-sticker-v3mum2"} 好啦～给你看！', + ], + }), + ); + assert.equal(r.fromJson, true); + const kinds = r.parts.map((p) => p.kind); + assert.ok(kinds.includes("sticker"), JSON.stringify(r.parts)); + assert.ok(kinds.includes("text"), JSON.stringify(r.parts)); + const st = r.parts.find((p) => p.kind === "sticker"); + assert.equal(st && st.kind === "sticker" ? st.slug : "", "s-1001-sticker-v3mum2"); + // User-facing bubbles must not still contain raw sticker JSON + assert.ok( + !r.displayText.includes('"type"'), + r.displayText, + ); + }); + + it("never leaks sticker JSON to text even when multi-bubble has embedded form", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + "哎呀，这么喜欢我家的洗澡小猫咪呀？", + "(๑˃̵ᴗ˂̵) 好好好～", + "再给你看一遍！", + "不过就这一次啦～", + '{"type":"sticker","slug":"s-1001-sticker-v3mum2"} 嘿嘿，是不是超可爱？\n我可没骗你吧喵～', + ], + }), + { maxBubbles: 5 }, + ); + assert.ok( + r.parts.some((p) => p.kind === "sticker"), + `expected sticker part: ${JSON.stringify(r.parts)}`, + ); + for (const p of r.parts) { + if (p.kind === "text") { + assert.ok( + !/"type"\s*:\s*"sticker"/i.test(p.text), + `leaked JSON in text: ${p.text}`, + ); + assert.ok(!p.text.includes("s-1001-sticker-v3mum2") || !p.text.includes("{"), p.text); + } + } + }); + + it("caps stickers per reply", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + { type: "sticker", slug: "a" }, + { type: "sticker", slug: "b" }, + { type: "sticker", slug: "c" }, + ], + }), + { maxStickers: 2, maxBubbles: 5 }, + ); + const stickers = r.parts.filter((p) => p.kind === "sticker"); + assert.equal(stickers.length, 2); + }); + + it("falls back to text split when not json", () => { + const r = parseMultiBubbleReply("今天天气真好呢！我们去散步吧？", { + maxBubbles: 5, + }); + assert.equal(r.fromJson, false); + assert.ok(r.bubbles.length >= 1); + }); + + it("caps max bubbles", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ messages: ["1", "2", "3", "4", "5", "6"] }), + { maxBubbles: 3 }, + ); + assert.equal(r.bubbles.length, 3); + }); + + it("re-splits when model stuffs whole reply into one messages element", () => { + const essay = + "今天天气真的很好呢！我们下午去公园散步怎么样？记得带上水杯哦～"; + const r = parseMultiBubbleReply(JSON.stringify({ messages: [essay] }), { + maxBubbles: 5, + maxChunkChars: 24, + expandLongBubbles: true, + }); + assert.equal(r.fromJson, true); + assert.ok( + r.bubbles.length >= 2, + `expected >=2 bubbles, got ${r.bubbles.length}: ${JSON.stringify(r.bubbles)}`, + ); + assert.ok(!r.bubbles.some((b) => b.includes('{"messages"'))); + }); + + it("splits plain long prose without json", () => { + const r = parseMultiBubbleReply( + "第一句话在这里结束。第二句也不短一点。第三句继续说下去吧！", + { maxBubbles: 5, maxChunkChars: 20 }, + ); + assert.ok(r.bubbles.length >= 2); + }); + + // The model reads its own history, where a sticker was rendered as + // `[表情:slug]`, and imitates that notation as plain text. Until this was + // parseable it shipped to WeChat verbatim. + it("recovers a sticker from the [表情:slug] history token", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ + messages: [ + "刚才不是给你发了一个嘛", + "[表情:s-66707-sticker-tjgwdw]", + "这个可爱吧?", + ], + }), + ); + assert.deepEqual(r.parts, [ + { kind: "text", text: "刚才不是给你发了一个嘛" }, + { kind: "sticker", slug: "s-66707-sticker-tjgwdw" }, + { kind: "text", text: "这个可爱吧?" }, + ]); + }); + + it("splits a token inlined mid-sentence, and accepts full-width forms", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ messages: ["给你看～［表情：Happy-Cat］喜欢吗"] }), + ); + assert.deepEqual(r.parts, [ + { kind: "text", text: "给你看～" }, + { kind: "sticker", slug: "happy-cat" }, + { kind: "text", text: "喜欢吗" }, + ]); + }); + + it("strips a malformed token rather than sending it as text", () => { + // Slug too long for the token grammar → cannot become a sticker part. + const bad = `[表情:${"x".repeat(80)}]`; + assert.equal(stripAllStickerJson(`看这个 ${bad} 呀`), "看这个 呀"); + const r = parseMultiBubbleReply(JSON.stringify({ messages: [bad] })); + assert.deepEqual(r.parts, []); + }); + + it("leaves ordinary bracketed text alone", () => { + const r = parseMultiBubbleReply( + JSON.stringify({ messages: ["[公告] 明天见", "笑死[捂脸]"] }), + ); + assert.deepEqual(r.bubbles, ["[公告] 明天见", "笑死[捂脸]"]); + }); +}); + +describe("renderAssistantHistoryForModel", () => { + it("replays stored stickers in the format the model is told to emit", () => { + assert.equal( + renderAssistantHistoryForModel("唔 给你看个好看的\n[表情:S-66707-Sticker-Tjgwdw]"), + '唔 给你看个好看的\n{"type":"sticker","slug":"s-66707-sticker-tjgwdw"}', + ); + }); + + it("round-trips: rewritten history parses back to the same sticker", () => { + const stored = "好呀～\n[表情:happy-cat]\n下次见"; + const replayed = renderAssistantHistoryForModel(stored); + const r = parseMultiBubbleReply(JSON.stringify({ messages: [replayed] })); + assert.deepEqual(r.parts, [ + { kind: "text", text: "好呀～" }, + { kind: "sticker", slug: "happy-cat" }, + { kind: "text", text: "下次见" }, + ]); + }); + + it("leaves user text and plain assistant text untouched", () => { + assert.equal(renderAssistantHistoryForModel("在的，怎么啦"), "在的，怎么啦"); + assert.equal(renderAssistantHistoryForModel(""), ""); + }); +}); + +describe("hasStickerTextToken", () => { + it("detects every spelling the renderer or the model can produce", () => { + for (const s of [ + "[表情:happy-cat]", + "［表情：happy-cat］", + "看这个 [sticker:wave] 呀", + "[emoji: wave ]", + ]) { + assert.ok(hasStickerTextToken(s), s); + } + }); + + it("is stateless across calls", () => { + assert.ok(hasStickerTextToken("[表情:wave]")); + assert.ok(hasStickerTextToken("[表情:wave]")); + }); + + it("does not fire on ordinary text", () => { + assert.equal(hasStickerTextToken("[公告] 明天见"), false); + assert.equal(hasStickerTextToken("表情包好可爱"), false); + assert.equal(hasStickerTextToken(""), false); + }); +}); diff --git a/packages/core/src/reply-format.ts b/packages/core/src/reply-format.ts new file mode 100644 index 0000000..d91c8ad --- /dev/null +++ b/packages/core/src/reply-format.ts @@ -0,0 +1,657 @@ +import { splitReplyIntoBubbles } from "./split-reply.js"; + +/** Injected into system prompt so the model returns multi-bubble JSON. */ +export const REPLY_FORMAT_INSTRUCTION = ` +## 输出格式（系统强制，对用户不可见，优先级最高） +你必须**只**输出一个 JSON 对象：不要 markdown 代码块（禁止 \`\`\`）、不要前后解释、不要旁白。 + +{"messages":["给你看～",{"type":"sticker","slug":"示例slug"},"喜欢吗"]} + +硬性规则（违反任一条视为失败）： +1. messages 是数组；**每一个元素 = 微信里单独发出的一条消息**（一次 send） +2. **微信不能「图文同条」**：一条消息只能是「纯文字」或「纯图片」，绝不能混在同一条里 +3. 元素只能是下面两种之一（互斥）： + - 字符串 → 只发文字（不能含 sticker JSON 字面量） + - 对象 {"type":"sticker","slug":"..."} → 只发图片（该元素不能带任何文字、caption、说明） +4. 若要「先说话再发表情」：必须用**相邻的两个数组元素**，例如 "文字" 然后 {"type":"sticker",...}；系统会先后发两条消息 +5. **禁止**把 sticker 的 JSON 写进字符串里；**禁止**在 sticker 对象里塞 text/caption；**禁止**编造不在「可用表情包」列表里的 slug +6. 正常闲聊拆成 2～4 条；每条文字尽量 ≤40 字；不要小作文、不要编号列表 +7. 表情仅合适时用，每条回复最多 2 个 sticker +8. 除该 JSON 外不要输出任何字符 +`.trim(); + +/** When sticker library is empty, keep simpler instruction (no sticker objects). */ +export const REPLY_FORMAT_INSTRUCTION_TEXT_ONLY = ` +## 输出格式（系统强制，对用户不可见，优先级最高） +你必须**只**输出一个 JSON 对象：不要 markdown 代码块（禁止 \`\`\`）、不要前后解释、不要旁白。 + +{"messages":["第一句","第二句","第三句"]} + +硬性规则（违反任一条视为失败）： +1. messages 是字符串数组；**每一条数组元素 = 微信里单独发出的一条气泡** +2. **禁止**把整段回复塞进 messages 的唯一一个元素；正常闲聊必须拆成 **2～4 条**（极短附和可用 1 条） +3. 每条尽量 ≤40 字，口语、可情绪递进；不要小作文、不要编号「1. 2.」 +4. 内容只能是角色对白；禁止出现系统说明、格式说明、JSON 字样 +5. 除该 JSON 外不要输出任何字符 +`.trim(); + +export type ReplyPart = + | { kind: "text"; text: string } + | { kind: "sticker"; slug: string }; + +export interface ParsedBubbles { + /** + * Ordered WeChat bubbles including stickers. + * Prefer this for sending. + */ + parts: ReplyPart[]; + /** + * Text-only bubbles (sticker rendered as `[表情:slug]`). + * Kept for backward compatibility / split heuristics consumers. + */ + bubbles: string[]; + /** Plain text for history / logs (joined) */ + displayText: string; + /** Whether parsed from model JSON (vs heuristic split) */ + fromJson: boolean; +} + +export interface ParseMultiBubbleOptions { + maxBubbles?: number; + /** Soft max chars per bubble; longer content is re-split (default 72) */ + maxChunkChars?: number; + /** When JSON missing, split plain text (default true) */ + fallbackSplit?: boolean; + /** + * Re-split long bubbles even if model returned JSON (default true). + * Fixes models that put the whole reply in messages[0]. + */ + expandLongBubbles?: boolean; + /** Cap stickers per reply (default 2) */ + maxStickers?: number; +} + +/** + * Parse model output into multiple chat bubbles (text + optional stickers). + * Preferred: {"messages":["..."]} or mixed sticker objects + * Fallback: punctuation/paragraph split of raw text. + */ +export function parseMultiBubbleReply( + raw: string, + opts?: ParseMultiBubbleOptions, +): ParsedBubbles { + const maxBubbles = opts?.maxBubbles ?? 5; + const maxChunkChars = opts?.maxChunkChars ?? 72; + const fallbackSplit = opts?.fallbackSplit !== false; + const expandLong = opts?.expandLongBubbles !== false; + const maxStickers = opts?.maxStickers ?? 2; + const text = (raw ?? "").trim(); + if (!text) { + return { parts: [], bubbles: [], displayText: "", fromJson: false }; + } + + let fromJson = false; + let parts: ReplyPart[] = []; + + const parsed = tryParseJsonParts(text, maxBubbles, maxStickers); + if (parsed) { + fromJson = true; + parts = parsed; + } else if (fallbackSplit) { + const bubbles = splitReplyIntoBubbles(text, { + maxChunks: maxBubbles, + maxChunkChars, + minChunkChars: 6, + }); + parts = (bubbles.length ? bubbles : [text]).map((t) => ({ + kind: "text" as const, + text: t, + })); + } else { + parts = [{ kind: "text", text }]; + } + + // Collapse before expand so a model that emitted the same multi-line + // string twice (messages:["A\nB","A\nB"]) does not expand into A,B,A,B. + parts = collapseConsecutiveTextDupes(parts); + + if (expandLong) { + parts = expandLongTextParts(parts, maxBubbles, maxChunkChars); + } + + // Always re-scan text for inlined sticker JSON (models ignore format rules) + // and NEVER leave raw {"type":"sticker",...} in user-visible text. + parts = sanitizePartsStripStickerJson(parts, maxStickers); + + // Cap total parts (preserve stickers — do not drop into text merge) + if (parts.length > maxBubbles) { + parts = capParts(parts, maxBubbles); + } + + parts = finalizeParts(parts); + // Final pass after cap (cap may merge text) + parts = sanitizePartsStripStickerJson(parts, maxStickers); + parts = finalizeParts(parts); + // Models occasionally emit the same string twice in messages[]; collapse + // so the worker never ships two identical WeChat bubbles back-to-back. + // Run after expandLong so "foo\nbar"+"foo\nbar" → (foo,bar,foo,bar) still + // collapses consecutive equals; also covers the pre-expand shape when + // expandLong is off. + parts = collapseConsecutiveTextDupes(parts); + + if (!parts.length) { + // An envelope that reduced to nothing is a no-op reply. `text` is the JSON + // wrapper itself here, so there is nothing safe to salvage from it. + if (fromJson) { + return { parts: [], bubbles: [], displayText: "", fromJson }; + } + // Last resort: strip raw sticker JSON from whole reply text + const cleaned = stripAllStickerJson(text).trim(); + return { + parts: cleaned ? [{ kind: "text", text: cleaned }] : [], + bubbles: cleaned ? [cleaned] : [], + displayText: cleaned, + fromJson, + }; + } + + return { + parts, + bubbles: partsToLegacyBubbles(parts), + displayText: partsToDisplayText(parts), + fromJson, + }; +} + +/** + * Public helper: ensure no sticker JSON leaks into text bubbles. + * Safe to call again in the worker before send. + */ +export function sanitizePartsStripStickerJson( + parts: ReplyPart[], + maxStickers = 2, +): ReplyPart[] { + const out: ReplyPart[] = []; + let stickerCount = 0; + for (const p of parts) { + if (p.kind === "sticker") { + if (stickerCount >= maxStickers) continue; + const slug = p.slug.trim().toLowerCase(); + if (slug) { + out.push({ kind: "sticker", slug }); + stickerCount++; + } + continue; + } + const extracted = extractEmbeddedStickersFromText( + p.text, + maxStickers - stickerCount, + ); + for (const e of extracted) { + if (e.kind === "sticker") { + if (stickerCount >= maxStickers) continue; + out.push(e); + stickerCount++; + } else { + const cleaned = stripAllStickerJson(e.text).trim(); + if (cleaned) out.push({ kind: "text", text: cleaned }); + } + } + } + return out; +} + +/** + * `[表情:slug]` — the notation partsToDisplayText writes for history and logs. + * + * It has to be readable as *input* too, not just written as output: the stored + * assistant turn is replayed to the model verbatim (see prompt.ts), so the + * model reads its own past stickers in this shape and imitates it as plain + * text. Every other guard here is anchored on `{`, so before this existed such + * a bubble matched nothing and shipped to WeChat as the literal `[表情:s-…]`. + * + * Full-width brackets/colon and the `[sticker:x]` spelling are accepted too — + * the model is copying by eye, not by grammar. + */ +const STICKER_TOKEN_SOURCE = + "[\\[［]\\s*(?:表情包|表情|sticker|emoji|emoticon)\\s*[:：]\\s*([a-zA-Z0-9_-]{2,64})\\s*[\\]］]"; + +/** + * Looser shape, for scrubbing and detection only. + * + * The parse form above is deliberately strict because its capture is looked up + * as a real slug. Anything merely token-shaped — a truncated or over-long slug, + * a model that spelled it wrong — can never become a sticker, and so must be + * deleted rather than handed to WeChat as text. + */ +const STICKER_TOKEN_LOOSE_SOURCE = + "[\\[［]\\s*(?:表情包|表情|sticker|emoji|emoticon)\\s*[:：][^\\]］\\n]{0,120}[\\]］]"; + +/** Stateless probe: does this text still carry a sticker token of any shape? */ +export function hasStickerTextToken(text: string): boolean { + return new RegExp(STICKER_TOKEN_LOOSE_SOURCE, "i").test(text ?? ""); +} + +/** + * Rewrite stored `[表情:slug]` tokens into the sticker JSON the output format + * actually mandates, so replayed history never teaches the model a notation it + * is forbidden to emit. Sticker JSON inlined in a text string is a shape the + * parser already recovers from (extractEmbeddedStickersFromText); the bracket + * token was not. + */ +export function renderAssistantHistoryForModel(content: string): string { + if (!content) return content; + return content.replace( + new RegExp(STICKER_TOKEN_SOURCE, "gi"), + (_full, slug: string) => `{"type":"sticker","slug":"${slug.toLowerCase()}"}`, + ); +} + +/** Remove any leftover sticker-shaped JSON fragments from a text bubble. */ +export function stripAllStickerJson(text: string): string { + if (!text) return ""; + let s = text; + // Full object forms + s = s.replace( + /\{\s*["'“”]type["'“”]\s*:\s*["'“”](?:sticker|emoji|image|emoticon)["'“”]\s*,\s*["'“”]slug["'“”]\s*:\s*["'“”][a-zA-Z0-9_-]+["'“”]\s*\}/gi, + " ", + ); + s = s.replace( + /\{\s*["'“”]slug["'“”]\s*:\s*["'“”][a-zA-Z0-9_-]+["'“”]\s*,\s*["'“”]type["'“”]\s*:\s*["'“”](?:sticker|emoji|image|emoticon)["'“”]\s*\}/gi, + " ", + ); + s = s.replace( + /\{\s*["'“”]sticker["'“”]\s*:\s*["'“”][a-zA-Z0-9_-]+["'“”]\s*\}/gi, + " ", + ); + // Truncated / broken fragments models sometimes emit mid-sentence + s = s.replace( + /\{\s*["'“”]?type["'“”]?\s*:\s*["'“”]?sticker["'“”]?[^}]{0,80}\}/gi, + " ", + ); + // Bracket token, last resort only: extractEmbeddedStickersFromText claims + // well-formed ones as real stickers first, so whatever is still here is + // malformed or unusable and must not reach WeChat as text. + s = s.replace(new RegExp(STICKER_TOKEN_LOOSE_SOURCE, "gi"), " "); + s = s.replace(/\s{2,}/g, " ").trim(); + return s; +} + +export function partsToDisplayText(parts: ReplyPart[]): string { + return parts + .map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ) + .filter(Boolean) + .join("\n"); +} + +export function partsToLegacyBubbles(parts: ReplyPart[]): string[] { + return parts.map((p) => + p.kind === "text" ? p.text : `[表情:${p.slug}]`, + ); +} + +function finalizeParts(parts: ReplyPart[]): ReplyPart[] { + const out: ReplyPart[] = []; + for (const p of parts) { + if (p.kind === "sticker") { + const slug = p.slug.trim().toLowerCase(); + if (slug) out.push({ kind: "sticker", slug }); + continue; + } + const t = p.text.trim(); + if (t) out.push({ kind: "text", text: t }); + } + return out; +} + +/** + * Drop consecutive identical text bubbles. + * + * Models sometimes emit the same string twice in `messages` (or the heuristic + * splitter produces a near-empty duplicate). Shipping both as separate WeChat + * messages is what the user sees as "流式坏了" — two identical bubbles in a row. + * Stickers are left alone; only pure-text neighbours are collapsed. + */ +function collapseConsecutiveTextDupes(parts: ReplyPart[]): ReplyPart[] { + if (parts.length <= 1) return parts; + const out: ReplyPart[] = []; + for (const p of parts) { + const prev = out[out.length - 1]; + if ( + prev && + prev.kind === "text" && + p.kind === "text" && + prev.text === p.text + ) { + continue; + } + out.push(p); + } + return out; +} + +function capParts(parts: ReplyPart[], maxBubbles: number): ReplyPart[] { + if (parts.length <= maxBubbles) return parts; + // Prefer keeping stickers: take first N parts but if we cut mid-way, + // append any remaining stickers that still fit by swapping trailing text. + const kept = parts.slice(0, maxBubbles); + const rest = parts.slice(maxBubbles); + const extraStickers = rest.filter( + (p): p is { kind: "sticker"; slug: string } => p.kind === "sticker", + ); + if (!extraStickers.length) return kept; + + const out = [...kept]; + for (const st of extraStickers) { + // Replace last pure-text bubble with sticker if over cap + if (out.length < maxBubbles) { + out.push(st); + continue; + } + let replaced = false; + for (let i = out.length - 1; i >= 0; i--) { + if (out[i]!.kind === "text") { + out[i] = st; + replaced = true; + break; + } + } + if (!replaced) break; + } + // Append remaining text from rest into last text if any room conceptually + const textTail = rest + .filter((p): p is { kind: "text"; text: string } => p.kind === "text") + .map((p) => p.text) + .join(""); + if (textTail.trim()) { + const lastTextIdx = [...out] + .map((p, i) => (p.kind === "text" ? i : -1)) + .filter((i) => i >= 0) + .pop(); + if (lastTextIdx !== undefined) { + const cur = out[lastTextIdx] as { kind: "text"; text: string }; + out[lastTextIdx] = { + kind: "text", + text: (cur.text + textTail).trim(), + }; + } else if (out.length < maxBubbles) { + out.push({ kind: "text", text: textTail.trim() }); + } + } + return out; +} + +/** + * If the model stuffed everything into one (or few) long string(s), + * split further so the worker can send multiple WeChat messages. + * Stickers are left intact. + */ +function expandLongTextParts( + parts: ReplyPart[], + maxBubbles: number, + maxChunkChars: number, +): ReplyPart[] { + if (!parts.length) return parts; + + const textOnly = parts.every((p) => p.kind === "text"); + if (textOnly && parts.length === 1) { + const only = (parts[0] as { kind: "text"; text: string }).text; + if ( + only.length > maxChunkChars || + /[。！？!?]/.test(only) || + /\n/.test(only) + ) { + const split = splitReplyIntoBubbles(only, { + maxChunks: maxBubbles, + maxChunkChars, + minChunkChars: 6, + }); + if (split.length > 1) { + return split.map((t) => ({ kind: "text" as const, text: t })); + } + } + return parts; + } + + const out: ReplyPart[] = []; + for (const p of parts) { + if (p.kind === "sticker") { + out.push(p); + continue; + } + if (p.text.length <= maxChunkChars * 1.2) { + out.push(p); + continue; + } + const split = splitReplyIntoBubbles(p.text, { + maxChunks: maxBubbles, + maxChunkChars, + minChunkChars: 6, + }); + if (split.length) { + for (const t of split) out.push({ kind: "text", text: t }); + } else { + out.push(p); + } + } + if (out.length > maxBubbles) { + return capParts(out, maxBubbles); + } + return out; +} + +function tryParseJsonParts( + text: string, + maxBubbles: number, + maxStickers: number, +): ReplyPart[] | null { + // strip optional ```json ... ``` + let body = text; + const fence = body.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fence?.[1]) body = fence[1].trim(); + + // Some models wrap with leading chatter — keep from first { or [ + const objStart = body.indexOf("{"); + const arrStart = body.indexOf("["); + let candidate = body; + if (objStart >= 0 && (arrStart < 0 || objStart < arrStart)) { + const end = body.lastIndexOf("}"); + if (end > objStart) candidate = body.slice(objStart, end + 1); + } else if (arrStart >= 0) { + const end = body.lastIndexOf("]"); + if (end > arrStart) candidate = body.slice(arrStart, end + 1); + } + + // Normalize common model quirks + candidate = candidate + .replace(/[\u201c\u201d]/g, '"') // “ ” + .replace(/[\u2018\u2019]/g, "'") // ‘ ’ + .replace(/,\s*([}\]])/g, "$1"); // trailing commas + + try { + const data = JSON.parse(candidate) as unknown; + // A recognised envelope wins even when it reduces to nothing usable — + // returning null there would fall through to the raw-text splitter and + // ship `{"messages":[…]}` to WeChat verbatim. + return normalizePartList(data, maxBubbles, maxStickers); + } catch { + return null; + } +} + +/** null when this is not a bubble envelope at all (vs. an empty one). */ +function normalizePartList( + data: unknown, + maxBubbles: number, + maxStickers: number, +): ReplyPart[] | null { + let arr: unknown[] | null = null; + if (Array.isArray(data)) { + arr = data; + } else if (data && typeof data === "object") { + const o = data as Record; + if (Array.isArray(o.messages)) arr = o.messages; + else if (Array.isArray(o.bubbles)) arr = o.bubbles; + else if (Array.isArray(o.replies)) arr = o.replies; + else if (typeof o.message === "string") arr = [o.message]; + else if (typeof o.content === "string") arr = [o.content]; + else if (typeof o.text === "string") arr = [o.text]; + } + if (!arr) return null; + + const parts: ReplyPart[] = []; + let stickerCount = 0; + for (const x of arr) { + if (parts.length >= maxBubbles) break; + if (typeof x === "string") { + // Models often embed {"type":"sticker","slug":"..."} inside a text string + const extracted = extractEmbeddedStickersFromText(x, maxStickers - stickerCount); + for (const p of extracted) { + if (parts.length >= maxBubbles) break; + if (p.kind === "sticker") { + if (stickerCount >= maxStickers) continue; + parts.push(p); + stickerCount++; + } else if (p.text.trim()) { + parts.push({ kind: "text", text: p.text.trim() }); + } + } + continue; + } + if (x && typeof x === "object") { + const o = x as Record; + const stickerSlug = extractStickerSlug(o); + if (stickerSlug) { + if (stickerCount >= maxStickers) continue; + parts.push({ kind: "sticker", slug: stickerSlug }); + stickerCount++; + continue; + } + if (typeof o.text === "string") { + const extracted = extractEmbeddedStickersFromText( + o.text, + maxStickers - stickerCount, + ); + for (const p of extracted) { + if (parts.length >= maxBubbles) break; + if (p.kind === "sticker") { + if (stickerCount >= maxStickers) continue; + parts.push(p); + stickerCount++; + } else if (p.text.trim()) { + parts.push({ kind: "text", text: p.text.trim() }); + } + } + continue; + } + if (typeof o.content === "string") { + const t = o.content.trim(); + if (t) parts.push({ kind: "text", text: t }); + continue; + } + } + } + return parts; +} + +/** + * Split a model text bubble that illegally inlined sticker JSON objects. + * e.g. `(/ω\\) {"type":"sticker","slug":"x"} 好啦` → text + sticker + text + */ +function extractEmbeddedStickersFromText( + text: string, + maxStickers: number, +): ReplyPart[] { + const raw = text ?? ""; + if (!raw.trim()) return []; + + // Flexible quotes: " ' “ ” + const Q = `["'“”]`; + const re = + new RegExp( + `\\{\\s*${Q}type${Q}\\s*:\\s*${Q}(?:sticker|emoji|image|emoticon)${Q}\\s*,\\s*${Q}slug${Q}\\s*:\\s*${Q}([a-zA-Z0-9_-]+)${Q}\\s*\\}`, + "gi", + ); + const reAlt = new RegExp( + `\\{\\s*${Q}sticker${Q}\\s*:\\s*${Q}([a-zA-Z0-9_-]+)${Q}\\s*\\}`, + "gi", + ); + const reSlugFirst = new RegExp( + `\\{\\s*${Q}slug${Q}\\s*:\\s*${Q}([a-zA-Z0-9_-]+)${Q}\\s*,\\s*${Q}type${Q}\\s*:\\s*${Q}(?:sticker|emoji|image|emoticon)${Q}\\s*\\}`, + "gi", + ); + // Loose: type/sticker anywhere then slug nearby inside braces + const reLoose = + /\{\s*[^}]{0,40}type[^}]{0,20}sticker[^}]{0,40}slug[^}]{0,10}["'“”]?([a-zA-Z0-9_-]{2,64})["'“”]?[^}]*\}/gi; + // Bracket token the history renderer emits — see STICKER_TOKEN_SOURCE + const reToken = new RegExp(STICKER_TOKEN_SOURCE, "gi"); + + type Hit = { start: number; end: number; slug: string }; + const hits: Hit[] = []; + for (const pattern of [re, reAlt, reSlugFirst, reLoose, reToken]) { + pattern.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = pattern.exec(raw)) !== null) { + hits.push({ + start: m.index, + end: m.index + m[0].length, + slug: m[1]!.toLowerCase(), + }); + } + } + if (!hits.length) { + // Still strip any garbage so raw JSON never goes to WeChat as text + const cleaned = stripAllStickerJson(raw); + return cleaned ? [{ kind: "text", text: cleaned }] : []; + } + hits.sort((a, b) => a.start - b.start); + const uniq: Hit[] = []; + for (const h of hits) { + if (uniq.some((u) => !(h.end <= u.start || h.start >= u.end))) continue; + uniq.push(h); + } + + const parts: ReplyPart[] = []; + let cursor = 0; + let used = 0; + for (const h of uniq) { + if (h.start > cursor) { + const before = stripAllStickerJson(raw.slice(cursor, h.start)).trim(); + if (before) parts.push({ kind: "text", text: before }); + } + if (used < maxStickers) { + parts.push({ kind: "sticker", slug: h.slug }); + used++; + } + cursor = h.end; + } + if (cursor < raw.length) { + const after = stripAllStickerJson(raw.slice(cursor)).trim(); + if (after) parts.push({ kind: "text", text: after }); + } + return parts.length ? parts : []; +} + +function extractStickerSlug(o: Record): string | null { + const type = typeof o.type === "string" ? o.type.toLowerCase() : ""; + if ( + type === "sticker" || + type === "emoji" || + type === "image" || + type === "emoticon" + ) { + const slug = o.slug ?? o.id ?? o.name; + if (typeof slug === "string" && slug.trim()) { + return slug.trim().toLowerCase(); + } + } + if (typeof o.sticker === "string" && o.sticker.trim()) { + return o.sticker.trim().toLowerCase(); + } + if (typeof o.emoji === "string" && o.emoji.trim()) { + return o.emoji.trim().toLowerCase(); + } + return null; +} diff --git a/packages/core/src/split-reply.test.ts b/packages/core/src/split-reply.test.ts new file mode 100644 index 0000000..4d2f99f --- /dev/null +++ b/packages/core/src/split-reply.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { humanDelayMs, splitReplyIntoBubbles } from "./split-reply.js"; + +describe("splitReplyIntoBubbles", () => { + it("keeps short text as one bubble", () => { + assert.deepEqual(splitReplyIntoBubbles("你好呀～"), ["你好呀～"]); + }); + + it("splits by sentences", () => { + const parts = splitReplyIntoBubbles( + "今天天气真好呢！我们去散步吧？记得带伞哦。", + { maxChunkChars: 20, maxChunks: 5, minChunkChars: 4 }, + ); + assert.ok(parts.length >= 2); + assert.equal(parts.join(""), "今天天气真好呢！我们去散步吧？记得带伞哦。"); + }); + + it("splits paragraphs", () => { + const parts = splitReplyIntoBubbles("第一段内容比较长一点的。\n\n第二段也要单独发。", { + maxChunkChars: 40, + }); + assert.ok(parts.length >= 2); + }); + + it("splits on a single newline even when the whole string is short", () => { + // Models often stuff multi-bubble intent into one string with `\n`. + // That used to ship as one WeChat bubble; must become three. + const parts = splitReplyIntoBubbles("在呢\n刚改完排版眼睛有点花\n你说", { + maxChunkChars: 80, + maxChunks: 5, + minChunkChars: 2, + }); + assert.deepEqual(parts, ["在呢", "刚改完排版眼睛有点花", "你说"]); + }); + + it("caps max chunks", () => { + const long = Array.from({ length: 10 }, (_, i) => `句子${i}号。`).join(""); + const parts = splitReplyIntoBubbles(long, { + maxChunks: 3, + maxChunkChars: 8, + minChunkChars: 2, + }); + assert.ok(parts.length <= 3); + }); +}); + +describe("humanDelayMs", () => { + it("returns positive delays", () => { + assert.ok(humanDelayMs("你好", 0) >= 900); + const d = humanDelayMs("这是一条稍微长一点的消息内容用来测延迟", 1); + assert.ok(d >= 1000 && d <= 6000); + }); +}); diff --git a/packages/core/src/split-reply.ts b/packages/core/src/split-reply.ts new file mode 100644 index 0000000..10fe1ff --- /dev/null +++ b/packages/core/src/split-reply.ts @@ -0,0 +1,183 @@ +/** + * Split a long assistant reply into multiple chat bubbles to feel human. + */ + +export interface SplitReplyOptions { + /** Max bubbles (default 5) */ + maxChunks?: number; + /** Prefer not to produce tiny fragments (default 8) */ + minChunkChars?: number; + /** Soft max chars per bubble before force-split (default 80) */ + maxChunkChars?: number; +} + +/** Must consume the terminator so split loops always advance. */ +const SENTENCE_END = /[。！？!?…～~]+["'」』）)\]]*\s*/g; +const CLAUSE_END = /[；;，,]+\s*/; + +/** + * Split reply text into ordered chunks for multi-message send. + */ +export function splitReplyIntoBubbles( + text: string, + opts: SplitReplyOptions = {}, +): string[] { + const maxChunks = opts.maxChunks ?? 5; + const minChunkChars = opts.minChunkChars ?? 8; + const maxChunkChars = opts.maxChunkChars ?? 80; + + const normalized = text.replace(/\r\n/g, "\n").trim(); + if (!normalized) return []; + + // Short enough and no line breaks — single bubble. + // A single `\n` is a deliberate bubble boundary (models often write + // "在呢\n刚改完排版\n你说" as one string); only skip the early-return when + // there is no newline at all. + if (normalized.length <= maxChunkChars && !/\n/.test(normalized)) { + return [normalized]; + } + + let parts: string[] = []; + + // 1) any newline (single or double) is a bubble boundary + if (/\n/.test(normalized)) { + parts = normalized + .split(/\n+/) + .map((p) => p.trim()) + .filter(Boolean); + } else { + // 2) sentences + parts = splitByRegex(normalized, SENTENCE_END); + } + + // 3) further split long paragraphs by clauses / length + parts = parts.flatMap((p) => splitLongPart(p, maxChunkChars)); + + // merge tiny fragments into previous + parts = mergeTiny(parts, minChunkChars); + + // cap chunk count by merging overflow into last + if (parts.length > maxChunks) { + const head = parts.slice(0, maxChunks - 1); + const tail = parts.slice(maxChunks - 1).join(""); + parts = [...head, tail]; + } + + return parts.map((p) => p.trim()).filter(Boolean); +} + +function splitByRegex(text: string, re: RegExp): string[] { + const out: string[] = []; + let last = 0; + const r = new RegExp( + re.source, + re.flags.includes("g") ? re.flags : `${re.flags}g`, + ); + let m: RegExpExecArray | null; + while ((m = r.exec(text)) !== null) { + // Prevent zero-length match infinite loops + if (m[0].length === 0) { + r.lastIndex += 1; + continue; + } + const end = m.index + m[0].length; + const chunk = text.slice(last, end).trim(); + if (chunk) out.push(chunk); + last = end; + } + const rest = text.slice(last).trim(); + if (rest) out.push(rest); + return out.length ? out : [text]; +} + +function splitLongPart(part: string, maxChunkChars: number): string[] { + if (part.length <= maxChunkChars) return [part]; + + // try clauses + const clauses = part.split(CLAUSE_END).map((s) => s.trim()).filter(Boolean); + if (clauses.length > 1) { + const rebuilt: string[] = []; + let buf = ""; + for (const c of clauses) { + const next = buf ? buf + c : c; + if (next.length > maxChunkChars && buf) { + rebuilt.push(buf); + buf = c; + } else { + buf = next; + } + } + if (buf) rebuilt.push(buf); + return rebuilt.flatMap((p) => + p.length > maxChunkChars * 1.5 + ? hardSlice(p, maxChunkChars) + : [p], + ); + } + + return hardSlice(part, maxChunkChars); +} + +function hardSlice(text: string, size: number): string[] { + const out: string[] = []; + for (let i = 0; i < text.length; i += size) { + out.push(text.slice(i, i + size)); + } + return out; +} + +function mergeTiny(parts: string[], minChunkChars: number): string[] { + if (parts.length <= 1) return parts; + const out: string[] = []; + for (const p of parts) { + if (out.length && p.length < minChunkChars) { + out[out.length - 1] = out[out.length - 1] + p; + } else { + out.push(p); + } + } + return out; +} + +export interface HumanDelayOptions { + /** ms per character when "typing" (default 90) */ + msPerChar?: number; + /** minimum delay between bubbles (default 1400) */ + minMs?: number; + /** maximum delay between bubbles (default 5500) */ + maxMs?: number; + /** first bubble delay after LLM ready (default 900–2200) */ + firstMinMs?: number; + firstMaxMs?: number; + /** extra "think" pause before 2nd+ bubbles (default 400) */ + thinkExtraMs?: number; +} + +/** Delay before sending a bubble, based on length + jitter (simulates human typing). */ +export function humanDelayMs( + chunk: string, + index: number, + opts: HumanDelayOptions = {}, +): number { + const msPerChar = opts.msPerChar ?? 90; + const minMs = opts.minMs ?? 1400; + const maxMs = opts.maxMs ?? 5500; + const firstMin = opts.firstMinMs ?? 900; + const firstMax = opts.firstMaxMs ?? 2200; + const thinkExtra = opts.thinkExtraMs ?? 400; + + if (index === 0) { + // Read message + start typing + return rand(firstMin, firstMax); + } + // Subsequent: typing time + small think pause + jitter + const typed = Math.round(chunk.length * msPerChar) + thinkExtra; + const base = Math.min(maxMs, Math.max(minMs, typed)); + // ±25% jitter so it doesn't feel mechanical + const jitter = base * (0.75 + Math.random() * 0.5); + return Math.round(jitter); +} + +function rand(a: number, b: number): number { + return Math.round(a + Math.random() * (b - a)); +} diff --git a/packages/core/src/try-chat-service.test.ts b/packages/core/src/try-chat-service.test.ts new file mode 100644 index 0000000..fef3210 --- /dev/null +++ b/packages/core/src/try-chat-service.test.ts @@ -0,0 +1,12 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { TryChatError } from "./try-chat-service.js"; + +describe("TryChatError", () => { + it("carries code", () => { + const e = new TryChatError("quota_day", "今日已满"); + assert.equal(e.code, "quota_day"); + assert.equal(e.message, "今日已满"); + assert.equal(e.name, "TryChatError"); + }); +}); diff --git a/packages/core/src/try-chat-service.ts b/packages/core/src/try-chat-service.ts new file mode 100644 index 0000000..a7bc183 --- /dev/null +++ b/packages/core/src/try-chat-service.ts @@ -0,0 +1,487 @@ +import { + type Db, + type MessageRow, + type Persona, + appendTryChatMessages, + createTryChatSession, + deleteTryChatSession, + getPersona, + getPublishedGraph, + getPublishedPrompt, + getTryChatDayCount, + getTryChatSession, + incrTryChatDayCount, + listTryChatMessages, + recordTokenUsage, + saveTryChatSession, + userCanUsePersona, +} from "@wechat-ai/db"; +import type { LlmClient } from "@wechat-ai/llm"; +import { buildChatMessages } from "./prompt.js"; +import { ChatflowEngine } from "./chatflow/engine.js"; +import { + parseMultiBubbleReply, + type ReplyPart, +} from "./reply-format.js"; +import { ReplyFilter } from "./reply-filter.js"; + +export interface TryChatServiceOptions { + sessionTtlSec: number; + maxHistory: number; + maxUserMsgsPerDay: number; + maxUserMsgsPerSession: number; + multiBubbleJson?: boolean; + maxReplyBubbles?: number; + maxChunkChars?: number; + /** + * Second-pass AI filter for multi-bubble JSON (default false). + * When true, primary model does not receive REPLY_FORMAT_INSTRUCTION. + */ + replyFilterEnabled?: boolean; + timeToolEnabled?: boolean; + timeToolTimeZone?: string; + /** + * Chatflow personas: run the published graph in try-chat too. + * Always platform upstream — never the author's custom provider. + */ + toolsBaseUrl?: string; + toolsApiKey?: string; + /** Wall clock for one tools-gateway search call */ + toolsTimeoutMs?: number; + webSearchEnabled?: boolean; + /** Default result count for chatflow search nodes (default 5) */ + webSearchMaxResults?: number; + chatflowHttpAllowHosts?: string[]; + chatflowMaxSteps?: number; + chatflowMaxNodes?: number; +} + +export interface StartTrySessionInput { + userId: string; + personaId: string; + botName?: string; +} + +export interface StartTrySessionResult { + sessionId: string; + persona: Pick; + botName: string; + expiresInSec: number; + remainingToday: number; +} + +export interface SendTryMessageInput { + userId: string; + sessionId: string; + text: string; + /** Optional username for usage stats */ + username?: string; +} + +export interface SendTryMessageResult { + parts: ReplyPart[]; + displayText: string; + usage: { + promptTokens: number; + completionTokens: number; + totalTokens: number; + }; + remainingToday: number; + remainingSession: number; +} + +export class TryChatError extends Error { + constructor( + public code: + | "disabled" + | "not_found" + | "forbidden" + | "quota_day" + | "quota_session" + | "empty" + | "llm" + | "no_prompt", + message: string, + ) { + super(message); + this.name = "TryChatError"; + } +} + +const DEFAULTS: TryChatServiceOptions = { + sessionTtlSec: 3600, + maxHistory: 40, + maxUserMsgsPerDay: 40, + maxUserMsgsPerSession: 20, + multiBubbleJson: true, + maxReplyBubbles: 5, + maxChunkChars: 72, + replyFilterEnabled: false, + timeToolEnabled: true, + timeToolTimeZone: "Asia/Shanghai", +}; + +export class TryChatService { + private opts: TryChatServiceOptions; + private replyFilter: ReplyFilter; + private chatflow: ChatflowEngine; + + constructor( + private db: Db, + private llm: LlmClient, + opts: Partial = {}, + ) { + this.opts = { ...DEFAULTS, ...opts }; + this.replyFilter = new ReplyFilter(llm, { + enabled: this.opts.replyFilterEnabled === true, + }); + this.chatflow = new ChatflowEngine({ + platformLlm: llm, + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + toolsTimeoutMs: this.opts.toolsTimeoutMs, + webSearchEnabled: this.opts.webSearchEnabled === true, + webSearchMaxResults: this.opts.webSearchMaxResults, + maxSteps: this.opts.chatflowMaxSteps ?? 32, + maxNodes: this.opts.chatflowMaxNodes ?? 40, + httpAllowHosts: this.opts.chatflowHttpAllowHosts, + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }); + } + + /** + * Apply admin-editable settings in place (runtime settings reload). + * Only the keys present in `patch` are touched. + */ + applyRuntimeOptions(patch: Partial): void { + Object.assign(this.opts, patch); + if ("replyFilterEnabled" in patch) { + this.replyFilter.setEnabled(this.opts.replyFilterEnabled === true); + } + this.chatflow.applyOptions({ + toolsBaseUrl: this.opts.toolsBaseUrl, + toolsApiKey: this.opts.toolsApiKey, + toolsTimeoutMs: this.opts.toolsTimeoutMs, + webSearchEnabled: this.opts.webSearchEnabled === true, + webSearchMaxResults: this.opts.webSearchMaxResults, + maxSteps: this.opts.chatflowMaxSteps ?? 32, + maxNodes: this.opts.chatflowMaxNodes ?? 40, + httpAllowHosts: this.opts.chatflowHttpAllowHosts, + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }); + } + + private primaryMultiBubbleJson(): boolean { + if (this.opts.replyFilterEnabled === true) return false; + return this.opts.multiBubbleJson !== false; + } + + async startSession( + input: StartTrySessionInput, + ): Promise { + const persona = await getPersona(this.db, input.personaId); + if (!persona || !persona.enabled) { + throw new TryChatError("not_found", "人设不存在或已下架"); + } + if (!(await userCanUsePersona(this.db, input.userId, persona.id))) { + // Public personas are usable without library for try-chat browse UX + if (!(persona.visibility === "public" && persona.enabled)) { + throw new TryChatError("forbidden", "无权试聊该人设"); + } + } + const prompt = await getPublishedPrompt(this.db, persona.id); + if (!prompt?.trim()) { + throw new TryChatError("no_prompt", "人设尚未发布可用提示词"); + } + + const usedToday = await getTryChatDayCount(this.db, input.userId); + if (usedToday >= this.opts.maxUserMsgsPerDay) { + throw new TryChatError( + "quota_day", + `今日试聊次数已用完（${this.opts.maxUserMsgsPerDay} 条）`, + ); + } + + const { sessionId, session, ttlSec } = await createTryChatSession(this.db, { + userId: input.userId, + personaId: persona.id, + botName: input.botName, + ttlSec: this.opts.sessionTtlSec, + }); + + return { + sessionId, + persona: { + id: persona.id, + slug: persona.slug, + display_name: persona.display_name, + description: persona.description, + }, + botName: session.botName, + expiresInSec: ttlSec, + remainingToday: Math.max(0, this.opts.maxUserMsgsPerDay - usedToday), + }; + } + + async sendMessage( + input: SendTryMessageInput, + ): Promise { + const text = (input.text || "").trim(); + if (!text) { + throw new TryChatError("empty", "消息不能为空"); + } + if (text.length > 2000) { + throw new TryChatError("empty", "消息过长（最多 2000 字）"); + } + + const session = await getTryChatSession(this.db, input.sessionId); + if (!session || session.userId !== input.userId) { + throw new TryChatError("not_found", "试聊会话不存在或已过期"); + } + if (session.msgCount >= this.opts.maxUserMsgsPerSession) { + throw new TryChatError( + "quota_session", + `本会话已达上限（${this.opts.maxUserMsgsPerSession} 条），请新开试聊`, + ); + } + + const usedToday = await getTryChatDayCount(this.db, input.userId); + if (usedToday >= this.opts.maxUserMsgsPerDay) { + throw new TryChatError( + "quota_day", + `今日试聊次数已用完（${this.opts.maxUserMsgsPerDay} 条）`, + ); + } + + const persona = await getPersona(this.db, session.personaId); + if (!persona || !persona.enabled) { + throw new TryChatError("not_found", "人设不存在或已下架"); + } + // Re-check access (takedown / private) + if ( + persona.visibility === "private" && + persona.owner_user_id !== input.userId + ) { + throw new TryChatError("forbidden", "无权试聊该人设"); + } + if ( + persona.visibility === "public" || + persona.owner_user_id === input.userId + ) { + // ok + } else if (!(await userCanUsePersona(this.db, input.userId, persona.id))) { + throw new TryChatError("forbidden", "无权试聊该人设"); + } + + const systemPrompt = await getPublishedPrompt(this.db, persona.id); + if (!systemPrompt?.trim()) { + throw new TryChatError("no_prompt", "人设尚未发布可用提示词"); + } + + // Reserve daily quota before LLM call + const dayCount = await incrTryChatDayCount(this.db, input.userId); + if (dayCount > this.opts.maxUserMsgsPerDay) { + throw new TryChatError( + "quota_day", + `今日试聊次数已用完（${this.opts.maxUserMsgsPerDay} 条）`, + ); + } + + const prior = await listTryChatMessages( + this.db, + input.sessionId, + this.opts.maxHistory, + ); + const history: MessageRow[] = prior.map((m, i) => ({ + id: `try_${i}`, + bot_account_id: "try-chat", + peer_id: input.userId, + persona_id: persona.id, + role: m.role, + content: m.content, + context_token: null, + created_at: new Date().toISOString(), + })); + + let usage: { + text: string; + promptTokens: number; + completionTokens: number; + }; + if (persona.mode === "chatflow") { + // Try-chat always runs on the platform upstream: never spend the + // author's custom provider quota / leak their key. + try { + const graph = await getPublishedGraph(this.db, persona.id); + const cf = await this.chatflow.run(graph, { + userText: text, + botName: session.botName, + systemPrompt, + history: history.map((m) => ({ + role: m.role, + content: m.content, + })), + memories: [], + webSearchEnabled: Boolean(persona.web_search_enabled), + upstream: null, + }); + usage = { + text: cf.text, + promptTokens: cf.promptTokens, + completionTokens: cf.completionTokens, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new TryChatError("llm", `Chatflow 执行失败：${msg}`); + } + } else { + const messages = buildChatMessages({ + systemPrompt, + memories: [], + history, + userText: text, + botName: session.botName, + multiBubbleJson: this.primaryMultiBubbleJson(), + stickers: [], + timeToolEnabled: this.opts.timeToolEnabled !== false, + }); + + try { + usage = await this.llm.chatWithUsage(messages, { + tools: + this.opts.timeToolEnabled !== false ? ["get_current_time"] : [], + timeZone: this.opts.timeToolTimeZone || "Asia/Shanghai", + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new TryChatError("llm", `模型调用失败：${msg}`); + } + } + + let parts: ReplyPart[]; + let displayText: string; + let filterPromptTokens = 0; + let filterCompletionTokens = 0; + + if (this.opts.replyFilterEnabled === true) { + const filtered = await this.replyFilter.filter({ + rawText: usage.text, + allowedStickerSlugs: [], + maxBubbles: this.opts.maxReplyBubbles ?? 5, + maxChunkChars: this.opts.maxChunkChars ?? 72, + maxStickers: 0, + }); + filterPromptTokens = filtered.promptTokens; + filterCompletionTokens = filtered.completionTokens; + parts = + filtered.parts.length > 0 + ? filtered.parts.map((p) => + p.kind === "sticker" + ? ({ kind: "text" as const, text: `[表情:${p.slug}]` }) + : p, + ) + : [{ kind: "text" as const, text: usage.text.trim() || "……" }]; + displayText = + filtered.displayText || + parts + .filter((p): p is { kind: "text"; text: string } => p.kind === "text") + .map((p) => p.text) + .join("\n"); + } else { + const parsed = parseMultiBubbleReply(usage.text, { + maxBubbles: this.opts.maxReplyBubbles ?? 5, + maxChunkChars: this.opts.maxChunkChars ?? 72, + maxStickers: 0, + fallbackSplit: true, + expandLongBubbles: true, + }); + parts = + parsed.parts.length > 0 + ? parsed.parts.map((p) => + p.kind === "sticker" + ? ({ kind: "text" as const, text: `[表情:${p.slug}]` }) + : p, + ) + : [ + { + kind: "text" as const, + // Never echo a recognised JSON envelope back at the preview. + text: (parsed.fromJson ? "" : usage.text.trim()) || "……", + }, + ]; + displayText = + parsed.displayText || + parts + .filter((p): p is { kind: "text"; text: string } => p.kind === "text") + .map((p) => p.text) + .join("\n"); + } + + await appendTryChatMessages( + this.db, + input.sessionId, + [ + { role: "user", content: text }, + { role: "assistant", content: displayText }, + ], + { + maxHistory: this.opts.maxHistory, + ttlSec: this.opts.sessionTtlSec, + }, + ); + + session.msgCount = Number(session.msgCount || 0) + 1; + await saveTryChatSession( + this.db, + input.sessionId, + session, + this.opts.sessionTtlSec, + ); + + await recordTokenUsage(this.db, { + userId: input.userId, + botId: "try-chat", + botName: "网页试聊", + username: input.username, + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + }); + if (filterPromptTokens > 0 || filterCompletionTokens > 0) { + await recordTokenUsage(this.db, { + userId: input.userId, + botId: "try-chat", + botName: "网页试聊", + username: input.username, + promptTokens: filterPromptTokens, + completionTokens: filterCompletionTokens, + }); + } + + const totalPrompt = usage.promptTokens + filterPromptTokens; + const totalCompletion = usage.completionTokens + filterCompletionTokens; + + return { + parts, + displayText, + usage: { + promptTokens: totalPrompt, + completionTokens: totalCompletion, + totalTokens: totalPrompt + totalCompletion, + }, + remainingToday: Math.max(0, this.opts.maxUserMsgsPerDay - dayCount), + remainingSession: Math.max( + 0, + this.opts.maxUserMsgsPerSession - session.msgCount, + ), + }; + } + + async endSession(userId: string, sessionId: string): Promise { + const session = await getTryChatSession(this.db, sessionId); + if (!session) return; + if (session.userId !== userId) { + throw new TryChatError("forbidden", "无权结束该会话"); + } + await deleteTryChatSession(this.db, sessionId); + } +} diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json new file mode 100644 index 0000000..a013e0c --- /dev/null +++ b/packages/core/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/packages/db/package.json b/packages/db/package.json new file mode 100644 index 0000000..1e38f48 --- /dev/null +++ b/packages/db/package.json @@ -0,0 +1,29 @@ +{ + "name": "@wechat-ai/db", + "version": "0.2.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "seed": "tsx src/cli-seed.ts", + "test": "node --import tsx --test src/**/*.test.ts" + }, + "dependencies": { + "ioredis": "^5.6.0" + }, + "devDependencies": { + "@types/node": "^22.13.10", + "tsx": "^4.19.3", + "typescript": "^5.8.2" + } +} diff --git a/packages/db/src/bot-login-repos.ts b/packages/db/src/bot-login-repos.ts new file mode 100644 index 0000000..e913da0 --- /dev/null +++ b/packages/db/src/bot-login-repos.ts @@ -0,0 +1,103 @@ +import type { RedisStore } from "./client.js"; +import { K } from "./keys.js"; + +/** Default QR login session TTL (seconds). Matches UI ~10min window. */ +export const BOT_LOGIN_TTL_SEC = 10 * 60; + +export type BotLoginSessionStatus = + | "pending" + | "wait_scan" + | "scanned" + | "confirmed" + | "expired" + | "error" + | "cancelled"; + +export type BotLoginMode = "create" | "rebind"; + +/** Serializable view shared across nodes for QR bot login. */ +export interface BotLoginSessionRecord { + sessionId: string; + displayName: string; + ownerUserId: string; + status: BotLoginSessionStatus; + mode: BotLoginMode; + rebindBotId?: string; + qrcode?: string; + openUrl?: string; + message?: string; + botId?: string; + createdAt: string; + updatedAt: string; +} + +export async function saveBotLoginSession( + db: RedisStore, + record: BotLoginSessionRecord, + ttlSec: number = BOT_LOGIN_TTL_SEC, +): Promise { + await db.setJson(K.botLogin(record.sessionId), record, ttlSec); + if (record.ownerUserId) { + try { + await db.redis.sadd(K.botLoginsByOwner(record.ownerUserId), record.sessionId); + // Keep owner index roughly in sync with session TTL + await db.redis.expire(K.botLoginsByOwner(record.ownerUserId), ttlSec + 60); + } catch { + /* index is best-effort */ + } + } +} + +export async function getBotLoginSession( + db: RedisStore, + sessionId: string, +): Promise { + return db.getJson(K.botLogin(sessionId)); +} + +export async function deleteBotLoginSession( + db: RedisStore, + sessionId: string, + ownerUserId?: string, +): Promise { + await db.del(K.botLogin(sessionId)); + if (ownerUserId) { + try { + await db.redis.srem(K.botLoginsByOwner(ownerUserId), sessionId); + } catch { + /* */ + } + } +} + +/** + * Mark session cancelled for cross-node cancel; poller checks status each tick. + * Keeps a short TTL so clients can still observe cancelled state briefly. + */ +export async function markBotLoginCancelled( + db: RedisStore, + sessionId: string, + ownerUserId: string, + message = "已取消", +): Promise { + const cur = await getBotLoginSession(db, sessionId); + if (!cur) return null; + if (cur.ownerUserId !== ownerUserId) return null; + if ( + cur.status === "confirmed" || + cur.status === "expired" || + cur.status === "error" || + cur.status === "cancelled" + ) { + return cur; + } + const next: BotLoginSessionRecord = { + ...cur, + status: "cancelled", + message, + updatedAt: new Date().toISOString(), + }; + // Short residual TTL after cancel + await saveBotLoginSession(db, next, 120); + return next; +} diff --git a/packages/db/src/broadcast-repos.ts b/packages/db/src/broadcast-repos.ts new file mode 100644 index 0000000..4712039 --- /dev/null +++ b/packages/db/src/broadcast-repos.ts @@ -0,0 +1,448 @@ +import type { RedisStore } from "./client.js"; +import { newId, nowIso } from "./client.js"; +import { K } from "./keys.js"; +import { + getBotAccount, + listBotAccounts, + listPeers, + listPeersForBots, +} from "./repos.js"; + +export type BroadcastScope = "all_bots" | "bots" | "targets"; + +export type BroadcastStatus = + | "pending" + | "running" + | "completed" + | "cancelled" + | "failed"; + +export interface BroadcastTarget { + botId: string; + peerId: string; +} + +export interface BroadcastStats { + total: number; + sent: number; + skipped: number; + failed: number; +} + +export interface BroadcastFailure { + botId: string; + peerId: string; + error: string; +} + +export interface BroadcastJob { + id: string; + createdBy: string; + createdAt: string; + updatedAt: string; + status: BroadcastStatus; + text: string; + scope: BroadcastScope; + botIds: string[]; + targets: BroadcastTarget[]; + /** Snapshot of deliverable (bot, peer) pairs at create time */ + recipients: BroadcastTarget[]; + stats: BroadcastStats; + error?: string | null; + startedAt?: string | null; + finishedAt?: string | null; + failures?: BroadcastFailure[]; + /** Cursor into recipients for resume after crash (optional) */ + cursor?: number; +} + +export interface ExpandResult { + recipients: BroadcastTarget[]; + /** Peers considered but missing context_token (not enqueued) */ + skippedNoToken: number; + /** Requested bots that do not exist */ + missingBots: string[]; +} + +const DEFAULT_HISTORY = 100; +const MAX_FAILURES = 20; + +function emptyStats(total = 0): BroadcastStats { + return { total, sent: 0, skipped: 0, failed: 0 }; +} + +/** + * Expand a broadcast request into (botId, peerId) pairs that currently have + * a context_token. Does not filter by approved — only token reachability. + */ +export async function expandBroadcastRecipients( + db: RedisStore, + input: { + scope: BroadcastScope; + botIds?: string[]; + targets?: BroadcastTarget[]; + }, +): Promise { + const missingBots: string[] = []; + let skippedNoToken = 0; + const recipients: BroadcastTarget[] = []; + + if (input.scope === "targets") { + const raw = (input.targets ?? []).filter( + (t) => t?.botId?.trim() && t?.peerId?.trim(), + ); + const seen = new Set(); + const pairs: BroadcastTarget[] = []; + for (const t of raw) { + const botId = t.botId.trim(); + const peerId = t.peerId.trim(); + const key = `${botId}|${peerId}`; + if (seen.has(key)) continue; + seen.add(key); + pairs.push({ botId, peerId }); + } + // One MGET, like the bots branch below — was one GET per target + const toks = await db.mgetStrings( + pairs.map((t) => K.contextToken(t.botId, t.peerId)), + ); + pairs.forEach((t, i) => { + if (!toks[i]) { + skippedNoToken++; + return; + } + recipients.push(t); + }); + return { recipients, skippedNoToken, missingBots }; + } + + let botIds: string[]; + if (input.scope === "all_bots") { + botIds = (await listBotAccounts(db)).map((b) => b.id); + } else { + const requested = [ + ...new Set((input.botIds ?? []).map((id) => id.trim()).filter(Boolean)), + ]; + const found = await Promise.all( + requested.map(async (id) => ({ id, bot: await getBotAccount(db, id) })), + ); + botIds = []; + for (const row of found) { + if (row.bot) botIds.push(row.id); + else missingBots.push(row.id); + } + } + + if (!botIds.length) { + return { recipients, skippedNoToken, missingBots }; + } + + const peers = await listPeersForBots(db, botIds); + if (!peers.length) { + return { recipients, skippedNoToken, missingBots }; + } + + // Batch context_token presence via MGET + const ctxKeys = peers.map((p) => + K.contextToken(p.bot_account_id, p.peer_id), + ); + const tokens = await db.mgetStrings(ctxKeys); + for (let i = 0; i < peers.length; i++) { + const p = peers[i]!; + const tok = tokens[i]; + if (!tok?.trim()) { + skippedNoToken++; + continue; + } + recipients.push({ botId: p.bot_account_id, peerId: p.peer_id }); + } + + return { recipients, skippedNoToken, missingBots }; +} + +export async function createBroadcastJob( + db: RedisStore, + input: { + createdBy: string; + text: string; + scope: BroadcastScope; + botIds?: string[]; + targets?: BroadcastTarget[]; + historyLimit?: number; + }, +): Promise { + const text = input.text.trim(); + if (!text) throw new Error("text required"); + + const expanded = await expandBroadcastRecipients(db, { + scope: input.scope, + botIds: input.botIds, + targets: input.targets, + }); + + if (input.scope === "bots" && expanded.missingBots.length) { + throw new Error(`bots not found: ${expanded.missingBots.join(", ")}`); + } + + const now = nowIso(); + const job: BroadcastJob = { + id: newId("bc"), + createdBy: input.createdBy, + createdAt: now, + updatedAt: now, + status: "pending", + text, + scope: input.scope, + botIds: + input.scope === "bots" + ? [...new Set((input.botIds ?? []).map((x) => x.trim()).filter(Boolean))] + : [], + targets: + input.scope === "targets" + ? (input.targets ?? []) + .filter((t) => t?.botId?.trim() && t?.peerId?.trim()) + .map((t) => ({ + botId: t.botId.trim(), + peerId: t.peerId.trim(), + })) + : [], + recipients: expanded.recipients, + stats: emptyStats(expanded.recipients.length), + // Peers without token are pre-skipped at expand; surface as initial skipped + // so UI can show "will skip M" without inflating total. + // We keep them out of recipients (cannot send); record count in stats.skipped + // only when targets mode requested unreachable peers — for all_bots/bots + // skippedNoToken is informational via expand preview, not job.stats. + error: null, + startedAt: null, + finishedAt: null, + failures: [], + cursor: 0, + }; + + // For explicit targets: unreachable ones never entered recipients — count as skipped + if (input.scope === "targets" && expanded.skippedNoToken > 0) { + job.stats.skipped = expanded.skippedNoToken; + job.stats.total = expanded.recipients.length + expanded.skippedNoToken; + } + + await db.setJson(K.broadcast(job.id), job); + const history = Math.max(10, input.historyLimit ?? DEFAULT_HISTORY); + await db.redis.lpush(K.broadcastsAll, job.id); + await db.redis.ltrim(K.broadcastsAll, 0, history - 1); + return job; +} + +export async function getBroadcastJob( + db: RedisStore, + id: string, +): Promise { + return db.getJson(K.broadcast(id)); +} + +export async function saveBroadcastJob( + db: RedisStore, + job: BroadcastJob, +): Promise { + job.updatedAt = nowIso(); + await db.setJson(K.broadcast(job.id), job); +} + +export async function listBroadcastJobs( + db: RedisStore, + limit = 50, +): Promise { + const n = Math.max(1, Math.min(200, limit)); + const ids = (await db.redis.lrange(K.broadcastsAll, 0, n - 1)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id) => K.broadcast(id)), + ); + return rows.filter((j): j is BroadcastJob => Boolean(j)); +} + +/** Oldest pending job (queue order: list is newest-first, so scan from end). */ +export async function findNextPendingBroadcast( + db: RedisStore, +): Promise { + // Prefer a previously running job that still has work (crash recovery) + const activeId = (await db.redis.get(K.broadcastActive)) as string | null; + if (activeId) { + const active = await getBroadcastJob(db, activeId); + if ( + active && + (active.status === "running" || active.status === "pending") && + (active.cursor ?? 0) < (active.recipients?.length ?? 0) + ) { + return active; + } + } + + // Scan recent jobs for any running first (resume), then oldest pending + const ids = (await db.redis.lrange(K.broadcastsAll, 0, 99)) as string[]; + if (!ids.length) return null; + const rows = await db.mgetJson( + ids.map((id) => K.broadcast(id)), + ); + const jobs = rows.filter((j): j is BroadcastJob => Boolean(j)); + + const running = jobs.find( + (j) => + j.status === "running" && + (j.cursor ?? 0) < (j.recipients?.length ?? 0), + ); + if (running) return running; + + // Pending: process oldest first (list is newest-first → reverse) + for (let i = jobs.length - 1; i >= 0; i--) { + const j = jobs[i]!; + if (j.status === "pending") return j; + } + return null; +} + +export async function hasRunningBroadcast(db: RedisStore): Promise { + const ids = (await db.redis.lrange(K.broadcastsAll, 0, 49)) as string[]; + if (!ids.length) return false; + const rows = await db.mgetJson( + ids.map((id) => K.broadcast(id)), + ); + return rows.some((j) => j?.status === "running"); +} + +export async function tryAcquireBroadcastLock( + db: RedisStore, + jobId: string, + workerId: string, + ttlSec = 60, +): Promise { + const ok = await db.redis.set( + K.broadcastLock(jobId), + workerId, + "EX", + Math.max(15, ttlSec), + "NX", + ); + return ok === "OK"; +} + +export async function renewBroadcastLock( + db: RedisStore, + jobId: string, + workerId: string, + ttlSec = 60, +): Promise { + const cur = await db.redis.get(K.broadcastLock(jobId)); + if (cur !== workerId) return false; + await db.redis.set( + K.broadcastLock(jobId), + workerId, + "EX", + Math.max(15, ttlSec), + ); + return true; +} + +export async function releaseBroadcastLock( + db: RedisStore, + jobId: string, + workerId?: string, +): Promise { + if (workerId) { + const cur = await db.redis.get(K.broadcastLock(jobId)); + if (cur && cur !== workerId) return; + } + await db.del(K.broadcastLock(jobId)); +} + +export async function setBroadcastActive( + db: RedisStore, + jobId: string | null, +): Promise { + if (!jobId) { + await db.del(K.broadcastActive); + return; + } + await db.redis.set(K.broadcastActive, jobId); +} + +export async function cancelBroadcastJob( + db: RedisStore, + jobId: string, +): Promise { + const job = await getBroadcastJob(db, jobId); + if (!job) return null; + if (job.status !== "pending" && job.status !== "running") { + throw new Error(`cannot cancel job in status ${job.status}`); + } + job.status = "cancelled"; + job.finishedAt = nowIso(); + await saveBroadcastJob(db, job); + const active = await db.redis.get(K.broadcastActive); + if (active === jobId) { + await setBroadcastActive(db, null); + } + return job; +} + +export function pushBroadcastFailure( + job: BroadcastJob, + fail: BroadcastFailure, +): void { + const list = job.failures ?? []; + list.push(fail); + if (list.length > MAX_FAILURES) { + job.failures = list.slice(list.length - MAX_FAILURES); + } else { + job.failures = list; + } +} + +/** Peers for a bot with hasContextToken flag (admin send-targets UI). */ +export async function listBotSendTargets( + db: RedisStore, + botId: string, +): Promise< + Array<{ + peerId: string; + displayName: string | null; + approved: boolean; + hasContextToken: boolean; + lastActivityAt: string | null; + }> +> { + const bot = await getBotAccount(db, botId); + if (!bot) return []; + const peers = await listPeers(db, botId); + if (!peers.length) return []; + const tokens = await db.mgetStrings( + peers.map((p) => K.contextToken(botId, p.peer_id)), + ); + return peers.map((p, i) => ({ + peerId: p.peer_id, + displayName: p.display_name, + approved: Boolean(p.approved), + hasContextToken: Boolean(tokens[i]?.trim()), + lastActivityAt: p.last_activity_at ?? null, + })); +} + +/** Public preview for UI before create (does not write). */ +export async function previewBroadcast( + db: RedisStore, + input: { + scope: BroadcastScope; + botIds?: string[]; + targets?: BroadcastTarget[]; + }, +): Promise<{ + deliverable: number; + skippedNoToken: number; + missingBots: string[]; +}> { + const r = await expandBroadcastRecipients(db, input); + return { + deliverable: r.recipients.length, + skippedNoToken: r.skippedNoToken, + missingBots: r.missingBots, + }; +} diff --git a/packages/db/src/cli-migrate.ts b/packages/db/src/cli-migrate.ts new file mode 100644 index 0000000..53a2edf --- /dev/null +++ b/packages/db/src/cli-migrate.ts @@ -0,0 +1,8 @@ +import { migrate, openDatabase } from "./client.js"; +import { defaultDbPath } from "./paths.js"; + +const dbPath = defaultDbPath(); +const db = openDatabase(dbPath); +migrate(db); +console.log(`Migrated database at ${dbPath}`); +db.close(); diff --git a/packages/db/src/cli-seed.ts b/packages/db/src/cli-seed.ts new file mode 100644 index 0000000..e20a217 --- /dev/null +++ b/packages/db/src/cli-seed.ts @@ -0,0 +1,54 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { openDatabase } from "./client.js"; +import { seedPersonas } from "./seed.js"; + +function resolveRepoRoot(): string { + let dir = path.resolve(process.cwd()); + for (let i = 0; i < 12; i++) { + if (fs.existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.."); +} + +function loadEnvFile(file: string): void { + if (!fs.existsSync(file)) return; + const text = fs.readFileSync(file, "utf8"); + for (const line of text.split(/\r?\n/)) { + const t = line.trim(); + if (!t || t.startsWith("#")) continue; + const i = t.indexOf("="); + if (i < 0) continue; + const key = t.slice(0, i).trim(); + let val = t.slice(i + 1).trim(); + if ( + (val.startsWith('"') && val.endsWith('"')) || + (val.startsWith("'") && val.endsWith("'")) + ) { + val = val.slice(1, -1); + } + if (process.env[key] === undefined) process.env[key] = val; + } +} + +const root = resolveRepoRoot(); +loadEnvFile(path.join(root, ".env")); + +const url = process.env.REDIS_URL ?? "redis://127.0.0.1:6379"; +console.log( + `[redis] using ${url.replace(/:\/\/([^:]+):([^@]+)@/, "://$1:***@")}`, +); + +const db = openDatabase(url); +try { + const pong = await db.ping(); + console.log(`[redis] ${pong}`); + await seedPersonas(db); + console.log("Seeded personas into Redis"); +} finally { + await db.close(); +} diff --git a/packages/db/src/client.test.ts b/packages/db/src/client.test.ts new file mode 100644 index 0000000..47fab0d --- /dev/null +++ b/packages/db/src/client.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { dayKey, dayKeyOffset, DEFAULT_DAY_TZ } from "./client.js"; + +describe("dayKey (Asia/Shanghai)", () => { + it("defaults timezone to Asia/Shanghai", () => { + assert.equal(DEFAULT_DAY_TZ, "Asia/Shanghai"); + }); + + it("uses China calendar date, not UTC, before 08:00 CST", () => { + // 2026-07-20 18:03 UTC == 2026-07-21 02:03 Asia/Shanghai + const d = new Date("2026-07-20T18:03:00.000Z"); + assert.equal(d.toISOString().slice(0, 10), "2026-07-20"); // UTC trap + assert.equal(dayKey(d), "2026-07-21"); + }); + + it("matches UTC date during China daytime after 08:00", () => { + // 2026-07-21 10:00 CST == 2026-07-21 02:00 UTC + const d = new Date("2026-07-21T02:00:00.000Z"); + assert.equal(dayKey(d), "2026-07-21"); + }); + + it("rolls over at China midnight (16:00 UTC previous day)", () => { + // 2026-07-20 15:59:59 UTC == 2026-07-20 23:59:59 CST + assert.equal(dayKey(new Date("2026-07-20T15:59:59.000Z")), "2026-07-20"); + // 2026-07-20 16:00:00 UTC == 2026-07-21 00:00:00 CST + assert.equal(dayKey(new Date("2026-07-20T16:00:00.000Z")), "2026-07-21"); + }); +}); + +describe("dayKeyOffset", () => { + it("returns yesterday/tomorrow relative to China calendar", () => { + const d = new Date("2026-07-20T18:03:00.000Z"); // China 07-21 02:03 + assert.equal(dayKeyOffset(0, d), "2026-07-21"); + assert.equal(dayKeyOffset(-1, d), "2026-07-20"); + assert.equal(dayKeyOffset(1, d), "2026-07-22"); + }); + + it("handles month boundaries", () => { + // China 2026-08-01 01:00 == 2026-07-31 17:00 UTC + const d = new Date("2026-07-31T17:00:00.000Z"); + assert.equal(dayKey(d), "2026-08-01"); + assert.equal(dayKeyOffset(-1, d), "2026-07-31"); + }); +}); diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts new file mode 100644 index 0000000..c7b7982 --- /dev/null +++ b/packages/db/src/client.ts @@ -0,0 +1,382 @@ +import { randomUUID } from "node:crypto"; +import { Redis, type RedisOptions } from "ioredis"; + +export type Db = RedisStore; + +/** + * Build ioredis options for local Redis or Upstash (rediss:// TLS). + * + * Upstash: Console → Connect → copy Redis URL (ioredis): + * REDIS_URL=rediss://default:****@xxxx.upstash.io:6379 + * + * Latency notes (esp. Upstash cross-region): + * - enableAutoPipelining coalesces concurrent commands in one RTT + * - keepAlive avoids cold TCP/TLS reconnects + * - prefer a region close to the API host + */ +export function buildRedisOptions(redisUrl: string): { + url: string; + options: RedisOptions; +} { + const url = redisUrl.trim(); + const isTls = + url.startsWith("rediss://") || + /upstash\.io/i.test(url) || + process.env.REDIS_TLS === "true"; + + const connectTimeout = Number( + process.env.REDIS_CONNECT_TIMEOUT_MS ?? "15000", + ); + const isUpstash = /upstash\.io/i.test(url); + + const options: RedisOptions = { + // Fewer blind retries on flaky links; fail fast so callers can recover + maxRetriesPerRequest: Number(process.env.REDIS_MAX_RETRIES ?? "2"), + enableReadyCheck: true, + // Connect immediately; queue commands until ready (CLI seed / API both need this) + lazyConnect: false, + enableOfflineQueue: true, + connectTimeout, + // Critical for remote Redis: concurrent awaits share one pipeline RTT + enableAutoPipelining: + process.env.REDIS_AUTO_PIPELINE !== "false", + // Prevent idle disconnect churn (TLS handshake is expensive) + keepAlive: Number(process.env.REDIS_KEEPALIVE_MS ?? "10000"), + // Prefer IPv4 for Upstash (dual-stack DNS can add latency); override with REDIS_FAMILY + ...(process.env.REDIS_FAMILY + ? { family: Number(process.env.REDIS_FAMILY) as 0 | 4 | 6 } + : isUpstash + ? { family: 4 as const } + : {}), + retryStrategy(times) { + if (times > (isUpstash ? 6 : 8)) return null; + return Math.min(times * 150, 1500); + }, + ...(isTls && !url.startsWith("rediss://") + ? { tls: { rejectUnauthorized: true } } + : {}), + }; + + let finalUrl = url; + if (/upstash\.io/i.test(url) && url.startsWith("redis://")) { + finalUrl = "rediss://" + url.slice("redis://".length); + } + + return { url: finalUrl, options }; +} + +/** Optional sampled command hook for admin activity stream (apps/api). */ +export type RedisCommandHook = (info: { + op: string; + key?: string; + keys?: number; + ms?: number; + ok?: boolean; +}) => void; + +let globalCommandHook: RedisCommandHook | null = null; + +/** Register process-wide Redis command observer (sampled by caller). */ +export function setRedisCommandHook(hook: RedisCommandHook | null): void { + globalCommandHook = hook; +} + +export function getRedisCommandHook(): RedisCommandHook | null { + return globalCommandHook; +} + +export class RedisStore { + readonly redis: Redis; + + constructor(redisUrl: string) { + const { url, options } = buildRedisOptions(redisUrl); + this.redis = new Redis(url, options); + this.redis.on("error", (err: Error) => { + if (process.env.LOG_LEVEL === "debug") { + console.error("[redis]", err.message); + } + }); + } + + private note( + op: string, + t0: number, + ok: boolean, + key?: string, + keys?: number, + ): void { + const hook = globalCommandHook; + if (!hook) return; + try { + hook({ op, key, keys, ms: Date.now() - t0, ok }); + } catch { + /* never break redis path */ + } + } + + async ping(): Promise { + const t0 = Date.now(); + try { + const r = await this.redis.ping(); + this.note("ping", t0, true); + return r; + } catch (e) { + this.note("ping", t0, false); + throw e; + } + } + + async close(): Promise { + try { + if (this.redis.status === "ready") await this.redis.quit(); + else this.redis.disconnect(); + } catch { + this.redis.disconnect(); + } + } + + async getJson(key: string): Promise { + const t0 = Date.now(); + try { + const raw = await this.redis.get(key); + this.note("get", t0, true, key); + if (!raw) return null; + return JSON.parse(raw) as T; + } catch (e) { + this.note("get", t0, false, key); + throw e; + } + } + + /** + * Run `task` over key slices with bounded concurrency. + * + * Chunks used to be awaited one after another, so a 2000-key MGET cost 10 + * serial round trips to a remote Redis. Running them together turns that + * into ~1 wall-clock RTT. Capped so a huge key list can't dump megabytes + * into the socket at once. + */ + private static readonly CHUNK = 200; + private static readonly MAX_IN_FLIGHT_CHUNKS = 8; + + private async forEachChunk( + keys: string[], + task: (slice: string[], offset: number) => Promise, + ): Promise { + const { CHUNK, MAX_IN_FLIGHT_CHUNKS } = RedisStore; + const offsets: number[] = []; + for (let off = 0; off < keys.length; off += CHUNK) offsets.push(off); + for (let i = 0; i < offsets.length; i += MAX_IN_FLIGHT_CHUNKS) { + const wave = offsets.slice(i, i + MAX_IN_FLIGHT_CHUNKS); + await Promise.all( + wave.map((off) => task(keys.slice(off, off + CHUNK), off)), + ); + } + } + + /** + * Batch JSON GET via Redis MGET — chunked for Upstash / large fleets. + * Order matches `keys`; missing / invalid entries are null. + */ + async mgetJson(keys: string[]): Promise<(T | null)[]> { + if (!keys.length) return []; + const out: (T | null)[] = new Array(keys.length); + await this.forEachChunk(keys, async (slice, off) => { + const t0 = Date.now(); + try { + const raws = (await this.redis.mget(...slice)) as (string | null)[]; + this.note("mget", t0, true, slice[0], slice.length); + for (let i = 0; i < slice.length; i++) { + const raw = raws[i]; + if (!raw) { + out[off + i] = null; + continue; + } + try { + out[off + i] = JSON.parse(raw) as T; + } catch { + out[off + i] = null; + } + } + } catch (e) { + this.note("mget", t0, false, slice[0], slice.length); + throw e; + } + }); + return out; + } + + /** + * Batch string GET (non-JSON) via MGET — for assignment keys etc. + * Order matches `keys`; missing entries are null. + */ + async mgetStrings(keys: string[]): Promise<(string | null)[]> { + if (!keys.length) return []; + const out: (string | null)[] = new Array(keys.length); + await this.forEachChunk(keys, async (slice, off) => { + const t0 = Date.now(); + try { + const raws = (await this.redis.mget(...slice)) as (string | null)[]; + this.note("mget", t0, true, slice[0], slice.length); + for (let i = 0; i < slice.length; i++) { + out[off + i] = raws[i] ?? null; + } + } catch (e) { + this.note("mget", t0, false, slice[0], slice.length); + throw e; + } + }); + return out; + } + + async setJson(key: string, value: unknown, ttlSec?: number): Promise { + const raw = JSON.stringify(value); + const t0 = Date.now(); + try { + if (ttlSec && ttlSec > 0) { + await this.redis.set(key, raw, "EX", ttlSec); + } else { + await this.redis.set(key, raw); + } + this.note("set", t0, true, key); + } catch (e) { + this.note("set", t0, false, key); + throw e; + } + } + + async del(...keys: string[]): Promise { + if (!keys.length) return; + const t0 = Date.now(); + try { + await this.redis.del(...keys); + this.note("del", t0, true, keys[0], keys.length); + } catch (e) { + this.note("del", t0, false, keys[0], keys.length); + throw e; + } + } + + /** Run a pipeline in chunks to avoid Upstash / huge-pipeline stalls. */ + private async pipelineChunked( + keys: string[], + add: (pipe: ReturnType, key: string) => void, + ): Promise> { + if (!keys.length) return []; + const out: Array<[Error | null, unknown] | null> = new Array(keys.length); + await this.forEachChunk(keys, async (slice, off) => { + const pipe = this.redis.pipeline(); + for (const k of slice) add(pipe, k); + const res = await pipe.exec(); + for (let i = 0; i < slice.length; i++) { + out[off + i] = res?.[i] ?? null; + } + }); + return out; + } + + /** Per-key EXISTS via pipeline (multi-key EXISTS only returns a count). */ + async existsMany(keys: string[]): Promise { + const res = await this.pipelineChunked(keys, (pipe, k) => pipe.exists(k)); + return res.map((row) => Number(row?.[1] ?? 0) > 0); + } + + /** Per-key SCARD via pipeline. */ + async scardMany(keys: string[]): Promise { + const res = await this.pipelineChunked(keys, (pipe, k) => pipe.scard(k)); + return res.map((row) => Number(row?.[1] ?? 0)); + } + + /** Per-key SMEMBERS via pipeline. */ + async smembersMany(keys: string[]): Promise { + const res = await this.pipelineChunked(keys, (pipe, k) => pipe.smembers(k)); + return res.map((row) => { + const v = row?.[1]; + return Array.isArray(v) ? (v as string[]) : []; + }); + } + + /** Per-key GET via pipeline (for mixed key patterns where MGET is awkward). */ + async getMany(keys: string[]): Promise<(string | null)[]> { + return this.mgetStrings(keys); + } +} + +export function openDatabase(redisUrl: string): RedisStore { + return new RedisStore(redisUrl); +} + +/** @deprecated no-op migrate for Redis */ +export async function migrate(_db: RedisStore): Promise { + // Redis is schema-less +} + +export function newId(prefix = ""): string { + const id = randomUUID().replace(/-/g, ""); + return prefix ? `${prefix}_${id}` : id; +} + +export function nowIso(): string { + return new Date().toISOString(); +} + +/** Calendar day keys (usage, proactive caps, p2p daily limits) use China local date. */ +export const DEFAULT_DAY_TZ = "Asia/Shanghai"; + +/** + * YYYY-MM-DD for `d` in `timeZone` (default Asia/Shanghai). + * Do NOT use `toISOString().slice(0, 10)` for "today" — that is UTC and is still + * the previous calendar day in China between 00:00 and 08:00 CST. + */ +export function dayKey(d: Date = new Date(), timeZone = DEFAULT_DAY_TZ): string { + try { + // en-CA yields YYYY-MM-DD + return new Intl.DateTimeFormat("en-CA", { + timeZone: timeZone?.trim() || DEFAULT_DAY_TZ, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).format(d); + } catch { + return d.toISOString().slice(0, 10); + } +} + +/** + * Shift a calendar day by `n` days in `timeZone` (negative = past). + * Safer than `Date.now() ± n*86400000` then UTC-slicing around midnight/DST edges. + */ +export function dayKeyOffset( + n: number, + from: Date = new Date(), + timeZone = DEFAULT_DAY_TZ, +): string { + const tz = timeZone?.trim() || DEFAULT_DAY_TZ; + let y: number; + let m: number; + let day: number; + try { + const parts = new Intl.DateTimeFormat("en-CA", { + timeZone: tz, + year: "numeric", + month: "2-digit", + day: "2-digit", + }).formatToParts(from); + y = Number(parts.find((p) => p.type === "year")?.value); + m = Number(parts.find((p) => p.type === "month")?.value); + day = Number(parts.find((p) => p.type === "day")?.value); + } catch { + y = from.getUTCFullYear(); + m = from.getUTCMonth() + 1; + day = from.getUTCDate(); + } + if (!Number.isFinite(y) || !Number.isFinite(m) || !Number.isFinite(day)) { + return dayKey(new Date(from.getTime() + n * 86400000), tz); + } + // Calendar arithmetic in UTC noon space (no DST in Asia/Shanghai) + const shifted = new Date(Date.UTC(y, m - 1, day + n, 12, 0, 0)); + const yy = shifted.getUTCFullYear(); + const mm = String(shifted.getUTCMonth() + 1).padStart(2, "0"); + const dd = String(shifted.getUTCDate()).padStart(2, "0"); + return `${yy}-${mm}-${dd}`; +} diff --git a/packages/db/src/doctor-stats.test.ts b/packages/db/src/doctor-stats.test.ts new file mode 100644 index 0000000..57880d3 --- /dev/null +++ b/packages/db/src/doctor-stats.test.ts @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + DEFAULT_DEEP_STATS_MAX_PEERS, + assignmentKeys, + deepStatsMaxPeers, + memoryKeys, + messageKeys, + pairKey, + parsePeerPairs, + shouldComputeDeepStats, +} from "./doctor-stats.js"; + +describe("parsePeerPairs", () => { + it("splits botId|peerId", () => { + assert.deepEqual(parsePeerPairs(["b1|p1", "b2|p2"]), [ + { botId: "b1", peerId: "p1" }, + { botId: "b2", peerId: "p2" }, + ]); + }); + + it("splits on the first separator only, since peerId is opaque", () => { + assert.deepEqual(parsePeerPairs(["bot|wx|abc|def"]), [ + { botId: "bot", peerId: "wx|abc|def" }, + ]); + }); + + it("drops malformed members instead of producing partial keys", () => { + assert.deepEqual(parsePeerPairs(["", "nosep", "|p", "b|", "b|p"]), [ + { botId: "b", peerId: "p" }, + ]); + }); + + it("handles an empty set", () => { + assert.deepEqual(parsePeerPairs([]), []); + }); +}); + +describe("key planning", () => { + const pairs = [ + { botId: "b1", peerId: "p1" }, + { botId: "b2", peerId: "p2" }, + ]; + + it("builds assignment keys aligned with the pair order", () => { + assert.deepEqual(assignmentKeys(pairs), [ + "wa:asg:b1:p1", + "wa:asg:b2:p2", + ]); + }); + + it("builds message keys aligned with the pair order", () => { + assert.deepEqual(messageKeys(pairs), [ + "wa:msgs:b1:p1", + "wa:msgs:b2:p2", + ]); + }); + + it("pairKey matches the assignment map convention", () => { + assert.equal(pairKey(pairs[0]!), "b1|p1"); + }); +}); + +describe("memoryKeys", () => { + const pairs = [ + { botId: "b1", peerId: "p1" }, + { botId: "b1", peerId: "p2" }, + ]; + + it("uses the assigned persona when there is one", () => { + const assigned = new Map([["b1|p1", "persona-x"]]); + assert.deepEqual(memoryKeys(pairs, assigned, null), [ + "wa:mem:b1:p1:persona-x", + ]); + }); + + it("falls back to the platform default for unassigned peers", () => { + assert.deepEqual(memoryKeys(pairs, new Map(), "persona-def"), [ + "wa:mem:b1:p1:persona-def", + "wa:mem:b1:p2:persona-def", + ]); + }); + + it("counts both assigned and default when they differ", () => { + const assigned = new Map([["b1|p1", "persona-x"]]); + assert.deepEqual(memoryKeys(pairs, assigned, "persona-def"), [ + "wa:mem:b1:p1:persona-x", + "wa:mem:b1:p1:persona-def", + "wa:mem:b1:p2:persona-def", + ]); + }); + + it("never emits the same key twice when assignment equals the default", () => { + const assigned = new Map([["b1|p1", "persona-def"]]); + const keys = memoryKeys(pairs, assigned, "persona-def"); + assert.deepEqual(keys, [ + "wa:mem:b1:p1:persona-def", + "wa:mem:b1:p2:persona-def", + ]); + assert.equal(new Set(keys).size, keys.length); + }); + + it("stays at most two keys per peer", () => { + const many = Array.from({ length: 50 }, (_, i) => ({ + botId: "b", + peerId: `p${i}`, + })); + const assigned = new Map(many.map((p) => [pairKey(p), `persona-${p.peerId}`])); + assert.equal(memoryKeys(many, assigned, "persona-def").length, 100); + }); + + it("produces nothing when there is neither an assignment nor a default", () => { + assert.deepEqual(memoryKeys(pairs, new Map(), null), []); + }); +}); + +describe("deep stat sizing", () => { + it("defaults the cap when unset or unparseable", () => { + assert.equal(deepStatsMaxPeers({}), DEFAULT_DEEP_STATS_MAX_PEERS); + assert.equal( + deepStatsMaxPeers({ DOCTOR_DEEP_STATS_MAX_PEERS: "abc" }), + DEFAULT_DEEP_STATS_MAX_PEERS, + ); + }); + + it("reads an explicit cap", () => { + assert.equal(deepStatsMaxPeers({ DOCTOR_DEEP_STATS_MAX_PEERS: "120" }), 120); + }); + + it("treats 0 and negatives as disabled", () => { + assert.equal(deepStatsMaxPeers({ DOCTOR_DEEP_STATS_MAX_PEERS: "0" }), 0); + assert.equal(deepStatsMaxPeers({ DOCTOR_DEEP_STATS_MAX_PEERS: "-5" }), 0); + assert.equal(shouldComputeDeepStats(0, 0), false); + assert.equal(shouldComputeDeepStats(1, 0), false); + }); + + it("computes at or below the cap and skips above it", () => { + assert.equal(shouldComputeDeepStats(0, 10), true); + assert.equal(shouldComputeDeepStats(10, 10), true); + assert.equal(shouldComputeDeepStats(11, 10), false); + }); +}); diff --git a/packages/db/src/doctor-stats.ts b/packages/db/src/doctor-stats.ts new file mode 100644 index 0000000..af84c94 --- /dev/null +++ b/packages/db/src/doctor-stats.ts @@ -0,0 +1,100 @@ +import { K } from "./keys.js"; + +/** + * Pure read-planning for the admin dashboard's deep counters. + * + * `assignments` / `messages` / `memories` used to be hardcoded to 0 because + * computing them naively means walking the keyspace. These helpers turn the + * peer list the snapshot already loaded into a bounded, pipelineable set of + * keys — no Redis access, so the sizing rules are unit-testable. + */ + +export interface PeerPair { + botId: string; + peerId: string; +} + +/** `wa:peers:all` members are `botId|peerId`. Malformed entries are dropped. */ +export function parsePeerPairs(members: readonly string[]): PeerPair[] { + const out: PeerPair[] = []; + for (const member of members) { + const raw = String(member); + // peerId is opaque and could itself contain "|", so split on the first only + const sep = raw.indexOf("|"); + if (sep <= 0) continue; + const botId = raw.slice(0, sep); + const peerId = raw.slice(sep + 1); + if (botId && peerId) out.push({ botId, peerId }); + } + return out; +} + +export function pairKey(pair: PeerPair): string { + return `${pair.botId}|${pair.peerId}`; +} + +export function assignmentKeys(pairs: readonly PeerPair[]): string[] { + return pairs.map((p) => K.assignment(p.botId, p.peerId)); +} + +export function messageKeys(pairs: readonly PeerPair[]): string[] { + return pairs.map((p) => K.messages(p.botId, p.peerId)); +} + +/** + * Memory lists worth counting, at most two per peer. + * + * Memories are keyed by (bot, peer, persona), and a peer can have leftover + * lists from personas it is no longer assigned to. Enumerating peers × all + * personas would explode, so this counts the persona actually in effect — the + * assignment, else the platform default. Memories orphaned by a *past* + * assignment are therefore not counted; that is the documented trade for + * keeping this O(peers) instead of O(peers × personas). + */ +export function memoryKeys( + pairs: readonly PeerPair[], + assignedPersonaByPair: ReadonlyMap, + defaultPersonaId: string | null, +): string[] { + const keys: string[] = []; + for (const pair of pairs) { + const assigned = assignedPersonaByPair.get(pairKey(pair)) ?? null; + const personaIds = new Set(); + if (assigned) personaIds.add(assigned); + if (defaultPersonaId) personaIds.add(defaultPersonaId); + for (const personaId of personaIds) { + keys.push(K.memories(pair.botId, pair.peerId, personaId)); + } + } + return keys; +} + +/** + * Whether the deep counters are affordable for this dataset. + * + * Each peer costs ~4 extra key reads (assignment + messages + up to 2 memory + * lists). Past the cap the snapshot reports `deepStats: false` so the UI can say + * "not measured" instead of showing a zero that reads like "none". + */ +export const DEFAULT_DEEP_STATS_MAX_PEERS = 5000; + +export function deepStatsMaxPeers( + env: NodeJS.ProcessEnv = process.env, +): number { + // Unset must not fall through to Number("") === 0, which would silently + // disable the counters in every default deployment. + const raw = (env.DOCTOR_DEEP_STATS_MAX_PEERS ?? "").trim(); + if (!raw) return DEFAULT_DEEP_STATS_MAX_PEERS; + const n = Number(raw); + if (!Number.isFinite(n)) return DEFAULT_DEEP_STATS_MAX_PEERS; + // An explicit 0 / negative disables the deep counters entirely. + return Math.max(0, Math.floor(n)); +} + +export function shouldComputeDeepStats( + peerCount: number, + maxPeers: number, +): boolean { + if (maxPeers <= 0) return false; + return peerCount <= maxPeers; +} diff --git a/packages/db/src/hash-sticker.test.ts b/packages/db/src/hash-sticker.test.ts new file mode 100644 index 0000000..7bd8a76 --- /dev/null +++ b/packages/db/src/hash-sticker.test.ts @@ -0,0 +1,13 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { hashStickerBlob } from "./repos.js"; + +test("hashStickerBlob is stable 16-hex", () => { + const a = hashStickerBlob(Buffer.from("hello")); + const b = hashStickerBlob(Buffer.from("hello")); + const c = hashStickerBlob(Buffer.from("world")); + assert.equal(a, b); + assert.equal(a.length, 16); + assert.match(a, /^[0-9a-f]{16}$/); + assert.notEqual(a, c); +}); diff --git a/packages/db/src/hot-cache.test.ts b/packages/db/src/hot-cache.test.ts new file mode 100644 index 0000000..6753ae6 --- /dev/null +++ b/packages/db/src/hot-cache.test.ts @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { SnapshotCache } from "./hot-cache.js"; + +const tick = () => new Promise((r) => setTimeout(r, 0)); + +describe("SnapshotCache", () => { + it("serves the cached value within the TTL", async () => { + let loads = 0; + const c = new SnapshotCache(10_000); + const load = async () => ++loads; + + assert.equal(await c.get(load), 1); + assert.equal(await c.get(load), 1); + assert.equal(await c.get(load), 1); + assert.equal(loads, 1); + }); + + it("reloads after invalidate", async () => { + let loads = 0; + const c = new SnapshotCache(10_000); + const load = async () => ++loads; + + await c.get(load); + c.invalidate(); + assert.equal(await c.get(load), 2); + assert.equal(loads, 2); + }); + + it("reloads after the TTL expires", async () => { + let loads = 0; + const c = new SnapshotCache(1); + const load = async () => ++loads; + + await c.get(load); + await new Promise((r) => setTimeout(r, 5)); + assert.equal(await c.get(load), 2); + }); + + it("coalesces concurrent misses into one load", async () => { + let loads = 0; + const c = new SnapshotCache(10_000); + const load = async () => { + loads++; + await tick(); + return loads; + }; + + const all = await Promise.all([c.get(load), c.get(load), c.get(load)]); + assert.deepEqual(all, [1, 1, 1]); + assert.equal(loads, 1); + }); + + it("does not cache a rejected load", async () => { + const c = new SnapshotCache(10_000); + await assert.rejects( + c.get(async () => { + throw new Error("boom"); + }), + ); + assert.equal( + await c.get(async () => 7), + 7, + ); + }); + + it("always loads when the TTL is zero", async () => { + let loads = 0; + const c = new SnapshotCache(0); + const load = async () => ++loads; + + await c.get(load); + await c.get(load); + assert.equal(loads, 2); + }); +}); diff --git a/packages/db/src/hot-cache.ts b/packages/db/src/hot-cache.ts new file mode 100644 index 0000000..b16c155 --- /dev/null +++ b/packages/db/src/hot-cache.ts @@ -0,0 +1,161 @@ +/** + * Process-local TTL cache for hot Redis keys (session / user). + * Cuts 1–2 remote RTTs from almost every authenticated API call. + * + * Not a correctness layer — always safe to miss and re-fetch Redis. + * Disable with REDIS_L1_CACHE=false. + */ + +const enabled = process.env.REDIS_L1_CACHE !== "false"; + +interface Entry { + value: T; + exp: number; +} + +export class TtlCache { + private map = new Map>(); + private readonly ttlMs: number; + private readonly max: number; + + constructor(ttlMs: number, max = 4000) { + this.ttlMs = Math.max(100, ttlMs); + this.max = Math.max(32, max); + } + + get(key: string): T | undefined { + if (!enabled) return undefined; + const e = this.map.get(key); + if (!e) return undefined; + if (Date.now() > e.exp) { + this.map.delete(key); + return undefined; + } + return e.value; + } + + set(key: string, value: T): void { + if (!enabled) return; + if (this.map.size >= this.max) { + // Drop oldest ~10% (Map insertion order) + const n = Math.ceil(this.max * 0.1); + let i = 0; + for (const k of this.map.keys()) { + this.map.delete(k); + if (++i >= n) break; + } + } + this.map.set(key, { value, exp: Date.now() + this.ttlMs }); + } + + del(key: string): void { + this.map.delete(key); + } + + clear(): void { + this.map.clear(); + } +} + +/** Session sid → userId (+ createdAt). Default 45s. */ +export const sessionCache = new TtlCache<{ userId: string; createdAt?: string }>( + Number(process.env.REDIS_L1_SESSION_MS ?? "45000"), + 8000, +); + +/** userId → User JSON shape. Default 30s. */ +export const userCache = new TtlCache( + Number(process.env.REDIS_L1_USER_MS ?? "30000"), + 4000, +); + +/** Default persona id cache. Default 60s. */ +export const defaultPersonaIdCache = new TtlCache( + Number(process.env.REDIS_L1_DEFAULT_PERSONA_MS ?? "60000"), + 4, +); + +/** Published prompt by personaId. Default 60s. */ +export const promptCache = new TtlCache( + Number(process.env.REDIS_L1_PROMPT_MS ?? "60000"), + 500, +); + +/** + * Single-flight TTL snapshot for whole-collection reads. + * + * Square listings (public personas / stickers) previously did + * SMEMBERS + MGET(all) on *every* page / search / sort request. The set only + * changes on publish / review, so one short-lived process snapshot serves the + * whole page-flip session, and concurrent requests share one Redis fetch + * instead of stampeding it. + * + * Multi-node note: a peer node sees a publish after at most `ttlMs`. Same + * tradeoff already taken by promptCache / userCache above. + */ +export class SnapshotCache { + private value: T | undefined; + private exp = 0; + private inflight: Promise | null = null; + private readonly ttlMs: number; + + constructor(ttlMs: number) { + this.ttlMs = Math.max(0, ttlMs); + } + + async get(load: () => Promise): Promise { + if (!enabled || this.ttlMs === 0) return load(); + if (this.value !== undefined && Date.now() < this.exp) return this.value; + if (this.inflight) return this.inflight; + const p = load() + .then((v) => { + this.value = v; + this.exp = Date.now() + this.ttlMs; + return v; + }) + .finally(() => { + this.inflight = null; + }); + this.inflight = p; + return p; + } + + invalidate(): void { + this.value = undefined; + this.exp = 0; + } +} + +/** Default TTL for public square snapshots. Keep short — listings must feel live. */ +export const SQUARE_SNAPSHOT_MS = Number( + process.env.REDIS_L1_SQUARE_MS ?? "10000", +); + +/** + * Super-admin id (earliest-created admin). Resolving it scans every user, and + * /auth/me asks for it on every admin page load. + */ +export const superAdminIdCache = new TtlCache( + Number(process.env.REDIS_L1_SUPERADMIN_MS ?? "120000"), + 4, +); + +export function invalidateSuperAdminCache(): void { + superAdminIdCache.clear(); +} + +export function invalidateUserCache(userId: string): void { + userCache.del(userId); +} + +export function invalidateSessionCache(sid: string): void { + sessionCache.del(sid); +} + +export function invalidatePromptCache(personaId: string): void { + promptCache.del(personaId); +} + +export function invalidateDefaultPersonaCache(): void { + defaultPersonaIdCache.clear(); +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts new file mode 100644 index 0000000..c9c0162 --- /dev/null +++ b/packages/db/src/index.ts @@ -0,0 +1,16 @@ +export * from "./client.js"; +export * from "./keys.js"; +export * from "./hot-cache.js"; +export * from "./password.js"; +export * from "./secret-crypto.js"; +export * from "./llm-provider-repos.js"; +export * from "./invite-repos.js"; +export * from "./doctor-stats.js"; +export * from "./repos.js"; +export * from "./p2p-repos.js"; +export * from "./broadcast-repos.js"; +export * from "./bot-login-repos.js"; +export * from "./seed.js"; +export * from "./worker-fleet.js"; +export * from "./ota-paths.js"; +export * from "./ota-repos.js"; diff --git a/packages/db/src/invite-repos.ts b/packages/db/src/invite-repos.ts new file mode 100644 index 0000000..c59c369 --- /dev/null +++ b/packages/db/src/invite-repos.ts @@ -0,0 +1,372 @@ +import { randomBytes } from "node:crypto"; +import type { RedisStore } from "./client.js"; +import { newId, nowIso } from "./client.js"; +import { K } from "./keys.js"; + +const INVITE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // no O/0/I/1 + +export interface InviteCode { + code: string; + inviterUserId: string; + inviterUsername: string; + createdAt: string; + expiresAt: string; + status: "pending" | "used" | "revoked"; + usedByUserId?: string; + usedAt?: string; +} + +export interface InviteSettings { + quotaWindowHours: number; + quotaMax: number; + codeTtlSec: number; + maxPendingPerUser: number; + codeLength: number; +} + +export const DEFAULT_INVITE_SETTINGS: InviteSettings = { + quotaWindowHours: 24, + quotaMax: 3, + codeTtlSec: 7 * 24 * 3600, + maxPendingPerUser: 20, + codeLength: 10, +}; + +export function generateInviteCode(length = 10): string { + const bytes = randomBytes(length); + let out = ""; + for (let i = 0; i < length; i++) { + out += INVITE_ALPHABET[bytes[i]! % INVITE_ALPHABET.length]!; + } + return out; +} + +export function normalizeInviteCode(code: string): string { + return code.trim().toUpperCase(); +} + +export async function getInviteSettings( + db: RedisStore, + defaults: Partial = {}, +): Promise { + const base: InviteSettings = { + ...DEFAULT_INVITE_SETTINGS, + ...defaults, + }; + const stored = await db.getJson>(K.inviteSettings); + if (!stored) return base; + return { + quotaWindowHours: + typeof stored.quotaWindowHours === "number" && stored.quotaWindowHours >= 0 + ? stored.quotaWindowHours + : base.quotaWindowHours, + quotaMax: + typeof stored.quotaMax === "number" && stored.quotaMax >= 0 + ? Math.floor(stored.quotaMax) + : base.quotaMax, + codeTtlSec: + typeof stored.codeTtlSec === "number" && stored.codeTtlSec >= 60 + ? Math.floor(stored.codeTtlSec) + : base.codeTtlSec, + maxPendingPerUser: + typeof stored.maxPendingPerUser === "number" && + stored.maxPendingPerUser >= 1 + ? Math.floor(stored.maxPendingPerUser) + : base.maxPendingPerUser, + codeLength: + typeof stored.codeLength === "number" && stored.codeLength >= 6 + ? Math.min(32, Math.floor(stored.codeLength)) + : base.codeLength, + }; +} + +export async function setInviteSettings( + db: RedisStore, + patch: Partial, + defaults: Partial = {}, +): Promise { + const cur = await getInviteSettings(db, defaults); + const next: InviteSettings = { + quotaWindowHours: + patch.quotaWindowHours !== undefined + ? Math.max(0, Number(patch.quotaWindowHours) || 0) + : cur.quotaWindowHours, + quotaMax: + patch.quotaMax !== undefined + ? Math.max(0, Math.floor(Number(patch.quotaMax) || 0)) + : cur.quotaMax, + codeTtlSec: + patch.codeTtlSec !== undefined + ? Math.max(60, Math.floor(Number(patch.codeTtlSec) || 60)) + : cur.codeTtlSec, + maxPendingPerUser: + patch.maxPendingPerUser !== undefined + ? Math.max(1, Math.floor(Number(patch.maxPendingPerUser) || 1)) + : cur.maxPendingPerUser, + codeLength: + patch.codeLength !== undefined + ? Math.min(32, Math.max(6, Math.floor(Number(patch.codeLength) || 10))) + : cur.codeLength, + }; + await db.setJson(K.inviteSettings, next); + return next; +} + +export interface InviteQuotaStatus { + used: number; + max: number; + windowHours: number; + remaining: number; + retryAfterSec: number; +} + +export async function getInviteQuotaStatus( + db: RedisStore, + userId: string, + settings: InviteSettings, +): Promise { + const windowMs = Math.max(0, settings.quotaWindowHours) * 3600 * 1000; + const max = settings.quotaMax; + if (max <= 0 || windowMs <= 0) { + return { + used: 0, + max: max <= 0 ? Number.POSITIVE_INFINITY : max, + windowHours: settings.quotaWindowHours, + remaining: max <= 0 ? Number.POSITIVE_INFINITY : max, + retryAfterSec: 0, + }; + } + const key = K.inviteGenLog(userId); + const now = Date.now(); + const cutoff = now - windowMs; + await db.redis.zremrangebyscore(key, 0, cutoff); + const used = await db.redis.zcard(key); + let retryAfterSec = 0; + if (used >= max) { + const oldest = await db.redis.zrange(key, 0, 0, "WITHSCORES"); + if (oldest.length >= 2) { + const oldestMs = Number(oldest[1]); + retryAfterSec = Math.max( + 1, + Math.ceil((oldestMs + windowMs - now) / 1000), + ); + } else { + retryAfterSec = Math.ceil(windowMs / 1000); + } + } + return { + used, + max, + windowHours: settings.quotaWindowHours, + remaining: Math.max(0, max - used), + retryAfterSec, + }; +} + +export async function countPendingInvites( + db: RedisStore, + userId: string, +): Promise { + return db.redis.scard(K.invitesByUser(userId)); +} + +export async function peekInviteCode( + db: RedisStore, + code: string, +): Promise { + const c = normalizeInviteCode(code); + if (!c) return null; + const rec = await db.getJson(K.invite(c)); + if (!rec || rec.status !== "pending") return null; + if (rec.expiresAt && Date.parse(rec.expiresAt) < Date.now()) return null; + return rec; +} + +/** + * Atomically consume a pending invite. Returns null if missing/expired/used. + */ +export async function consumeInviteCode( + db: RedisStore, + code: string, + usedByUserId: string, +): Promise { + const c = normalizeInviteCode(code); + if (!c) return null; + const key = K.invite(c); + const raw = await db.redis.get(key); + if (!raw) return null; + let rec: InviteCode; + try { + rec = JSON.parse(raw) as InviteCode; + } catch { + await db.redis.del(key); + return null; + } + if (rec.status !== "pending") { + await db.redis.del(key); + return null; + } + if (rec.expiresAt && Date.parse(rec.expiresAt) < Date.now()) { + await db.redis.del(key); + if (rec.inviterUserId) { + await db.redis.srem(K.invitesByUser(rec.inviterUserId), c); + } + return null; + } + await db.redis.del(key); + if (rec.inviterUserId) { + await db.redis.srem(K.invitesByUser(rec.inviterUserId), c); + } + const used: InviteCode = { + ...rec, + status: "used", + usedByUserId, + usedAt: nowIso(), + }; + return used; +} + +export async function createInviteCode( + db: RedisStore, + input: { + inviterUserId: string; + inviterUsername: string; + settings: InviteSettings; + }, +): Promise< + | { ok: true; invite: InviteCode; quota: InviteQuotaStatus } + | { + ok: false; + error: "invite_quota" | "invite_pending_limit"; + quota?: InviteQuotaStatus; + maxPending?: number; + } +> { + const { inviterUserId, inviterUsername, settings } = input; + const pending = await countPendingInvites(db, inviterUserId); + if (pending >= settings.maxPendingPerUser) { + return { + ok: false, + error: "invite_pending_limit", + maxPending: settings.maxPendingPerUser, + }; + } + + const quota = await getInviteQuotaStatus(db, inviterUserId, settings); + if ( + settings.quotaMax > 0 && + settings.quotaWindowHours > 0 && + quota.remaining <= 0 + ) { + return { ok: false, error: "invite_quota", quota }; + } + + const ttl = Math.max(60, settings.codeTtlSec); + const expiresAt = new Date(Date.now() + ttl * 1000).toISOString(); + const len = settings.codeLength || 10; + + for (let i = 0; i < 12; i++) { + const code = generateInviteCode(len); + const rec: InviteCode = { + code, + inviterUserId, + inviterUsername, + createdAt: nowIso(), + expiresAt, + status: "pending", + }; + const ok = await db.redis.set( + K.invite(code), + JSON.stringify(rec), + "EX", + ttl, + "NX", + ); + if (ok !== "OK") continue; + + await db.redis.sadd(K.invitesByUser(inviterUserId), code); + + // Record generation for quota (member unique) + if (settings.quotaMax > 0 && settings.quotaWindowHours > 0) { + const logKey = K.inviteGenLog(inviterUserId); + const now = Date.now(); + const member = `${now}:${code}:${newId("ig")}`; + await db.redis.zadd(logKey, now, member); + const windowMs = settings.quotaWindowHours * 3600 * 1000; + await db.redis.zremrangebyscore(logKey, 0, now - windowMs); + // Keep log a bit longer than window + await db.redis.pexpire(logKey, windowMs + 3600_000); + } + + const nextQuota = await getInviteQuotaStatus( + db, + inviterUserId, + settings, + ); + return { ok: true, invite: rec, quota: nextQuota }; + } + throw new Error("failed to allocate invite code"); +} + +export async function listPendingInvites( + db: RedisStore, + userId: string, +): Promise { + const codes = (await db.redis.smembers(K.invitesByUser(userId))) as string[]; + if (!codes.length) return []; + const rows = await db.mgetJson( + codes.map((c) => K.invite(c)), + ); + const out: InviteCode[] = []; + const stale: string[] = []; + for (let i = 0; i < codes.length; i++) { + const rec = rows[i]; + const code = codes[i]!; + if (!rec || rec.status !== "pending") { + stale.push(code); + continue; + } + if (rec.expiresAt && Date.parse(rec.expiresAt) < Date.now()) { + stale.push(code); + await db.redis.del(K.invite(code)); + continue; + } + out.push(rec); + } + if (stale.length) { + await db.redis.srem(K.invitesByUser(userId), ...stale); + } + return out.sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +export async function revokeInviteCode( + db: RedisStore, + userId: string, + code: string, +): Promise { + const c = normalizeInviteCode(code); + if (!c) return false; + const rec = await db.getJson(K.invite(c)); + if (!rec) { + await db.redis.srem(K.invitesByUser(userId), c); + return false; + } + if (rec.inviterUserId !== userId) return false; + await db.redis.del(K.invite(c)); + await db.redis.srem(K.invitesByUser(userId), c); + return true; +} + +/** Revoke all pending invites for a user (e.g. on account delete). */ +export async function revokeAllInvitesForUser( + db: RedisStore, + userId: string, +): Promise { + const codes = (await db.redis.smembers(K.invitesByUser(userId))) as string[]; + if (!codes.length) return 0; + for (const c of codes) { + await db.redis.del(K.invite(c)); + } + await db.redis.del(K.invitesByUser(userId)); + return codes.length; +} diff --git a/packages/db/src/keys.ts b/packages/db/src/keys.ts new file mode 100644 index 0000000..9c1d108 --- /dev/null +++ b/packages/db/src/keys.ts @@ -0,0 +1,216 @@ +export const K = { + user: (id: string) => `wa:user:${id}`, + usersAll: "wa:users:all", + adminIds: "wa:admins", + session: (sid: string) => `wa:session:${sid}`, + /** SET of session ids for a user (ban/kick all sessions) */ + sessionsByUser: (userId: string) => `wa:sessions:by:${userId}`, + oauthState: (state: string) => `wa:oauth:state:${state}`, + + // ── Invites (local password registration) ───────────── + /** Pending invite code JSON (TTL) */ + invite: (code: string) => `wa:invite:${code}`, + /** SET of pending invite codes created by user */ + invitesByUser: (userId: string) => `wa:invites:by:${userId}`, + /** ZSET score=ms of generation events for sliding-window quota */ + inviteGenLog: (userId: string) => `wa:invite:genlog:${userId}`, + /** Global invite policy overrides (JSON) */ + inviteSettings: "wa:settings:invites", + /** Admin-editable runtime config overrides on top of env (JSON) */ + runtimeSettings: "wa:settings:runtime", + /** Short lock serializing the read-modify-write of runtimeSettings */ + runtimeSettingsLock: "wa:settings:runtime:lock", + + bot: (id: string) => `wa:bot:${id}`, + /** Sensitive iLink bot_token — never expose via list/admin JSON without care */ + botCreds: (id: string) => `wa:bot:${id}:creds`, + botsAll: "wa:bots:all", + botsByOwner: (userId: string) => `wa:bots:owner:${userId}`, + + persona: (id: string) => `wa:persona:${id}`, + personaSlug: (slug: string) => `wa:persona:slug:${slug}`, + personasAll: "wa:personas:all", + personasPublic: "wa:personas:public", + /** Single-key pointer to current default persona id (avoids full list scan) */ + personaDefault: "wa:persona:default", + personasByOwner: (userId: string) => `wa:personas:owner:${userId}`, + personaLib: (userId: string) => `wa:user:${userId}:persona_lib`, + personaVersion: (id: string) => `wa:pver:${id}`, + personaVersions: (personaId: string) => `wa:persona:${personaId}:versions`, + + peer: (botId: string, peerId: string) => `wa:peer:${botId}:${peerId}`, + peersByBot: (botId: string) => `wa:peers:bot:${botId}`, + peersAll: "wa:peers:all", + + assignment: (botId: string, peerId: string) => + `wa:asg:${botId}:${peerId}`, + + messages: (botId: string, peerId: string) => + `wa:msgs:${botId}:${peerId}`, + msgCountUser: (botId: string, peerId: string) => + `wa:msgcount:${botId}:${peerId}`, + + memories: (botId: string, peerId: string, personaId: string) => + `wa:mem:${botId}:${peerId}:${personaId}`, + + contextToken: (botId: string, peerId: string) => + `wa:ctx:${botId}:${peerId}`, + + /** Peers with proactive outreach enabled for a bot (SET of peerId) */ + proactivePeersByBot: (botId: string) => `wa:proactive:peers:${botId}`, + /** Distributed lock while generating/sending a proactive message */ + proactiveLock: (botId: string, peerId: string) => + `wa:proactive:lock:${botId}:${peerId}`, + /** Daily proactive send counter per peer (YYYY-MM-DD) */ + proactiveDayCount: (botId: string, peerId: string, day: string) => + `wa:proactive:day:${botId}:${peerId}:${day}`, + + audit: "wa:audit", + usageDay: (day: string) => `wa:usage:day:${day}`, + usageDayUser: (day: string, userId: string) => + `wa:usage:day:${day}:user:${userId}`, + usageDayBot: (day: string, botId: string) => + `wa:usage:day:${day}:bot:${botId}`, + /** SET index of user ids that spent tokens on `day` (TTL like the hashes) */ + usageDayUsers: (day: string) => `wa:usage:day:${day}:users`, + /** SET index of bot ids that spent tokens on `day` */ + usageDayBots: (day: string) => `wa:usage:day:${day}:bots`, + + /** + * Fleet / scale-out keys (single host multi-worker or multi-host). + * - pollable: bots that should be long-polled (active + token + not paused) + * - lease: which worker currently owns getUpdates for a bot + * - inbox: inbound message jobs for reply consumers + */ + botsPollable: "wa:bots:pollable", + botLease: (botId: string) => `wa:bot:${botId}:lease`, + botPaused: (botId: string) => `wa:bot:${botId}:paused`, + workersReg: "wa:workers:reg", + workerMeta: (workerId: string) => `wa:worker:${workerId}`, + workerBots: (workerId: string) => `wa:worker:${workerId}:bots`, + /** + * Admin force-offline fence. While present, that worker must not + * re-register, renew, or claim leases (until cleared). + */ + workerFence: (workerId: string) => `wa:worker:${workerId}:fence`, + /** SET of currently fenced worker ids (admin list) */ + workersFenced: "wa:workers:fenced", + /** + * HASH workerId → WorkerWeight JSON. Admin load weight in percent + * (100 = default share). Only nodes with an override have a field. + * Written by admin actions only. + */ + workerWeights: "wa:workers:weights", + /** + * HASH workerId → ISO timestamp of the last heartbeat the fleet saw for a + * weighted node. Deliberately separate from `workerWeights`: the GC stamps + * this on its own cadence, and must never read-modify-write a weight record + * an admin may be editing at the same moment. + */ + workerWeightsSeen: "wa:workers:weights:seen", + /** LIST of JSON InboundJob — RPUSH / BLPOP */ + inbox: "wa:inbox", + /** + * Idempotency gate for one inbound WeChat message (SET NX EX). + * iLink delivers no stable msg_id, so the worker synthesizes a key from + * bot/peer/create_time_ms/text/media and uses this to stop a re-delivered + * long-poll batch from spawning a second LLM generation. + */ + inboundSeen: (dedupKey: string) => `wa:inbound:seen:${dedupKey}`, + /** Pub/Sub channel — workers re-run claim on "wake" */ + workerWake: "wa:worker:wake", + + /** + * WeChat QR bot login session (view JSON, TTL). + * Poll loop stays on the node that started login; any node can read status. + */ + botLogin: (sessionId: string) => `wa:botlogin:${sessionId}`, + /** SET of active login session ids for a user (optional cleanup index) */ + botLoginsByOwner: (userId: string) => `wa:botlogin:owner:${userId}`, + + /** Sticker square (blob in Redis; admin review for public) */ + sticker: (id: string) => `wa:sticker:${id}`, + /** Raw image bytes (binary value) */ + stickerBlob: (id: string) => `wa:sticker:${id}:blob`, + stickersAll: "wa:stickers:all", + /** approved + public + enabled — square listing */ + stickersPublic: "wa:stickers:public", + /** pending review */ + stickersPending: "wa:stickers:pending", + stickersByOwner: (userId: string) => `wa:stickers:owner:${userId}`, + stickerSlug: (slug: string) => `wa:sticker:slug:${slug}`, + stickerLib: (userId: string) => `wa:user:${userId}:sticker_lib`, + + // ── User custom LLM providers (secrets encrypted; egress via HF tools) ── + llmProvider: (id: string) => `wa:llmp:${id}`, + llmProvidersByOwner: (userId: string) => `wa:llmp:owner:${userId}`, + + // ── P2P / WeChat ↔ LINUX DO bind ───────────────────── + /** O(1) username → userId (lowercase) */ + userByName: (lowerUsername: string) => `wa:user:name:${lowerUsername}`, + /** Primary WeChat bind for a LINUX DO user (JSON UserWechatBind) */ + bindUser: (userId: string) => `wa:bind:user:${userId}`, + /** Reverse: peer endpoint → userId */ + bindPeer: (botId: string, peerId: string) => `wa:bind:peer:${botId}:${peerId}`, + /** Single-use bind code (TTL) */ + bindCode: (code: string) => `wa:bind:code:${code}`, + /** Connect request JSON (TTL) */ + connectReq: (requestId: string) => `wa:connect:req:${requestId}`, + connectFrom: (botId: string, peerId: string) => + `wa:connect:from:${botId}:${peerId}`, + connectTo: (botId: string, peerId: string) => + `wa:connect:to:${botId}:${peerId}`, + /** Active P2P session JSON (idle TTL, refreshed on activity) */ + p2pSession: (sessionId: string) => `wa:p2p:${sessionId}`, + p2pPeer: (botId: string, peerId: string) => `wa:p2p:peer:${botId}:${peerId}`, + /** Daily @ request counter per peer endpoint */ + p2pRequestDay: (botId: string, peerId: string, day: string) => + `wa:p2p:reqday:${botId}:${peerId}:${day}`, + /** + * SET of blocked LINUX DO userIds for a user. + * A blocks B → B cannot @ A (and A cannot @ B) for P2P. + */ + blockSet: (userId: string) => `wa:block:${userId}`, + + // ── Admin broadcast (text push jobs) ─────────────────── + /** Broadcast job JSON */ + broadcast: (id: string) => `wa:broadcast:${id}`, + /** LIST of recent broadcast job ids (newest first) */ + broadcastsAll: "wa:broadcasts:all", + /** Distributed lock while a worker processes a job */ + broadcastLock: (id: string) => `wa:broadcast:${id}:lock`, + /** Optional pointer to currently running job id */ + broadcastActive: "wa:broadcast:active", + + // ── Web try-chat (ephemeral, no WeChat) ─────────────── + /** Try-chat session meta JSON { userId, personaId, botName, createdAt, msgCount } */ + trySession: (sessionId: string) => `wa:try:${sessionId}`, + /** Try-chat message list (JSON {role,content}) */ + trySessionMsgs: (sessionId: string) => `wa:try:${sessionId}:msgs`, + /** Daily try-chat user-message counter (YYYY-MM-DD) */ + tryDayCount: (userId: string, day: string) => `wa:try:day:${userId}:${day}`, + + // ── OTA releases (file-level fleet update packs) ─────── + /** Current channel release meta (ReleaseMeta JSON, no blobs) */ + releaseCurrent: "wa:release:current", + /** Release meta by version string */ + releaseMeta: (version: string) => `wa:release:meta:${version}`, + /** ZSET score=ms createdAt, member=version — recent versions */ + releaseVersions: "wa:release:versions", + /** Content-addressed blob index { chunks, size, sha256 } */ + releaseBlobMeta: (sha256: string) => `wa:release:blob:${sha256}`, + /** Binary chunk n for content hash */ + releaseBlobChunk: (sha256: string, n: number) => + `wa:release:blob:${sha256}:${n}`, + /** Per-worker update job (NodeUpdateJob JSON, TTL) */ + workerUpdate: (workerId: string) => `wa:worker:${workerId}:update`, + /** Per-worker update progress/status (NodeUpdateStatus JSON, TTL) */ + workerUpdateStatus: (workerId: string) => + `wa:worker:${workerId}:update:status`, + + // ── Admin live activity stream (fleet fan-in) ────────── + /** Pub/Sub channel for important stream events (not redis.cmd samples) */ + streamChannel: "wa:stream:events", + /** LIST of recent stream events JSON (newest first, LTRIM) */ + streamRecent: "wa:stream:recent", +}; diff --git a/packages/db/src/llm-provider-repos.ts b/packages/db/src/llm-provider-repos.ts new file mode 100644 index 0000000..b19abab --- /dev/null +++ b/packages/db/src/llm-provider-repos.ts @@ -0,0 +1,233 @@ +import type { RedisStore } from "./client.js"; +import { newId, nowIso } from "./client.js"; +import { K } from "./keys.js"; +import { + decryptSecret, + encryptSecret, + maskApiKey, +} from "./secret-crypto.js"; + +export interface UserLlmProvider { + id: string; + owner_user_id: string; + name: string; + base_url: string; + /** AES-GCM ciphertext; never expose via public API */ + api_key_enc: string; + default_model: string; + enabled: number; + created_at: string; + updated_at: string; +} + +export interface UserLlmProviderPublic { + id: string; + name: string; + baseUrl: string; + defaultModel: string; + enabled: boolean; + apiKeyMasked: string; + createdAt: string; + updatedAt: string; +} + +function normalizeBaseUrl(url: string): string { + return url.trim().replace(/\/+$/, ""); +} + +function assertSafeBaseUrl(url: string): string { + const u = normalizeBaseUrl(url); + if (!u) throw new Error("base_url required"); + let parsed: URL; + try { + parsed = new URL(u); + } catch { + throw new Error("base_url invalid"); + } + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new Error("base_url must be http(s)"); + } + const host = (parsed.hostname || "").toLowerCase(); + if ( + host === "localhost" || + host === "127.0.0.1" || + host === "0.0.0.0" || + host.endsWith(".local") || + host === "metadata" || + host === "metadata.google.internal" + ) { + throw new Error("base_url host not allowed"); + } + // Main site still must not dial user URL; HF tools re-checks private IPs. + return u; +} + +export function toPublicProvider( + p: UserLlmProvider, + secret: string, +): UserLlmProviderPublic { + let masked = "****"; + try { + masked = maskApiKey(decryptSecret(p.api_key_enc, secret)); + } catch { + masked = "****"; + } + return { + id: p.id, + name: p.name, + baseUrl: p.base_url, + defaultModel: p.default_model, + enabled: Boolean(p.enabled), + apiKeyMasked: masked, + createdAt: p.created_at, + updatedAt: p.updated_at, + }; +} + +export async function getLlmProvider( + db: RedisStore, + id: string, +): Promise { + return db.getJson(K.llmProvider(id)); +} + +export async function listLlmProvidersByOwner( + db: RedisStore, + ownerUserId: string, +): Promise { + const ids = await db.redis.smembers(K.llmProvidersByOwner(ownerUserId)); + if (!ids.length) return []; + const rows = await Promise.all(ids.map((id) => getLlmProvider(db, id))); + return rows + .filter((r): r is UserLlmProvider => Boolean(r)) + .sort((a, b) => (a.created_at < b.created_at ? 1 : -1)); +} + +export async function createLlmProvider( + db: RedisStore, + input: { + ownerUserId: string; + name: string; + baseUrl: string; + apiKey: string; + defaultModel: string; + secret: string; + }, +): Promise { + const name = input.name.trim().slice(0, 64); + if (!name) throw new Error("name required"); + const baseUrl = assertSafeBaseUrl(input.baseUrl); + const apiKey = input.apiKey.trim(); + if (!apiKey) throw new Error("api_key required"); + const model = input.defaultModel.trim().slice(0, 128); + if (!model) throw new Error("default_model required"); + + const id = newId("llmp"); + const now = nowIso(); + const row: UserLlmProvider = { + id, + owner_user_id: input.ownerUserId, + name, + base_url: baseUrl, + api_key_enc: encryptSecret(apiKey, input.secret), + default_model: model, + enabled: 1, + created_at: now, + updated_at: now, + }; + await db.setJson(K.llmProvider(id), row); + await db.redis.sadd(K.llmProvidersByOwner(input.ownerUserId), id); + return row; +} + +export async function updateLlmProvider( + db: RedisStore, + id: string, + ownerUserId: string, + patch: { + name?: string; + baseUrl?: string; + apiKey?: string; + defaultModel?: string; + enabled?: boolean; + secret: string; + }, +): Promise { + const row = await getLlmProvider(db, id); + if (!row || row.owner_user_id !== ownerUserId) { + throw new Error("not_found"); + } + if (patch.name !== undefined) { + const name = patch.name.trim().slice(0, 64); + if (!name) throw new Error("name required"); + row.name = name; + } + if (patch.baseUrl !== undefined) { + row.base_url = assertSafeBaseUrl(patch.baseUrl); + } + if (patch.apiKey !== undefined && patch.apiKey.trim()) { + row.api_key_enc = encryptSecret(patch.apiKey.trim(), patch.secret); + } + if (patch.defaultModel !== undefined) { + const model = patch.defaultModel.trim().slice(0, 128); + if (!model) throw new Error("default_model required"); + row.default_model = model; + } + if (patch.enabled !== undefined) { + row.enabled = patch.enabled ? 1 : 0; + } + row.updated_at = nowIso(); + await db.setJson(K.llmProvider(id), row); + return row; +} + +export async function deleteLlmProvider( + db: RedisStore, + id: string, + ownerUserId: string, +): Promise { + const row = await getLlmProvider(db, id); + if (!row || row.owner_user_id !== ownerUserId) return false; + await db.redis.del(K.llmProvider(id)); + await db.redis.srem(K.llmProvidersByOwner(ownerUserId), id); + return true; +} + +export interface ResolvedUpstream { + baseUrl: string; + apiKey: string; + model: string; + providerId: string; +} + +/** + * Resolve user custom upstream for a persona. Returns null → use platform LLM. + * Caller must only use this with tools gateway (never dial baseUrl from main). + */ +export async function resolvePersonaUpstream( + db: RedisStore, + opts: { + llmProviderId: string | null | undefined; + /** Bot owner must own the provider */ + ownerUserId: string | null | undefined; + secret: string; + }, +): Promise { + const pid = (opts.llmProviderId || "").trim(); + if (!pid || !opts.ownerUserId) return null; + const row = await getLlmProvider(db, pid); + if (!row || !row.enabled) return null; + if (row.owner_user_id !== opts.ownerUserId) return null; + let apiKey: string; + try { + apiKey = decryptSecret(row.api_key_enc, opts.secret); + } catch { + return null; + } + return { + baseUrl: row.base_url, + apiKey, + model: row.default_model, + providerId: row.id, + }; +} diff --git a/packages/db/src/ota-paths.test.ts b/packages/db/src/ota-paths.test.ts new file mode 100644 index 0000000..7059196 --- /dev/null +++ b/packages/db/src/ota-paths.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + isAllowedOtaPath, + isValidReleaseVersion, + normalizeOtaPath, + pathRequiresInstall, +} from "./ota-paths.js"; + +describe("ota-paths", () => { + it("normalizes and rejects traversal", () => { + assert.equal(normalizeOtaPath("apps/api/src/index.ts"), "apps/api/src/index.ts"); + assert.equal(normalizeOtaPath("apps\\api\\src\\a.ts"), "apps/api/src/a.ts"); + assert.equal(normalizeOtaPath("../etc/passwd"), null); + assert.equal(normalizeOtaPath("/abs"), null); + assert.equal(normalizeOtaPath("apps/api/../../x"), null); + }); + + it("allows whitelist paths", () => { + assert.equal(isAllowedOtaPath("package.json"), true); + assert.equal(isAllowedOtaPath("apps/api/src/routes.ts"), true); + assert.equal(isAllowedOtaPath("apps/api/public/admin.html"), true); + assert.equal(isAllowedOtaPath("packages/db/src/keys.ts"), true); + assert.equal(isAllowedOtaPath("packages/db/package.json"), true); + assert.equal(isAllowedOtaPath("scripts/release-pack.mjs"), true); + }); + + it("denies secrets, data, node_modules, non-src package files", () => { + assert.equal(isAllowedOtaPath(".env"), false); + assert.equal(isAllowedOtaPath("apps/api/data/x.db"), false); + assert.equal(isAllowedOtaPath("apps/api/node_modules/x"), false); + assert.equal(isAllowedOtaPath("packages/db/README.md"), false); + assert.equal(isAllowedOtaPath("Dockerfile"), false); + assert.equal(isAllowedOtaPath("cloudflare-worker/src/index.ts"), false); + }); + + it("detects install triggers", () => { + assert.equal(pathRequiresInstall("pnpm-lock.yaml"), true); + assert.equal(pathRequiresInstall("packages/core/package.json"), true); + assert.equal(pathRequiresInstall("apps/api/src/index.ts"), false); + }); + + it("validates version labels", () => { + assert.equal(isValidReleaseVersion("0.2.1"), true); + assert.equal(isValidReleaseVersion("0.2.1-ota.1"), true); + assert.equal(isValidReleaseVersion(""), false); + assert.equal(isValidReleaseVersion("../x"), false); + }); +}); diff --git a/packages/db/src/ota-paths.ts b/packages/db/src/ota-paths.ts new file mode 100644 index 0000000..f67e657 --- /dev/null +++ b/packages/db/src/ota-paths.ts @@ -0,0 +1,124 @@ +/** + * OTA path whitelist / validation shared by pack CLI and node apply. + * Paths are POSIX-style relative to monorepo root (no leading slash). + */ + +/** Exact root files included in a release pack. */ +export const OTA_ROOT_FILES = [ + "package.json", + "pnpm-workspace.yaml", + "pnpm-lock.yaml", + "tsconfig.base.json", +] as const; + +/** Directory prefixes allowed (recursive). */ +export const OTA_DIR_PREFIXES = [ + "apps/api/", + "packages/core/", + "packages/db/", + "packages/ilink/", + "packages/llm/", + "scripts/", +] as const; + +const DENY_SEGMENT = new Set([ + "node_modules", + "data", + ".git", + ".wa-update-staging", + ".wa-backup", + "dist", + "coverage", +]); + +const DENY_BASENAME = new Set([ + ".env", + ".env.local", + ".env.production", + ".ds_store", +]); + +const DENY_EXT = new Set([".db", ".db-wal", ".db-shm", ".log", ".bak"]); + +/** Files that force requiresInstall when present in the changed set. */ +export const OTA_INSTALL_TRIGGER_FILES = new Set([ + "pnpm-lock.yaml", + "package.json", + "pnpm-workspace.yaml", + "apps/api/package.json", + "packages/core/package.json", + "packages/db/package.json", + "packages/ilink/package.json", + "packages/llm/package.json", +]); + +/** + * Normalize to POSIX relative path without leading `./` or `/`. + * Returns null if path escapes or is empty. + */ +export function normalizeOtaPath(input: string): string | null { + if (!input || typeof input !== "string") return null; + let p = input.replace(/\\/g, "/").trim(); + if (!p || p.startsWith("/") || /^[a-zA-Z]:/.test(p)) return null; + // collapse ./ and // + const parts: string[] = []; + for (const seg of p.split("/")) { + if (!seg || seg === ".") continue; + if (seg === "..") return null; + parts.push(seg); + } + if (!parts.length) return null; + return parts.join("/"); +} + +export function isDeniedOtaPath(relPath: string): boolean { + const lower = relPath.toLowerCase(); + const base = lower.split("/").pop() || ""; + if (DENY_BASENAME.has(base)) return true; + if (base.startsWith(".env")) return true; + for (const ext of DENY_EXT) { + if (lower.endsWith(ext)) return true; + } + const segs = lower.split("/"); + for (const s of segs) { + if (DENY_SEGMENT.has(s)) return true; + } + return false; +} + +/** + * Whether a normalized relative path may appear in an OTA pack / be written. + */ +export function isAllowedOtaPath(relPath: string): boolean { + const n = normalizeOtaPath(relPath); + if (!n) return false; + if (isDeniedOtaPath(n)) return false; + + if ((OTA_ROOT_FILES as readonly string[]).includes(n)) return true; + + for (const prefix of OTA_DIR_PREFIXES) { + if (!n.startsWith(prefix)) continue; + // packages/* : only package.json, tsconfig.json, and src/** + if (prefix.startsWith("packages/")) { + const rest = n.slice(prefix.length); + if (rest === "package.json" || rest === "tsconfig.json") return true; + if (rest.startsWith("src/")) return true; + return false; + } + // apps/api and scripts: all non-denied files under prefix + return true; + } + return false; +} + +export function pathRequiresInstall(relPath: string): boolean { + const n = normalizeOtaPath(relPath); + if (!n) return false; + return OTA_INSTALL_TRIGGER_FILES.has(n); +} + +/** Validate version string for release ids (semver-ish or freeform label). */ +export function isValidReleaseVersion(v: string): boolean { + if (!v || v.length > 64) return false; + return /^[0-9A-Za-z][0-9A-Za-z._+-]*$/.test(v); +} diff --git a/packages/db/src/ota-repos.test.ts b/packages/db/src/ota-repos.test.ts new file mode 100644 index 0000000..dff828d --- /dev/null +++ b/packages/db/src/ota-repos.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + buildReleaseMeta, + computePackSha256, + diffReleaseFiles, + sha256Buffer, +} from "./ota-repos.js"; + +describe("ota-repos pure helpers", () => { + it("builds meta and pack hash", () => { + const a = { + path: "apps/api/src/a.ts", + sha256: "a".repeat(64), + size: 10, + }; + const b = { + path: "package.json", + sha256: "b".repeat(64), + size: 20, + }; + const meta = buildReleaseMeta({ version: "0.3.0", files: [b, a] }); + assert.equal(meta.version, "0.3.0"); + assert.equal(meta.fileCount, 2); + assert.equal(meta.requiresInstall, true); // package.json + assert.equal(meta.totalBytes, 30); + assert.equal(meta.packSha256, computePackSha256(meta.files)); + assert.equal(meta.files[0]!.path, "apps/api/src/a.ts"); + }); + + it("rejects bad paths", () => { + assert.throws( + () => + buildReleaseMeta({ + version: "1.0.0", + files: [{ path: ".env", sha256: "c".repeat(64), size: 1 }], + }), + /path_not_allowed/, + ); + }); + + it("diffs local hashes", () => { + const meta = buildReleaseMeta({ + version: "1.0.0", + files: [ + { path: "apps/api/src/a.ts", sha256: "a".repeat(64), size: 1 }, + { path: "apps/api/src/b.ts", sha256: "b".repeat(64), size: 1 }, + ], + }); + const local = new Map([["apps/api/src/a.ts", "a".repeat(64)]]); + const needed = diffReleaseFiles(meta, local); + assert.equal(needed.length, 1); + assert.equal(needed[0]!.path, "apps/api/src/b.ts"); + }); + + it("hashes buffers", () => { + const h = sha256Buffer(Buffer.from("hello")); + assert.equal(h.length, 64); + assert.equal(h, sha256Buffer(Buffer.from("hello"))); + }); +}); diff --git a/packages/db/src/ota-repos.ts b/packages/db/src/ota-repos.ts new file mode 100644 index 0000000..82b2d59 --- /dev/null +++ b/packages/db/src/ota-repos.ts @@ -0,0 +1,413 @@ +import { createHash } from "node:crypto"; +import type { RedisStore } from "./client.js"; +import { nowIso } from "./client.js"; +import { K } from "./keys.js"; +import { + isAllowedOtaPath, + isValidReleaseVersion, + normalizeOtaPath, + pathRequiresInstall, +} from "./ota-paths.js"; + +/** ~384 KiB raw per chunk — safe for Upstash / pipeline sizes. */ +export const OTA_BLOB_CHUNK_BYTES = 384 * 1024; + +/** Keep last N release versions in index. */ +export const OTA_RELEASE_HISTORY = 20; + +/** Update job / status TTL (seconds). */ +export const OTA_JOB_TTL_SEC = 30 * 60; + +export interface ReleaseFileEntry { + path: string; + sha256: string; + size: number; +} + +export interface ReleaseMeta { + version: string; + createdAt: string; + createdBy?: string | null; + files: ReleaseFileEntry[]; + requiresInstall: boolean; + totalBytes: number; + /** Hash of sorted "path:sha256" lines for integrity */ + packSha256: string; + fileCount: number; +} + +export interface ReleaseBlobMeta { + sha256: string; + size: number; + chunks: number; +} + +export type NodeUpdatePhase = + | "pending" + | "downloading" + | "applying" + | "installing" + | "restarting" + | "failed" + | "done"; + +export interface NodeUpdateJob { + version: string; + requestedAt: string; + requestedBy?: string | null; + requestedByUsername?: string | null; + /** Force re-apply even if already on version */ + force?: boolean; +} + +export interface NodeUpdateStatus { + workerId: string; + version: string; + phase: NodeUpdatePhase; + error?: string | null; + startedAt: string; + updatedAt: string; + progress?: { + done: number; + total: number; + bytesDone?: number; + bytesTotal?: number; + }; + changedFiles?: number; + message?: string | null; +} + +export function computePackSha256(files: ReleaseFileEntry[]): string { + const lines = [...files] + .map((f) => `${f.path}:${f.sha256}`) + .sort() + .join("\n"); + return createHash("sha256").update(lines, "utf8").digest("hex"); +} + +export function sha256Buffer(buf: Buffer): string { + return createHash("sha256").update(buf).digest("hex"); +} + +export function buildReleaseMeta(input: { + version: string; + files: ReleaseFileEntry[]; + createdBy?: string | null; + createdAt?: string; +}): ReleaseMeta { + const version = input.version.trim(); + if (!isValidReleaseVersion(version)) { + throw new Error("invalid_version"); + } + const files: ReleaseFileEntry[] = []; + for (const f of input.files) { + const p = normalizeOtaPath(f.path); + if (!p || !isAllowedOtaPath(p)) { + throw new Error(`path_not_allowed:${f.path}`); + } + if (!/^[a-f0-9]{64}$/i.test(f.sha256)) { + throw new Error(`invalid_sha256:${f.path}`); + } + if (!Number.isFinite(f.size) || f.size < 0) { + throw new Error(`invalid_size:${f.path}`); + } + files.push({ + path: p, + sha256: f.sha256.toLowerCase(), + size: Math.floor(f.size), + }); + } + files.sort((a, b) => a.path.localeCompare(b.path)); + // dedupe paths + for (let i = 1; i < files.length; i++) { + if (files[i]!.path === files[i - 1]!.path) { + throw new Error(`duplicate_path:${files[i]!.path}`); + } + } + const requiresInstall = files.some((f) => pathRequiresInstall(f.path)); + const totalBytes = files.reduce((a, f) => a + f.size, 0); + return { + version, + createdAt: input.createdAt ?? nowIso(), + createdBy: input.createdBy ?? null, + files, + requiresInstall, + totalBytes, + packSha256: computePackSha256(files), + fileCount: files.length, + }; +} + +export async function getCurrentRelease( + db: RedisStore, +): Promise { + return db.getJson(K.releaseCurrent); +} + +export async function getReleaseMeta( + db: RedisStore, + version: string, +): Promise { + const v = version.trim(); + if (!v) return null; + return db.getJson(K.releaseMeta(v)); +} + +export async function listReleaseVersions( + db: RedisStore, + limit = 20, +): Promise> { + const lim = Math.max(1, Math.min(100, limit)); + const members = (await db.redis.zrevrange( + K.releaseVersions, + 0, + lim - 1, + )) as string[]; + if (!members.length) return []; + const metas = await db.mgetJson( + members.map((v) => K.releaseMeta(v)), + ); + const out: Array<{ version: string; createdAt: string }> = []; + members.forEach((v, i) => { + const m = metas[i]; + out.push({ + version: v, + createdAt: m?.createdAt ?? "", + }); + }); + return out; +} + +export async function blobExists( + db: RedisStore, + sha256: string, +): Promise { + return (await db.redis.exists(K.releaseBlobMeta(sha256.toLowerCase()))) === 1; +} + +export async function putBlobChunks( + db: RedisStore, + sha256: string, + data: Buffer, +): Promise { + const hash = sha256.toLowerCase(); + if (sha256Buffer(data) !== hash) { + throw new Error("blob_sha256_mismatch"); + } + if (await blobExists(db, hash)) { + const existing = await db.getJson(K.releaseBlobMeta(hash)); + if (existing) return existing; + } + // empty file: one empty chunk + const n = + data.length === 0 + ? 1 + : Math.ceil(data.length / OTA_BLOB_CHUNK_BYTES); + const pipe = db.redis.pipeline(); + for (let i = 0; i < n; i++) { + const start = i * OTA_BLOB_CHUNK_BYTES; + const slice = + data.length === 0 + ? Buffer.alloc(0) + : data.subarray(start, start + OTA_BLOB_CHUNK_BYTES); + pipe.set(K.releaseBlobChunk(hash, i), slice); + } + const meta: ReleaseBlobMeta = { + sha256: hash, + size: data.length, + chunks: n, + }; + pipe.set(K.releaseBlobMeta(hash), JSON.stringify(meta)); + await pipe.exec(); + return meta; +} + +export async function getBlob( + db: RedisStore, + sha256: string, +): Promise { + const hash = sha256.toLowerCase(); + const meta = await db.getJson(K.releaseBlobMeta(hash)); + if (!meta || meta.chunks < 1) return null; + const parts: Buffer[] = []; + for (let i = 0; i < meta.chunks; i++) { + const buf = await db.redis.getBuffer(K.releaseBlobChunk(hash, i)); + if (buf == null) return null; + parts.push(Buffer.isBuffer(buf) ? buf : Buffer.from(buf)); + } + const data = Buffer.concat(parts); + if (data.length !== meta.size) return null; + if (sha256Buffer(data) !== hash) return null; + return data; +} + +/** + * Register release meta + set as current. Blobs must already exist for all files. + */ +export async function publishRelease( + db: RedisStore, + meta: ReleaseMeta, + opts?: { setCurrent?: boolean }, +): Promise { + const setCurrent = opts?.setCurrent !== false; + // verify blobs present + const missing: string[] = []; + for (const f of meta.files) { + if (!(await blobExists(db, f.sha256))) { + missing.push(f.path); + if (missing.length >= 10) break; + } + } + if (missing.length) { + throw new Error(`missing_blobs:${missing.join(",")}`); + } + await db.setJson(K.releaseMeta(meta.version), meta); + const score = Date.parse(meta.createdAt) || Date.now(); + await db.redis.zadd(K.releaseVersions, score, meta.version); + // trim history + const card = await db.redis.zcard(K.releaseVersions); + if (card > OTA_RELEASE_HISTORY) { + await db.redis.zremrangebyrank( + K.releaseVersions, + 0, + card - OTA_RELEASE_HISTORY - 1, + ); + } + if (setCurrent) { + await db.setJson(K.releaseCurrent, meta); + } +} + +export async function setCurrentRelease( + db: RedisStore, + version: string, +): Promise { + const meta = await getReleaseMeta(db, version); + if (!meta) throw new Error("release_not_found"); + await db.setJson(K.releaseCurrent, meta); + return meta; +} + +export async function getWorkerUpdateJob( + db: RedisStore, + workerId: string, +): Promise { + return db.getJson(K.workerUpdate(workerId)); +} + +export async function getWorkerUpdateStatus( + db: RedisStore, + workerId: string, +): Promise { + return db.getJson(K.workerUpdateStatus(workerId)); +} + +export async function getWorkerUpdateStatuses( + db: RedisStore, + workerIds: string[], +): Promise> { + const map = new Map(); + if (!workerIds.length) return map; + const rows = await db.mgetJson( + workerIds.map((id) => K.workerUpdateStatus(id)), + ); + workerIds.forEach((id, i) => { + const row = rows[i]; + if (row) map.set(id, row); + }); + return map; +} + +export async function enqueueWorkerUpdate( + db: RedisStore, + workerId: string, + job: NodeUpdateJob, +): Promise { + const id = workerId.trim(); + if (!id) throw new Error("workerId required"); + if (!isValidReleaseVersion(job.version)) throw new Error("invalid_version"); + const meta = await getReleaseMeta(db, job.version); + if (!meta) { + // also accept current pointer version only if meta stored + const cur = await getCurrentRelease(db); + if (!cur || cur.version !== job.version) { + throw new Error("release_not_found"); + } + } + const now = nowIso(); + const status: NodeUpdateStatus = { + workerId: id, + version: job.version, + phase: "pending", + error: null, + startedAt: now, + updatedAt: now, + progress: { done: 0, total: 0 }, + message: "queued", + }; + await db.setJson(K.workerUpdate(id), job, OTA_JOB_TTL_SEC); + await db.setJson(K.workerUpdateStatus(id), status, OTA_JOB_TTL_SEC); + return status; +} + +export async function setWorkerUpdateStatus( + db: RedisStore, + status: NodeUpdateStatus, + ttlSec: number = OTA_JOB_TTL_SEC, +): Promise { + const next = { ...status, updatedAt: nowIso() }; + await db.setJson( + K.workerUpdateStatus(status.workerId), + next, + Math.max(60, ttlSec), + ); +} + +export async function clearWorkerUpdateJob( + db: RedisStore, + workerId: string, +): Promise { + await db.del(K.workerUpdate(workerId)); +} + +/** Diff local hashes against release; returns files that need download. */ +export function diffReleaseFiles( + release: ReleaseMeta, + localHashes: Map, +): ReleaseFileEntry[] { + const needed: ReleaseFileEntry[] = []; + for (const f of release.files) { + if (localHashes.get(f.path) !== f.sha256) { + needed.push(f); + } + } + return needed; +} + +export function releaseSummary(meta: ReleaseMeta | null): { + version: string | null; + fileCount: number; + totalBytes: number; + requiresInstall: boolean; + createdAt: string | null; + createdBy: string | null; +} { + if (!meta) { + return { + version: null, + fileCount: 0, + totalBytes: 0, + requiresInstall: false, + createdAt: null, + createdBy: null, + }; + } + return { + version: meta.version, + fileCount: meta.fileCount, + totalBytes: meta.totalBytes, + requiresInstall: meta.requiresInstall, + createdAt: meta.createdAt, + createdBy: meta.createdBy ?? null, + }; +} diff --git a/packages/db/src/p2p-repos.ts b/packages/db/src/p2p-repos.ts new file mode 100644 index 0000000..76a82e8 --- /dev/null +++ b/packages/db/src/p2p-repos.ts @@ -0,0 +1,611 @@ +import { randomBytes } from "node:crypto"; +import type { RedisStore } from "./client.js"; +import { dayKey, newId, nowIso } from "./client.js"; +import { K } from "./keys.js"; +import { getBotCredentials, getContextToken, getUser } from "./repos.js"; + +// ── Types ────────────────────────────────────────────── + +export interface PeerEndpoint { + botId: string; + peerId: string; +} + +export interface PeerIdentity extends PeerEndpoint { + userId: string; + username: string; +} + +export interface UserWechatBind { + userId: string; + username: string; + botId: string; + peerId: string; + boundAt: string; +} + +export interface BindCodeRecord { + code: string; + userId: string; + username: string; + createdAt: string; +} + +export interface ConnectRequest { + id: string; + from: PeerIdentity; + to: PeerIdentity; + createdAt: string; +} + +export interface P2PSession { + id: string; + a: PeerIdentity; + b: PeerIdentity; + createdAt: string; + lastActivityAt: string; +} + +// ── Helpers ──────────────────────────────────────────── + +const BIND_CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // no O/0/I/1 + +export function normalizeUsername(username: string): string { + return username.trim().toLowerCase(); +} + +export function peerKey(ep: PeerEndpoint): string { + return `${ep.botId}|${ep.peerId}`; +} + +export function generateBindCode(length = 6): string { + const bytes = randomBytes(length); + let out = ""; + for (let i = 0; i < length; i++) { + out += BIND_CODE_ALPHABET[bytes[i]! % BIND_CODE_ALPHABET.length]!; + } + return out; +} + +function sameEndpoint(a: PeerEndpoint, b: PeerEndpoint): boolean { + return a.botId === b.botId && a.peerId === b.peerId; +} + +// ── Bind codes ───────────────────────────────────────── + +export async function createBindCode( + db: RedisStore, + userId: string, + username: string, + ttlSec: number, +): Promise { + // Best-effort: try a few times to avoid collision + for (let i = 0; i < 8; i++) { + const code = generateBindCode(6); + const rec: BindCodeRecord = { + code, + userId, + username, + createdAt: nowIso(), + }; + const ok = await db.redis.set( + K.bindCode(code), + JSON.stringify(rec), + "EX", + Math.max(30, ttlSec), + "NX", + ); + if (ok === "OK") return rec; + } + throw new Error("failed to allocate bind code"); +} + +/** + * Atomically consume a bind code (GET + DEL). Returns null if missing/expired. + */ +export async function consumeBindCode( + db: RedisStore, + code: string, +): Promise { + const rawCode = code.trim().toUpperCase(); + if (!rawCode) return null; + const key = K.bindCode(rawCode); + const raw = await db.redis.get(key); + if (!raw) return null; + await db.redis.del(key); + try { + return JSON.parse(raw) as BindCodeRecord; + } catch { + return null; + } +} + +// ── Primary bind ─────────────────────────────────────── + +export async function getBindByUser( + db: RedisStore, + userId: string, +): Promise { + return db.getJson(K.bindUser(userId)); +} + +export async function getBindByPeer( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const userId = await db.redis.get(K.bindPeer(botId, peerId)); + if (!userId) return null; + return getBindByUser(db, userId); +} + +/** + * Set primary WeChat bind for a LINUX DO user. + * Replaces previous bind for that user and clears reverse pointer of old peer. + * Also ends any connect request / p2p session for both old and new endpoints. + */ +export async function setPrimaryBind( + db: RedisStore, + bind: UserWechatBind, +): Promise<{ previous: UserWechatBind | null }> { + const previous = await getBindByUser(db, bind.userId); + + // If this peer was bound to someone else, clear that user's bind + const existingPeerOwner = await db.redis.get( + K.bindPeer(bind.botId, bind.peerId), + ); + if (existingPeerOwner && existingPeerOwner !== bind.userId) { + const other = await getBindByUser(db, existingPeerOwner); + if ( + other && + other.botId === bind.botId && + other.peerId === bind.peerId + ) { + await clearBindPointers(db, other); + await endSessionsAndRequestsForPeer(db, other.botId, other.peerId); + } + } + + if (previous) { + await clearBindPointers(db, previous); + await endSessionsAndRequestsForPeer(db, previous.botId, previous.peerId); + } + + // End state on the new peer endpoint too + await endSessionsAndRequestsForPeer(db, bind.botId, bind.peerId); + + await db.setJson(K.bindUser(bind.userId), bind); + await db.redis.set(K.bindPeer(bind.botId, bind.peerId), bind.userId); + return { previous }; +} + +export async function clearPrimaryBind( + db: RedisStore, + userId: string, +): Promise { + const bind = await getBindByUser(db, userId); + if (!bind) return null; + await endSessionsAndRequestsForPeer(db, bind.botId, bind.peerId); + await clearBindPointers(db, bind); + return bind; +} + +async function clearBindPointers( + db: RedisStore, + bind: UserWechatBind, +): Promise { + await db.del(K.bindUser(bind.userId)); + const mapped = await db.redis.get(K.bindPeer(bind.botId, bind.peerId)); + if (mapped === bind.userId) { + await db.del(K.bindPeer(bind.botId, bind.peerId)); + } +} + +// ── Reachability ─────────────────────────────────────── + +/** Target can receive pushes if bind exists, context_token exists, bot has credentials. */ +export async function isPeerReachable( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const [token, creds] = await Promise.all([ + getContextToken(db, botId, peerId), + getBotCredentials(db, botId), + ]); + return Boolean(token && creds?.botToken); +} + +// ── Connect requests ─────────────────────────────────── + +export async function getConnectRequest( + db: RedisStore, + requestId: string, +): Promise { + return db.getJson(K.connectReq(requestId)); +} + +export async function getOutboundRequestId( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + return db.redis.get(K.connectFrom(botId, peerId)); +} + +export async function getInboundRequestId( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + return db.redis.get(K.connectTo(botId, peerId)); +} + +export async function getOutboundRequest( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const id = await getOutboundRequestId(db, botId, peerId); + if (!id) return null; + const req = await getConnectRequest(db, id); + if (!req) { + await db.del(K.connectFrom(botId, peerId)); + return null; + } + return req; +} + +export async function getInboundRequest( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const id = await getInboundRequestId(db, botId, peerId); + if (!id) return null; + const req = await getConnectRequest(db, id); + if (!req) { + await db.del(K.connectTo(botId, peerId)); + return null; + } + return req; +} + +/** + * Create a connect request. Fails (returns null + reason) if either side busy + * or keys cannot be claimed with NX. + */ +export async function createConnectRequest( + db: RedisStore, + from: PeerIdentity, + to: PeerIdentity, + ttlSec: number, +): Promise< + | { ok: true; request: ConnectRequest } + | { ok: false; reason: "from_busy" | "to_busy" | "race" } +> { + // Pre-check + const [fromOut, fromIn, toOut, toIn, fromSess, toSess] = await Promise.all([ + getOutboundRequestId(db, from.botId, from.peerId), + getInboundRequestId(db, from.botId, from.peerId), + getOutboundRequestId(db, to.botId, to.peerId), + getInboundRequestId(db, to.botId, to.peerId), + getP2PSessionId(db, from.botId, from.peerId), + getP2PSessionId(db, to.botId, to.peerId), + ]); + if (fromOut || fromIn || fromSess) return { ok: false, reason: "from_busy" }; + if (toOut || toIn || toSess) return { ok: false, reason: "to_busy" }; + + const request: ConnectRequest = { + id: newId("creq"), + from, + to, + createdAt: nowIso(), + }; + const ttl = Math.max(30, ttlSec); + + const fromOk = await db.redis.set( + K.connectFrom(from.botId, from.peerId), + request.id, + "EX", + ttl, + "NX", + ); + if (fromOk !== "OK") return { ok: false, reason: "race" }; + + const toOk = await db.redis.set( + K.connectTo(to.botId, to.peerId), + request.id, + "EX", + ttl, + "NX", + ); + if (toOk !== "OK") { + await db.del(K.connectFrom(from.botId, from.peerId)); + return { ok: false, reason: "race" }; + } + + await db.setJson(K.connectReq(request.id), request, ttl); + return { ok: true, request }; +} + +export async function deleteConnectRequest( + db: RedisStore, + request: ConnectRequest, +): Promise { + await db.del( + K.connectReq(request.id), + K.connectFrom(request.from.botId, request.from.peerId), + K.connectTo(request.to.botId, request.to.peerId), + ); +} + +// ── Sessions ─────────────────────────────────────────── + +export async function getP2PSessionId( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + return db.redis.get(K.p2pPeer(botId, peerId)); +} + +export async function getP2PSession( + db: RedisStore, + sessionId: string, +): Promise { + return db.getJson(K.p2pSession(sessionId)); +} + +export async function getP2PSessionForPeer( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const id = await getP2PSessionId(db, botId, peerId); + if (!id) return null; + const sess = await getP2PSession(db, id); + if (!sess) { + await db.del(K.p2pPeer(botId, peerId)); + return null; + } + return sess; +} + +export async function createP2PSession( + db: RedisStore, + a: PeerIdentity, + b: PeerIdentity, + idleTtlSec: number, +): Promise { + const now = nowIso(); + const session: P2PSession = { + id: newId("p2p"), + a, + b, + createdAt: now, + lastActivityAt: now, + }; + const ttl = Math.max(60, idleTtlSec); + await db.setJson(K.p2pSession(session.id), session, ttl); + await db.redis.set(K.p2pPeer(a.botId, a.peerId), session.id, "EX", ttl); + await db.redis.set(K.p2pPeer(b.botId, b.peerId), session.id, "EX", ttl); + return session; +} + +export async function touchP2PSession( + db: RedisStore, + session: P2PSession, + idleTtlSec: number, +): Promise { + const ttl = Math.max(60, idleTtlSec); + const next: P2PSession = { + ...session, + lastActivityAt: nowIso(), + }; + // Runs on every relayed message — one round trip, not three + await db.redis + .pipeline() + .set(K.p2pSession(session.id), JSON.stringify(next), "EX", ttl) + .set(K.p2pPeer(session.a.botId, session.a.peerId), session.id, "EX", ttl) + .set(K.p2pPeer(session.b.botId, session.b.peerId), session.id, "EX", ttl) + .exec(); + return next; +} + +export async function deleteP2PSession( + db: RedisStore, + session: P2PSession, +): Promise { + await db.del( + K.p2pSession(session.id), + K.p2pPeer(session.a.botId, session.a.peerId), + K.p2pPeer(session.b.botId, session.b.peerId), + ); +} + +export function otherParty( + session: P2PSession, + botId: string, + peerId: string, +): PeerIdentity | null { + if (sameEndpoint(session.a, { botId, peerId })) return session.b; + if (sameEndpoint(session.b, { botId, peerId })) return session.a; + return null; +} + +export function selfParty( + session: P2PSession, + botId: string, + peerId: string, +): PeerIdentity | null { + if (sameEndpoint(session.a, { botId, peerId })) return session.a; + if (sameEndpoint(session.b, { botId, peerId })) return session.b; + return null; +} + +// ── Cleanup helpers ──────────────────────────────────── + +async function endSessionsAndRequestsForPeer( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const sess = await getP2PSessionForPeer(db, botId, peerId); + if (sess) await deleteP2PSession(db, sess); + + const out = await getOutboundRequest(db, botId, peerId); + if (out) await deleteConnectRequest(db, out); + + const inn = await getInboundRequest(db, botId, peerId); + if (inn) await deleteConnectRequest(db, inn); +} + +// ── Daily request rate ───────────────────────────────── + +export async function incrP2PRequestDay( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const day = dayKey(); + const key = K.p2pRequestDay(botId, peerId, day); + const n = await db.redis.incr(key); + if (n === 1) await db.redis.expire(key, 48 * 3600); + return n; +} + +export async function getP2PRequestDayCount( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const day = dayKey(); + const raw = await db.redis.get(K.p2pRequestDay(botId, peerId, day)); + return Number(raw || 0); +} + +// ── Block list (LINUX DO userId → SET of blocked userIds) ─ + +/** True if either side has blocked the other. */ +export async function isBlockedEitherWay( + db: RedisStore, + userIdA: string, + userIdB: string, +): Promise { + if (!userIdA || !userIdB || userIdA === userIdB) return false; + const [ab, ba] = await Promise.all([ + db.redis.sismember(K.blockSet(userIdA), userIdB), + db.redis.sismember(K.blockSet(userIdB), userIdA), + ]); + return ab === 1 || ba === 1; +} + +export async function blockUser( + db: RedisStore, + blockerUserId: string, + blockedUserId: string, +): Promise<{ ok: true } | { ok: false; reason: "self" | "already" }> { + if (!blockerUserId || !blockedUserId) { + return { ok: false, reason: "self" }; + } + if (blockerUserId === blockedUserId) { + return { ok: false, reason: "self" }; + } + const added = await db.redis.sadd(K.blockSet(blockerUserId), blockedUserId); + // End any in-flight connect / session between their WeChat endpoints + await endRelationsBetweenUsers(db, blockerUserId, blockedUserId); + if (added === 0) return { ok: false, reason: "already" }; + return { ok: true }; +} + +export async function unblockUser( + db: RedisStore, + blockerUserId: string, + blockedUserId: string, +): Promise { + const n = await db.redis.srem(K.blockSet(blockerUserId), blockedUserId); + return n > 0; +} + +export async function listBlockedUserIds( + db: RedisStore, + userId: string, +): Promise { + return (await db.redis.smembers(K.blockSet(userId))) as string[]; +} + +/** + * End P2P sessions / requests between two LINUX DO users (if both have binds). + */ +async function endRelationsBetweenUsers( + db: RedisStore, + userIdA: string, + userIdB: string, +): Promise { + const [bindA, bindB] = await Promise.all([ + getBindByUser(db, userIdA), + getBindByUser(db, userIdB), + ]); + if (bindA) await endSessionsAndRequestsForPeer(db, bindA.botId, bindA.peerId); + if (bindB) await endSessionsAndRequestsForPeer(db, bindB.botId, bindB.peerId); +} + +// ── Accept helper ────────────────────────────────────── + +/** + * Accept inbound request: delete request, create session. + * Returns null if request gone / not for this peer. + */ +export async function acceptConnectRequest( + db: RedisStore, + botId: string, + peerId: string, + idleTtlSec: number, +): Promise< + | { ok: true; session: P2PSession; request: ConnectRequest } + | { ok: false; reason: "no_request" | "not_target" } +> { + const request = await getInboundRequest(db, botId, peerId); + if (!request) return { ok: false, reason: "no_request" }; + if (request.to.botId !== botId || request.to.peerId !== peerId) { + return { ok: false, reason: "not_target" }; + } + + // Clear request first so neither side can double-accept + await deleteConnectRequest(db, request); + + // Ensure no leftover session on either side + const [sa, sb] = await Promise.all([ + getP2PSessionForPeer(db, request.from.botId, request.from.peerId), + getP2PSessionForPeer(db, request.to.botId, request.to.peerId), + ]); + if (sa) await deleteP2PSession(db, sa); + if (sb && sb.id !== sa?.id) await deleteP2PSession(db, sb); + + const session = await createP2PSession( + db, + request.from, + request.to, + idleTtlSec, + ); + return { ok: true, session, request }; +} + +/** Resolve bind + username for display if user still exists. */ +export async function resolveBindIdentity( + db: RedisStore, + botId: string, + peerId: string, +): Promise { + const bind = await getBindByPeer(db, botId, peerId); + if (!bind) return null; + const user = await getUser(db, bind.userId); + const username = user?.username || bind.username; + return { + botId: bind.botId, + peerId: bind.peerId, + userId: bind.userId, + username, + }; +} diff --git a/packages/db/src/password.test.ts b/packages/db/src/password.test.ts new file mode 100644 index 0000000..ce93b47 --- /dev/null +++ b/packages/db/src/password.test.ts @@ -0,0 +1,37 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + assertPasswordPolicy, + hashPassword, + verifyPassword, +} from "./password.js"; + +describe("password", () => { + it("hashes and verifies", async () => { + const h = await hashPassword("correct-horse-battery"); + assert.ok(h.startsWith("scrypt$")); + assert.equal(await verifyPassword("correct-horse-battery", h), true); + assert.equal(await verifyPassword("wrong-password", h), false); + }); + + it("rejects weak passwords", () => { + assert.throws(() => assertPasswordPolicy("short", 8), /weak_password/); + assert.throws(() => assertPasswordPolicy(" ", 8), /weak_password/); + }); + + it("rejects tampered encoding", async () => { + const h = await hashPassword("secret-pass-99"); + const bad = h.slice(0, -4) + "XXXX"; + assert.equal(await verifyPassword("secret-pass-99", bad), false); + assert.equal(await verifyPassword("secret-pass-99", null), false); + assert.equal(await verifyPassword("secret-pass-99", "not-a-hash"), false); + }); + + it("different salts for same password", async () => { + const a = await hashPassword("same-password-ok"); + const b = await hashPassword("same-password-ok"); + assert.notEqual(a, b); + assert.equal(await verifyPassword("same-password-ok", a), true); + assert.equal(await verifyPassword("same-password-ok", b), true); + }); +}); diff --git a/packages/db/src/password.ts b/packages/db/src/password.ts new file mode 100644 index 0000000..fbd9516 --- /dev/null +++ b/packages/db/src/password.ts @@ -0,0 +1,100 @@ +import { randomBytes, scrypt, timingSafeEqual } from "node:crypto"; +import { promisify } from "node:util"; + +const scryptAsync = promisify(scrypt) as ( + password: string, + salt: Buffer, + keylen: number, + options: { N: number; r: number; p: number }, +) => Promise; + +const DEFAULT_N = 16384; +const DEFAULT_R = 8; +const DEFAULT_P = 1; +const KEYLEN = 32; +const SALT_LEN = 16; +const MAX_PASSWORD_LEN = 128; + +/** Encoded: scrypt$N$r$p$saltB64$urlsafe$hashB64$urlsafe */ +export function assertPasswordPolicy( + plain: string, + minLength = 8, +): void { + if (typeof plain !== "string" || !plain) { + throw new Error("weak_password"); + } + if (plain.length < minLength) { + throw new Error("weak_password"); + } + if (plain.length > MAX_PASSWORD_LEN) { + throw new Error("weak_password"); + } + if (!plain.trim()) { + throw new Error("weak_password"); + } +} + +function b64url(buf: Buffer): string { + return buf + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); +} + +function fromB64url(s: string): Buffer { + const pad = s.length % 4 === 0 ? "" : "=".repeat(4 - (s.length % 4)); + const b64 = s.replace(/-/g, "+").replace(/_/g, "/") + pad; + return Buffer.from(b64, "base64"); +} + +export async function hashPassword(plain: string): Promise { + assertPasswordPolicy(plain, 1); // length already checked by caller typically + const salt = randomBytes(SALT_LEN); + const hash = await scryptAsync(plain, salt, KEYLEN, { + N: DEFAULT_N, + r: DEFAULT_R, + p: DEFAULT_P, + }); + return `scrypt$${DEFAULT_N}$${DEFAULT_R}$${DEFAULT_P}$${b64url(salt)}$${b64url(hash)}`; +} + +/** + * Constant-time-ish verify. Returns false on any parse/mismatch error. + */ +export async function verifyPassword( + plain: string, + encoded: string | null | undefined, +): Promise { + if (!plain || !encoded || typeof encoded !== "string") return false; + const parts = encoded.split("$"); + if (parts.length !== 6 || parts[0] !== "scrypt") return false; + const N = Number(parts[1]); + const r = Number(parts[2]); + const p = Number(parts[3]); + if (!Number.isFinite(N) || !Number.isFinite(r) || !Number.isFinite(p)) { + return false; + } + if (N < 1024 || N > 1 << 20 || r < 1 || p < 1) return false; + let salt: Buffer; + let expected: Buffer; + try { + salt = fromB64url(parts[4]!); + expected = fromB64url(parts[5]!); + } catch { + return false; + } + if (!salt.length || !expected.length) return false; + try { + const actual = await scryptAsync(plain, salt, expected.length, { N, r, p }); + if (actual.length !== expected.length) return false; + return timingSafeEqual(actual, expected); + } catch { + return false; + } +} + +/** Dummy hash for constant-time path when user missing */ +export async function dummyPasswordHash(): Promise { + return hashPassword("dummy-password-for-timing-" + "x".repeat(16)); +} diff --git a/packages/db/src/paths.ts b/packages/db/src/paths.ts new file mode 100644 index 0000000..f76d72a --- /dev/null +++ b/packages/db/src/paths.ts @@ -0,0 +1,30 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export function resolveRepoRoot(start = process.cwd()): string { + let dir = path.resolve(start); + for (let i = 0; i < 12; i++) { + if (fs.existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + const fromFile = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../..", + ); + if (fs.existsSync(path.join(fromFile, "pnpm-workspace.yaml"))) { + return fromFile; + } + return path.resolve(start); +} + +export function defaultDbPath(): string { + const root = resolveRepoRoot(); + const raw = process.env.WECHAT_AI_DB_PATH?.trim(); + if (raw) { + return path.isAbsolute(raw) ? path.normalize(raw) : path.join(root, raw); + } + return path.join(root, "data", "wechat-ai.db"); +} diff --git a/packages/db/src/persona-square.test.ts b/packages/db/src/persona-square.test.ts new file mode 100644 index 0000000..00cc2da --- /dev/null +++ b/packages/db/src/persona-square.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + generatePersonaSlug, + personaHeatScore, +} from "./repos.js"; + +describe("generatePersonaSlug", () => { + it("produces unique-ish slugs", () => { + const a = generatePersonaSlug("user123", "腹黑学姐"); + const b = generatePersonaSlug("user123", "腹黑学姐"); + assert.match(a, /^p-/); + assert.notEqual(a, b); + }); +}); + +describe("personaHeatScore", () => { + it("weights use / assign / fork", () => { + assert.equal(personaHeatScore({}), 0); + assert.equal( + personaHeatScore({ use_count: 1, assign_count: 1, fork_count: 1 }), + 2 + 5 + 3, + ); + assert.equal(personaHeatScore({ use_count: 3 }), 6); + }); +}); diff --git a/packages/db/src/rebalance.test.ts b/packages/db/src/rebalance.test.ts new file mode 100644 index 0000000..9c445d5 --- /dev/null +++ b/packages/db/src/rebalance.test.ts @@ -0,0 +1,60 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { computeRebalanceShedCount } from "./worker-fleet.js"; + +describe("computeRebalanceShedCount", () => { + it("does not shed with a single worker", () => { + assert.equal( + computeRebalanceShedCount({ localCount: 100, peerCounts: [] }), + 0, + ); + }); + + it("does not shed when already balanced", () => { + assert.equal( + computeRebalanceShedCount({ + localCount: 50, + peerCounts: [50], + slack: 2, + }), + 0, + ); + }); + + it("sheds excess above fair share + slack", () => { + // total 1122, 2 workers → fair 561; local 1122 → shed to 561+2 + const shed = computeRebalanceShedCount({ + localCount: 1122, + peerCounts: [0], + slack: 2, + maxPerTick: 50, + }); + assert.equal(shed, 50); // capped by maxPerTick + }); + + it("respects maxPerTick and slack", () => { + // total 100, 2 workers fair=50; local 60 slack 2 → targetMax 52 → shed 8 + assert.equal( + computeRebalanceShedCount({ + localCount: 60, + peerCounts: [40], + slack: 2, + maxPerTick: 100, + }), + 8, + ); + }); + + it("three-way fair share", () => { + // total 300, 3 workers fair=100; local 200 peers 50,50 → targetMax 102 → shed 98 + assert.equal( + computeRebalanceShedCount({ + localCount: 200, + peerCounts: [50, 50], + slack: 2, + maxPerTick: 1000, + }), + 98, + ); + }); +}); diff --git a/packages/db/src/repos.ts b/packages/db/src/repos.ts new file mode 100644 index 0000000..67faf8b --- /dev/null +++ b/packages/db/src/repos.ts @@ -0,0 +1,3780 @@ +import crypto from "node:crypto"; +import type { RedisStore } from "./client.js"; +import { newId, nowIso, dayKey } from "./client.js"; +import { K } from "./keys.js"; +import { + defaultPersonaIdCache, + invalidateDefaultPersonaCache, + invalidatePromptCache, + invalidateSessionCache, + invalidateSuperAdminCache, + invalidateUserCache, + promptCache, + sessionCache, + SnapshotCache, + SQUARE_SNAPSHOT_MS, + superAdminIdCache, + userCache, +} from "./hot-cache.js"; +import { + forceReleaseBotLease, + markBotPollable, + unmarkBotPollable, +} from "./worker-fleet.js"; +import { + assignmentKeys, + deepStatsMaxPeers, + memoryKeys, + messageKeys, + pairKey, + parsePeerPairs, + shouldComputeDeepStats, + type PeerPair, +} from "./doctor-stats.js"; + +/** Short content hash for CDN cache-busting (sha256 hex prefix). */ +export function hashStickerBlob(data: Buffer): string { + return crypto.createHash("sha256").update(data).digest("hex").slice(0, 16); +} + +export type AuthProvider = "linuxdo" | "local"; + +export interface User { + id: string; + username: string; + name: string; + avatar_url: string | null; + trust_level: number; + is_admin: number; + created_at: string; + updated_at: string; + /** Missing on legacy rows → treat as "linuxdo" */ + auth_provider?: AuthProvider; + /** scrypt encoded; local accounts only; never expose in API DTOs */ + password_hash?: string | null; + invited_by?: string | null; + invite_code_used?: string | null; + /** 0|1; missing = not banned */ + is_banned?: number; + banned_at?: string | null; + banned_reason?: string | null; + banned_by?: string | null; +} + +export function isUserBanned(user: User | null | undefined): boolean { + return Boolean(user?.is_banned); +} + +export function userAuthProvider(user: User): AuthProvider { + return user.auth_provider === "local" ? "local" : "linuxdo"; +} + +/** Public DTO fields (no password_hash). */ +export function userPublicFields(user: User) { + return { + id: user.id, + username: user.username, + name: user.name, + avatarUrl: user.avatar_url, + isAdmin: Boolean(user.is_admin), + trustLevel: user.trust_level, + authProvider: userAuthProvider(user), + isBanned: isUserBanned(user), + bannedAt: user.banned_at ?? null, + bannedReason: user.banned_reason ?? null, + createdAt: user.created_at, + }; +} + +const USERNAME_RE = /^[a-zA-Z][a-zA-Z0-9_]{2,31}$/; +const RESERVED_USERNAMES = new Set([ + "admin", + "system", + "local", + "api", + "root", + "me", + "null", + "undefined", + "support", + "official", +]); + +export function validateLocalUsername(username: string): string { + const u = username.trim(); + if (!USERNAME_RE.test(u)) { + throw new Error("invalid_username"); + } + if (RESERVED_USERNAMES.has(u.toLowerCase())) { + throw new Error("reserved_username"); + } + return u; +} + +export async function claimUsername( + db: RedisStore, + username: string, + userId: string, +): Promise<"ok" | "taken"> { + const lower = username.trim().toLowerCase(); + if (!lower) return "taken"; + const key = K.userByName(lower); + const ok = await db.redis.set(key, userId, "NX"); + if (ok === "OK") return "ok"; + const existing = await db.redis.get(key); + if (existing === userId) return "ok"; + return "taken"; +} + +export async function releaseUsername( + db: RedisStore, + username: string, + userId: string, +): Promise { + const lower = username.trim().toLowerCase(); + if (!lower) return; + const key = K.userByName(lower); + const mapped = await db.redis.get(key); + if (mapped === userId) await db.redis.del(key); +} + +export interface BotAccount { + id: string; + owner_user_id: string; + display_name: string; + account_ref: string | null; + base_url: string | null; + updates_cursor: string; + status: string; + created_at: string; + updated_at: string; + /** Bot-level proactive outreach master switch (0/1). Missing = off. */ + proactive_enabled?: number; + /** Idle hours before proactive contact (overrides global default when set). */ + proactive_idle_hours?: number; + /** Min hours between two proactive sends to the same peer. */ + proactive_min_interval_hours?: number; + /** Max proactive sends per peer per calendar day. */ + proactive_max_per_day?: number; + /** Quiet hours e.g. "0-8" (local Asia/Shanghai); empty/undefined = off. */ + proactive_quiet_hours?: string | null; +} + +/** iLink credentials (sensitive) — separate Redis key from BotAccount metadata */ +export interface BotCredentials { + botId: string; + botToken: string; + baseUrl?: string | null; + accountRef?: string | null; + displayName?: string | null; + savedAt: string; +} + +export type PersonaVisibility = "public" | "private"; +export type PersonaMode = "prompt" | "chatflow"; + +export interface Persona { + id: string; + slug: string; + display_name: string; + description: string; + content_policy: string; + is_default: number; + enabled: number; + published_version_id: string | null; + /** system = official seed; otherwise LINUX DO user id */ + owner_user_id: string; + visibility: PersonaVisibility; + tags: string[]; + /** Library-add popularity (non-owner adds) */ + use_count: number; + /** Cumulative peer assignments (only increments when persona changes) */ + assign_count?: number; + /** How many times this persona was forked */ + fork_count?: number; + /** Lineage when this persona was forked from another */ + forked_from_id?: string | null; + forked_from_slug?: string | null; + forked_from_name?: string | null; + system_prompt?: string; + /** + * Execution mode. Default prompt (classic system prompt). + * chatflow uses graph_json on the published version (engine later). + */ + mode?: PersonaMode; + /** User custom LLM provider id; null/absent = platform LLM. Egress via HF tools. */ + llm_provider_id?: string | null; + /** Allow web_search tool when global WEB_SEARCH_ENABLED */ + web_search_enabled?: number; + created_at?: string; + updated_at?: string; +} + +/** heatScore = use*2 + assign*5 + fork*3 */ +export function personaHeatScore(p: { + use_count?: number; + assign_count?: number; + fork_count?: number; +}): number { + return ( + Number(p.use_count || 0) * 2 + + Number(p.assign_count || 0) * 5 + + Number(p.fork_count || 0) * 3 + ); +} + +export interface TryChatSession { + userId: string; + personaId: string; + botName: string; + createdAt: string; + /** User messages sent in this session */ + msgCount: number; +} + +export interface TryChatMessage { + role: "user" | "assistant"; + content: string; +} + +const PROMPT_MAX_CHARS = 8000; + +export interface PersonaVersion { + id: string; + persona_id: string; + version: number; + system_prompt: string; + /** + * Chatflow graph JSON (version 1). Optional; when persona.mode=chatflow + * and missing, engine falls back to default start→llm→answer. + */ + graph_json?: string | null; + created_at: string; +} + +export interface Peer { + id: string; + bot_account_id: string; + peer_id: string; + display_name: string | null; + approved: number; + approved_at?: string | null; + created_at?: string; + /** Owner opt-in: allow bot to proactively message this peer (0/1). Default 0. */ + proactive_enabled?: number; + /** Last user/assistant chat activity (ISO). */ + last_activity_at?: string | null; + /** Last successful proactive outbound (ISO). */ + last_proactive_at?: string | null; + /** Last proactive attempt including skip (ISO) — used for scan cooldown. */ + last_proactive_attempt_at?: string | null; +} + +export interface MessageRow { + id: string; + bot_account_id: string; + peer_id: string; + persona_id: string | null; + role: string; + content: string; + context_token: string | null; + created_at: string; + /** + * Running count of user messages for this peer, from the INCR that + * insertMessage already issues. Present only on freshly inserted user rows — + * never persisted, so it is absent on anything read back from Redis. + */ + user_count?: number; +} + +export interface MemoryRow { + id: string; + bot_account_id: string; + peer_id: string; + persona_id: string; + kind: string; + content: string; + created_at?: string; + updated_at?: string; +} + +export interface AuditRow { + id: string; + action: string; + actor: string; + meta_json: string; + created_at: string; +} + +export type StickerVisibility = "public" | "private"; +export type StickerReviewStatus = "pending" | "approved" | "rejected"; + +/** + * Sticker square entry. Image bytes in Redis `wa:sticker:{id}:blob`. + * Public stickers require admin approval before appearing in the square. + */ +export interface Sticker { + id: string; + slug: string; + display_name: string; + description: string; + tags: string[]; + mime: string; + size_bytes: number; + /** Logical name only; blob is always in Redis by id */ + file_name: string; + owner_user_id: string; + visibility: StickerVisibility; + review_status: StickerReviewStatus; + reject_reason?: string; + reviewed_at?: string; + reviewed_by?: string; + enabled: number; + use_count: number; + /** sha256 hex prefix of blob; used for /cdn/s/?v= cache-busting */ + content_hash?: string; + created_at: string; + updated_at: string; +} + +/** Compact sticker row for LLM prompt injection */ +export interface StickerPromptEntry { + slug: string; + display_name: string; + description: string; + tags: string[]; +} + +const STICKER_SLUG_RE = /^[a-z0-9][a-z0-9_-]{1,63}$/; + +export interface UsageDayStats { + day: string; + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + requests: number; + by_user: Record; + by_bot: Record; +} + +// ── Users ────────────────────────────────────────────── + +export async function countUsers(db: RedisStore): Promise { + return db.redis.scard(K.usersAll); +} + +export async function upsertUser( + db: RedisStore, + input: { + id: string; + username: string; + name?: string; + avatarUrl?: string | null; + trustLevel?: number; + forceAdmin?: boolean; + authProvider?: AuthProvider; + inviteCodeUsed?: string | null; + invitedBy?: string | null; + }, + adminIds: Set, + opts?: { firstUserIsAdmin?: boolean }, +): Promise { + const existing = await db.getJson(K.user(input.id)); + const totalUsers = await countUsers(db); + const bootstrapAdmin = + Boolean(opts?.firstUserIsAdmin) && + !existing && + totalUsers === 0 && + adminIds.size === 0; + const isAdmin = + input.forceAdmin || + bootstrapAdmin || + adminIds.has(input.id) || + adminIds.has(input.username) || + Boolean(existing?.is_admin); + + // Resolve username with global index ownership + let username = (input.username || "").trim() || input.id; + if (existing?.username) { + const wantLower = username.toLowerCase(); + const oldLower = existing.username.trim().toLowerCase(); + if (wantLower !== oldLower) { + const mapped = await db.redis.get(K.userByName(wantLower)); + if (mapped && mapped !== input.id) { + // Keep old username — do not steal index + username = existing.username; + } + } + } else { + // New user: if name taken by someone else, fall back + const wantLower = username.toLowerCase(); + const mapped = await db.redis.get(K.userByName(wantLower)); + if (mapped && mapped !== input.id) { + username = `ld_${input.id}`.slice(0, 32); + } + } + + const user: User = { + id: input.id, + username, + name: input.name || username, + avatar_url: input.avatarUrl ?? existing?.avatar_url ?? null, + trust_level: input.trustLevel ?? existing?.trust_level ?? 0, + is_admin: isAdmin ? 1 : 0, + created_at: existing?.created_at ?? nowIso(), + updated_at: nowIso(), + // Preserve local auth fields across OAuth re-login + auth_provider: + existing?.auth_provider ?? input.authProvider ?? "linuxdo", + password_hash: existing?.password_hash ?? null, + invited_by: existing?.invited_by ?? input.invitedBy ?? null, + invite_code_used: + existing?.invite_code_used ?? input.inviteCodeUsed ?? null, + is_banned: existing?.is_banned ?? 0, + banned_at: existing?.banned_at ?? null, + banned_reason: existing?.banned_reason ?? null, + banned_by: existing?.banned_by ?? null, + }; + // Runs on every OAuth login. Read both username-index pointers together, + // then apply all writes in one pipeline (was ~6 sequential round trips). + const newNameKey = K.userByName(user.username.trim().toLowerCase()); + const oldNameKey = existing?.username + ? K.userByName(existing.username.trim().toLowerCase()) + : null; + const renamed = Boolean(oldNameKey && oldNameKey !== newNameKey); + + const [oldMapped, curMapped] = await Promise.all([ + renamed + ? (db.redis.get(oldNameKey!) as Promise) + : Promise.resolve(null), + db.redis.get(newNameKey) as Promise, + ]); + + const pipe = db.redis.pipeline(); + pipe.set(K.user(user.id), JSON.stringify(user)); + pipe.sadd(K.usersAll, user.id); + if (user.is_admin) pipe.sadd(K.adminIds, user.id); + if (renamed && oldMapped === user.id) pipe.del(oldNameKey!); + // Only claim the name index if free or already ours + if (!curMapped || curMapped === user.id) pipe.set(newNameKey, user.id); + await pipe.exec(); + + userCache.set(user.id, user); + if (user.is_admin) invalidateSuperAdminCache(); + return user; +} + +/** + * Create a local password user. Caller must have already claimed username + * and consumed invite. id = newId("loc"). + */ +export async function createLocalUser( + db: RedisStore, + input: { + username: string; + passwordHash: string; + name?: string; + invitedBy?: string | null; + inviteCodeUsed?: string | null; + forceAdmin?: boolean; + }, + adminIds: Set, + opts?: { firstUserIsAdmin?: boolean }, +): Promise { + const username = validateLocalUsername(input.username); + const id = newId("loc"); + const totalUsers = await countUsers(db); + const bootstrapAdmin = + Boolean(opts?.firstUserIsAdmin) && + totalUsers === 0 && + adminIds.size === 0; + const isAdmin = + input.forceAdmin || + bootstrapAdmin || + adminIds.has(username) || + adminIds.has(id); + + const claim = await claimUsername(db, username, id); + if (claim === "taken") { + throw new Error("username_taken"); + } + + const now = nowIso(); + const user: User = { + id, + username, + name: (input.name || username).trim() || username, + avatar_url: null, + trust_level: 0, + is_admin: isAdmin ? 1 : 0, + created_at: now, + updated_at: now, + auth_provider: "local", + password_hash: input.passwordHash, + invited_by: input.invitedBy ?? null, + invite_code_used: input.inviteCodeUsed ?? null, + is_banned: 0, + }; + const pipe = db.redis.pipeline(); + pipe.set(K.user(user.id), JSON.stringify(user)); + pipe.sadd(K.usersAll, user.id); + if (user.is_admin) pipe.sadd(K.adminIds, user.id); + await pipe.exec(); + userCache.set(user.id, user); + if (user.is_admin) invalidateSuperAdminCache(); + return user; +} + +export async function getUser( + db: RedisStore, + id: string, +): Promise { + const cached = userCache.get(id) as User | undefined; + if (cached) return cached; + const user = (await db.getJson(K.user(id))) ?? undefined; + if (user) userCache.set(id, user); + return user; +} + +/** Lookup user by LINUX DO username (case-insensitive). */ +export async function getUserByUsername( + db: RedisStore, + username: string, +): Promise { + const name = username.trim().toLowerCase(); + if (!name) return undefined; + const id = await db.redis.get(K.userByName(name)); + if (id) return getUser(db, id); + + // One-shot backfill for users created before the username index existed. + // Guarded: without this, every lookup of a non-existent username re-scanned + // and re-wrote the whole user table (N+1 writes per 404). + if (usernameIndexBackfilled) return undefined; + const users = await listUsers(db); + const pipe = db.redis.pipeline(); + let hit: User | undefined; + for (const u of users) { + const un = (u.username || "").trim().toLowerCase(); + if (!un) continue; + pipe.set(K.userByName(un), u.id); + if (un === name) hit = u; + } + await pipe.exec(); + usernameIndexBackfilled = true; + return hit; +} + +/** Set once the legacy username index has been rebuilt in this process. */ +let usernameIndexBackfilled = false; + +export async function listUsers(db: RedisStore): Promise { + const ids = (await db.redis.smembers(K.usersAll)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson(ids.map((id: string) => K.user(id))); + const users = rows.filter((u): u is User => Boolean(u)); + return users.sort((a, b) => b.created_at.localeCompare(a.created_at)); +} + +/** + * System super-admin: the earliest-created user who is still an admin. + * ("系统的第一个管理员") Used for sensitive ops like fleet node force-offline. + */ +export async function resolveSuperAdminId( + db: RedisStore, +): Promise { + // Scans every user; /auth/me asks for it on every admin page load, so cache + // it. Invalidated on admin grant/revoke and account delete. + const cached = superAdminIdCache.get("id"); + if (cached !== undefined) return cached; + const users = await listUsers(db); + const admins = users.filter((u) => Boolean(u.is_admin)); + if (!admins.length) { + superAdminIdCache.set("id", null); + return null; + } + admins.sort((a, b) => { + const c = (a.created_at || "").localeCompare(b.created_at || ""); + if (c !== 0) return c; + return a.id.localeCompare(b.id); + }); + const id = admins[0]!.id; + superAdminIdCache.set("id", id); + return id; +} + +export async function isSuperAdmin( + db: RedisStore, + userId: string, +): Promise { + if (!userId) return false; + const sid = await resolveSuperAdminId(db); + return Boolean(sid && sid === userId); +} + +/** SCARD of each owner's bot set — one pipeline RTT. */ +export async function countBotsByOwners( + db: RedisStore, + userIds: string[], +): Promise> { + if (!userIds.length) return {}; + const counts = await db.scardMany(userIds.map((id) => K.botsByOwner(id))); + const out: Record = {}; + userIds.forEach((id, i) => { + out[id] = counts[i] ?? 0; + }); + return out; +} + +/** Batch load users by id (MGET). Missing ids omitted. */ +export async function getUsersByIds( + db: RedisStore, + userIds: string[], +): Promise> { + const uniq = [...new Set(userIds.filter(Boolean))]; + const map = new Map(); + if (!uniq.length) return map; + const rows = await db.mgetJson(uniq.map((id) => K.user(id))); + uniq.forEach((id, i) => { + const u = rows[i]; + if (u) map.set(id, u); + }); + return map; +} + +/** Grant or revoke admin. Caller must enforce safety (self / last admin). */ +export async function setUserAdmin( + db: RedisStore, + userId: string, + isAdmin: boolean, +): Promise { + const user = await getUser(db, userId); + if (!user) throw new Error("user not found"); + user.is_admin = isAdmin ? 1 : 0; + user.updated_at = nowIso(); + const pipe = db.redis.pipeline(); + pipe.set(K.user(userId), JSON.stringify(user)); + if (isAdmin) pipe.sadd(K.adminIds, userId); + else pipe.srem(K.adminIds, userId); + await pipe.exec(); + userCache.set(userId, user); + invalidateSuperAdminCache(); + return user; +} + +export async function setUserBanned( + db: RedisStore, + userId: string, + banned: boolean, + opts?: { reason?: string | null; actorId?: string | null }, +): Promise { + const user = await getUser(db, userId); + if (!user) throw new Error("user not found"); + if (banned) { + user.is_banned = 1; + user.banned_at = nowIso(); + user.banned_reason = opts?.reason?.trim() || null; + user.banned_by = opts?.actorId ?? null; + } else { + user.is_banned = 0; + user.banned_at = null; + user.banned_reason = null; + user.banned_by = null; + } + user.updated_at = nowIso(); + await db.setJson(K.user(userId), user); + userCache.set(userId, user); + return user; +} + +/** + * Delete user account (cascade bots, sessions, indexes). Best-effort libs/bind. + */ +export async function deleteUserAccount( + db: RedisStore, + userId: string, +): Promise { + const user = await getUser(db, userId); + if (!user) return false; + + // Cascade bots (sequential — each one touches leases / fleet state) + const botIds = (await db.redis.smembers(K.botsByOwner(userId))) as string[]; + for (const botId of botIds) { + await deleteBotAccount(db, botId); + } + + // Independent cleanups — run together instead of chaining round trips + const [, , codes] = await Promise.all([ + destroyAllSessionsForUser(db, userId), + user.username + ? releaseUsername(db, user.username, userId) + : Promise.resolve(), + db.redis.smembers(K.invitesByUser(userId)) as Promise, + db.redis.srem(K.usersAll, userId), + db.redis.srem(K.adminIds, userId), + ]); + + // Pending invites owned by user — one DEL + if (codes.length) { + await db.redis.del(...codes.map((c) => K.invite(c))); + } + + // Best-effort related keys + await db.del( + K.user(userId), + K.personaLib(userId), + K.stickerLib(userId), + K.personasByOwner(userId), + K.stickersByOwner(userId), + K.botsByOwner(userId), + K.bindUser(userId), + K.blockSet(userId), + K.invitesByUser(userId), + K.inviteGenLog(userId), + K.sessionsByUser(userId), + ); + + invalidateUserCache(userId); + invalidateSuperAdminCache(); + return true; +} + +// ── Sessions (app login) ─────────────────────────────── + +export async function createAppSession( + db: RedisStore, + userId: string, + ttlSec = 7 * 24 * 3600, +): Promise { + const sid = newId("sess"); + await db.setJson(K.session(sid), { userId, createdAt: nowIso() }, ttlSec); + await db.redis.sadd(K.sessionsByUser(userId), sid); + // Align index TTL roughly with session (refresh not needed for ban kicks) + await db.redis.expire(K.sessionsByUser(userId), ttlSec + 3600); + return sid; +} + +export async function getAppSession( + db: RedisStore, + sid: string, +): Promise<{ userId: string } | null> { + const cached = sessionCache.get(sid); + if (cached) return cached; + const sess = await db.getJson<{ userId: string; createdAt?: string }>( + K.session(sid), + ); + if (sess?.userId) sessionCache.set(sid, sess); + return sess; +} + +export async function destroyAppSession( + db: RedisStore, + sid: string, +): Promise { + invalidateSessionCache(sid); + const sess = await db.getJson<{ userId: string }>(K.session(sid)); + await db.del(K.session(sid)); + if (sess?.userId) { + await db.redis.srem(K.sessionsByUser(sess.userId), sid); + } +} + +/** Destroy every session for a user (ban / password revoke). */ +export async function destroyAllSessionsForUser( + db: RedisStore, + userId: string, +): Promise { + const sids = (await db.redis.smembers(K.sessionsByUser(userId))) as string[]; + if (!sids.length) { + await db.redis.del(K.sessionsByUser(userId)); + return 0; + } + // One DEL for every session key + the index (was N+1 round trips) + for (const sid of sids) invalidateSessionCache(sid); + await db.redis.del( + ...sids.map((sid) => K.session(sid)), + K.sessionsByUser(userId), + ); + return sids.length; +} + +export async function saveOauthState( + db: RedisStore, + state: string, + payload: { redirect?: string }, + ttlSec = 600, +): Promise { + await db.setJson(K.oauthState(state), payload, ttlSec); +} + +export async function takeOauthState( + db: RedisStore, + state: string, +): Promise<{ redirect?: string } | null> { + const v = await db.getJson<{ redirect?: string }>(K.oauthState(state)); + if (v) await db.del(K.oauthState(state)); + return v; +} + +// ── Bots ─────────────────────────────────────────────── + +export async function saveBotCredentials( + db: RedisStore, + creds: BotCredentials, +): Promise { + const token = (creds.botToken || "").trim(); + if (!token) throw new Error("botToken required"); + const row: BotCredentials = { + botId: creds.botId, + botToken: token, + baseUrl: creds.baseUrl ?? null, + accountRef: creds.accountRef ?? null, + displayName: creds.displayName ?? null, + savedAt: creds.savedAt || nowIso(), + }; + await db.setJson(K.botCreds(creds.botId), row); +} + +export async function getBotCredentials( + db: RedisStore, + botId: string, +): Promise { + const row = await db.getJson(K.botCreds(botId)); + if (!row?.botToken?.trim()) return null; + return row; +} + +export async function hasBotCredentials( + db: RedisStore, + botId: string, +): Promise { + // EXISTS is 1 RTT and avoids shipping the full token JSON + return (await db.redis.exists(K.botCreds(botId))) === 1; +} + +/** Batch credential presence check (pipeline EXISTS). */ +export async function hasBotCredentialsMany( + db: RedisStore, + botIds: string[], +): Promise> { + if (!botIds.length) return {}; + const flags = await db.existsMany(botIds.map((id) => K.botCreds(id))); + const out: Record = {}; + botIds.forEach((id, i) => { + out[id] = Boolean(flags[i]); + }); + return out; +} + +export async function deleteBotCredentials( + db: RedisStore, + botId: string, +): Promise { + await db.del(K.botCreds(botId)); +} + +export async function upsertBotAccount( + db: RedisStore, + row: { + id?: string; + ownerUserId: string; + displayName: string; + accountRef?: string; + baseUrl?: string; + /** Store iLink token in Redis (`wa:bot:{id}:creds`) */ + botToken?: string; + }, +): Promise { + const id = row.id ?? newId("bot"); + const existing = await db.getJson(K.bot(id)); + const bot: BotAccount = { + id, + owner_user_id: row.ownerUserId || existing?.owner_user_id || "", + display_name: row.displayName, + account_ref: row.accountRef ?? existing?.account_ref ?? null, + base_url: row.baseUrl ?? existing?.base_url ?? null, + updates_cursor: existing?.updates_cursor ?? "", + status: existing?.status ?? "active", + created_at: existing?.created_at ?? nowIso(), + updated_at: nowIso(), + // Preserve proactive settings across re-login / rename upserts + proactive_enabled: existing?.proactive_enabled, + proactive_idle_hours: existing?.proactive_idle_hours, + proactive_min_interval_hours: existing?.proactive_min_interval_hours, + proactive_max_per_day: existing?.proactive_max_per_day, + proactive_quiet_hours: existing?.proactive_quiet_hours, + }; + await db.setJson(K.bot(id), bot); + await db.redis.sadd(K.botsAll, id); + if (bot.owner_user_id) { + await db.redis.sadd(K.botsByOwner(bot.owner_user_id), id); + } + if (row.botToken != null && row.botToken.trim()) { + await saveBotCredentials(db, { + botId: id, + botToken: row.botToken, + baseUrl: row.baseUrl ?? bot.base_url, + accountRef: row.accountRef ?? bot.account_ref, + displayName: row.displayName, + savedAt: nowIso(), + }); + } + // Keep fleet pollable set in sync (active + has token) + if (bot.status === "active") { + const hasTok = + (row.botToken != null && row.botToken.trim().length > 0) || + (await hasBotCredentials(db, id)); + if (hasTok) await markBotPollable(db, id); + } + return bot; +} + +export async function deleteBotAccount( + db: RedisStore, + botId: string, +): Promise { + const bot = await getBotAccount(db, botId); + if (!bot) return false; + await unmarkBotPollable(db, botId); + await forceReleaseBotLease(db, botId); + await db.del(K.bot(botId)); + await deleteBotCredentials(db, botId); + await db.redis.srem(K.botsAll, botId); + if (bot.owner_user_id) { + await db.redis.srem(K.botsByOwner(bot.owner_user_id), botId); + } + // clean peers/messages indexes lightly + const peerKeys = await db.redis.smembers(K.peersByBot(botId)); + for (const pk of peerKeys) { + await db.del(K.peer(botId, pk), K.assignment(botId, pk), K.messages(botId, pk)); + } + await db.del(K.peersByBot(botId), K.proactivePeersByBot(botId)); + return true; +} + +export async function listBotAccounts(db: RedisStore): Promise { + const ids = (await db.redis.smembers(K.botsAll)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id: string) => K.bot(id)), + ); + const out = rows.filter((b): b is BotAccount => Boolean(b)); + return out.sort((a, b) => a.created_at.localeCompare(b.created_at)); +} + +export async function listBotsByOwner( + db: RedisStore, + userId: string, +): Promise { + const ids = (await db.redis.smembers(K.botsByOwner(userId))) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id: string) => K.bot(id)), + ); + return rows.filter((b): b is BotAccount => Boolean(b)); +} + +/** + * Peer total + unapproved counts per bot — 2 RTTs (pipeline SMEMBERS + MGET). + * Avoids N×(listPeers) round-trips on admin bot list. + */ +export async function peerStatsByBots( + db: RedisStore, + botIds: string[], +): Promise> { + const out: Record = + {}; + for (const id of botIds) { + out[id] = { peerCount: 0, unapprovedPeerCount: 0 }; + } + if (!botIds.length) return out; + + const peerIdLists = await db.smembersMany( + botIds.map((id) => K.peersByBot(id)), + ); + + const flatKeys: string[] = []; + const ownerOfKey: string[] = []; + botIds.forEach((botId, i) => { + const pids = peerIdLists[i] ?? []; + out[botId]!.peerCount = pids.length; + for (const pid of pids) { + flatKeys.push(K.peer(botId, pid)); + ownerOfKey.push(botId); + } + }); + + if (!flatKeys.length) return out; + + // mgetJson already chunks (and now runs those chunks concurrently) — the + // extra outer loop here only re-serialized them. + const peers = await db.mgetJson(flatKeys); + peers.forEach((p, j) => { + if (p && !p.approved) { + const botId = ownerOfKey[j]!; + out[botId]!.unapprovedPeerCount += 1; + } + }); + return out; +} + +export async function getBotAccount( + db: RedisStore, + id: string, +): Promise { + return (await db.getJson(K.bot(id))) ?? undefined; +} + +/** Batch bot fetch — avoids N+1 on admin dashboard worker list (hundreds of bots). */ +export async function getBotAccountsByIds( + db: RedisStore, + botIds: string[], +): Promise> { + const uniq = [...new Set(botIds.filter(Boolean))]; + const map = new Map(); + if (!uniq.length) return map; + const rows = await db.mgetJson(uniq.map((id) => K.bot(id))); + uniq.forEach((id, i) => { + const b = rows[i]; + if (b) map.set(id, b); + }); + return map; +} + +export async function setBotCursor( + db: RedisStore, + botId: string, + cursor: string, +): Promise { + const bot = await getBotAccount(db, botId); + if (!bot) return; + bot.updates_cursor = cursor; + bot.updated_at = nowIso(); + await db.setJson(K.bot(botId), bot); +} + +export async function setBotStatus( + db: RedisStore, + botId: string, + status: "active" | "inactive", +): Promise { + const bot = await getBotAccount(db, botId); + if (!bot) throw new Error("bot not found"); + bot.status = status === "inactive" ? "inactive" : "active"; + bot.updated_at = nowIso(); + await db.setJson(K.bot(botId), bot); + if (bot.status === "active" && (await hasBotCredentials(db, botId))) { + await markBotPollable(db, botId); + } else { + await unmarkBotPollable(db, botId); + await forceReleaseBotLease(db, botId); + } + return bot; +} + +export async function updateBotDisplayName( + db: RedisStore, + botId: string, + displayName: string, +): Promise { + const bot = await getBotAccount(db, botId); + if (!bot) throw new Error("bot not found"); + const name = displayName.trim(); + if (!name) throw new Error("displayName required"); + if (name.length > 32) throw new Error("displayName too long (max 32)"); + bot.display_name = name; + bot.updated_at = nowIso(); + await db.setJson(K.bot(botId), bot); + return bot; +} + +export interface BotProactivePatch { + proactiveEnabled?: boolean; + proactiveIdleHours?: number; + proactiveMinIntervalHours?: number; + proactiveMaxPerDay?: number; + /** null or "" clears quiet hours */ + proactiveQuietHours?: string | null; +} + +export async function updateBotProactiveSettings( + db: RedisStore, + botId: string, + patch: BotProactivePatch, +): Promise { + const bot = await getBotAccount(db, botId); + if (!bot) throw new Error("bot not found"); + + if (patch.proactiveEnabled !== undefined) { + bot.proactive_enabled = patch.proactiveEnabled ? 1 : 0; + } + if (patch.proactiveIdleHours !== undefined) { + const h = Number(patch.proactiveIdleHours); + // 0.25h = 15min minimum idle; allow up to 30 days + if (!Number.isFinite(h) || h < 0.25 || h > 24 * 30) { + throw new Error("空闲小时须在 0.25～720 之间（0.25 = 15 分钟）"); + } + bot.proactive_idle_hours = h; + } + if (patch.proactiveMinIntervalHours !== undefined) { + const h = Number(patch.proactiveMinIntervalHours); + // 0 = no min interval between proactive sends + if (!Number.isFinite(h) || h < 0 || h > 24 * 30) { + throw new Error("最小间隔须在 0～720 小时（0 = 不限制）"); + } + bot.proactive_min_interval_hours = h; + } + if (patch.proactiveMaxPerDay !== undefined) { + const n = Math.floor(Number(patch.proactiveMaxPerDay)); + // 0 = unlimited daily sends + if (!Number.isFinite(n) || n < 0 || n > 9999) { + throw new Error("每日上限须在 0～9999（0 = 不限制）"); + } + bot.proactive_max_per_day = n; + } + if (patch.proactiveQuietHours !== undefined) { + const raw = patch.proactiveQuietHours; + if (raw == null || String(raw).trim() === "") { + bot.proactive_quiet_hours = null; + } else { + const s = String(raw).trim(); + if (!/^\d{1,2}-\d{1,2}$/.test(s)) { + throw new Error('proactiveQuietHours must look like "0-8"'); + } + bot.proactive_quiet_hours = s; + } + } + bot.updated_at = nowIso(); + await db.setJson(K.bot(botId), bot); + return bot; +} + +// ── Personas / Square ────────────────────────────────── + +function normalizePersona(raw: Persona | null | undefined): Persona | undefined { + if (!raw) return undefined; + return { + ...raw, + owner_user_id: raw.owner_user_id || "system", + visibility: raw.visibility === "private" ? "private" : "public", + tags: Array.isArray(raw.tags) ? raw.tags : [], + use_count: Number(raw.use_count || 0), + assign_count: Number(raw.assign_count || 0), + fork_count: Number(raw.fork_count || 0), + forked_from_id: raw.forked_from_id ?? null, + forked_from_slug: raw.forked_from_slug ?? null, + forked_from_name: raw.forked_from_name ?? null, + is_default: raw.is_default ? 1 : 0, + enabled: raw.enabled === 0 ? 0 : 1, + mode: raw.mode === "chatflow" ? "chatflow" : "prompt", + llm_provider_id: raw.llm_provider_id ?? null, + web_search_enabled: raw.web_search_enabled ? 1 : 0, + }; +} + +export function generatePersonaSlug(ownerUserId: string, title: string): string { + const base = title + .toLowerCase() + .replace(/[^a-z0-9\u4e00-\u9fff]+/gi, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 24); + const shortOwner = ownerUserId.replace(/[^a-zA-Z0-9]/g, "").slice(-6) || "u"; + const rand = Math.random().toString(36).slice(2, 8); + return `p-${shortOwner}-${base || "persona"}-${rand}`.slice(0, 64); +} + +export async function createPersona( + db: RedisStore, + input: { + slug?: string; + displayName: string; + description?: string; + contentPolicy?: string; + systemPrompt: string; + isDefault?: boolean; + ownerUserId?: string; + visibility?: PersonaVisibility; + tags?: string[]; + mode?: PersonaMode; + llmProviderId?: string | null; + webSearchEnabled?: boolean; + /** Optional initial chatflow graph (object or JSON string) */ + graphJson?: string | object | null; + }, +): Promise { + const prompt = input.systemPrompt.trim(); + if (!prompt) throw new Error("systemPrompt required"); + if (prompt.length > PROMPT_MAX_CHARS) { + throw new Error(`systemPrompt too long (max ${PROMPT_MAX_CHARS})`); + } + const owner = input.ownerUserId || "system"; + const visibility: PersonaVisibility = + input.visibility === "private" ? "private" : "public"; + const slug = + input.slug?.trim() || generatePersonaSlug(owner, input.displayName); + if (await getPersonaBySlug(db, slug)) { + throw new Error("slug exists"); + } + + const personaId = newId("persona"); + const versionId = newId("pver"); + if (input.isDefault) { + await clearDefaultPersonaFlags(db); + } + let graph_json: string | null = null; + if (input.graphJson != null && input.graphJson !== "") { + graph_json = + typeof input.graphJson === "string" + ? input.graphJson + : JSON.stringify(input.graphJson); + } + const version: PersonaVersion = { + id: versionId, + persona_id: personaId, + version: 1, + system_prompt: prompt, + graph_json, + created_at: nowIso(), + }; + const persona: Persona = { + id: personaId, + slug, + display_name: input.displayName.trim(), + description: (input.description ?? "").trim(), + content_policy: input.contentPolicy ?? "standard", + is_default: input.isDefault ? 1 : 0, + enabled: 1, + published_version_id: versionId, + owner_user_id: owner, + visibility, + tags: (input.tags ?? []).map((t) => t.trim()).filter(Boolean).slice(0, 12), + use_count: 0, + assign_count: 0, + fork_count: 0, + forked_from_id: null, + forked_from_slug: null, + forked_from_name: null, + mode: input.mode === "chatflow" ? "chatflow" : "prompt", + llm_provider_id: input.llmProviderId?.trim() || null, + web_search_enabled: input.webSearchEnabled ? 1 : 0, + created_at: nowIso(), + updated_at: nowIso(), + }; + // All writes in one round trip (was 6–9 sequential RTTs against remote Redis) + const pipe = db.redis.pipeline(); + pipe.set(K.persona(personaId), JSON.stringify(persona)); + pipe.set(K.personaVersion(versionId), JSON.stringify(version)); + pipe.sadd(K.personasAll, personaId); + pipe.set(K.personaSlug(slug), personaId); + pipe.rpush(K.personaVersions(personaId), versionId); + if (visibility === "public") { + pipe.sadd(K.personasPublic, personaId); + } + if (owner !== "system") { + pipe.sadd(K.personasByOwner(owner), personaId); + pipe.sadd(K.personaLib(owner), personaId); + } + if (persona.is_default) { + pipe.set(K.personaDefault, personaId); + } + await pipe.exec(); + if (persona.is_default) defaultPersonaIdCache.set("id", personaId); + promptCache.set(personaId, prompt); + invalidatePublicPersonasSnapshot(); + return persona; +} + +export async function listPersonas(db: RedisStore): Promise { + const ids = (await db.redis.smembers(K.personasAll)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id: string) => K.persona(id)), + ); + const out: Persona[] = []; + for (const raw of rows) { + const p = normalizePersona(raw ?? undefined); + if (p) out.push(p); + } + return out.sort((a, b) => a.slug.localeCompare(b.slug)); +} + +export async function getPersona( + db: RedisStore, + id: string, +): Promise { + const raw = await db.getJson(K.persona(id)); + const p = normalizePersona(raw ?? undefined); + if (!p) return undefined; + // migrate legacy indexes once + if (!raw?.owner_user_id || !raw?.visibility) { + await db.setJson(K.persona(id), p); + if (p.visibility === "public" && p.enabled) { + await db.redis.sadd(K.personasPublic, id); + invalidatePublicPersonasSnapshot(); + } + } + return p; +} + +/** Batch persona fetch (MGET). */ +export async function getPersonasByIds( + db: RedisStore, + personaIds: string[], +): Promise> { + const uniq = [...new Set(personaIds.filter(Boolean))]; + const map = new Map(); + if (!uniq.length) return map; + const rows = await db.mgetJson( + uniq.map((id) => K.persona(id)), + ); + uniq.forEach((id, i) => { + const p = normalizePersona(rows[i] ?? undefined); + if (p) map.set(id, p); + }); + return map; +} + +export async function getPersonaBySlug( + db: RedisStore, + slug: string, +): Promise { + const id = await db.redis.get(K.personaSlug(slug)); + if (!id) return undefined; + return getPersona(db, id); +} + +/** Prompt for a persona we already loaded (1 RTT, not 2). */ +export async function getPublishedPromptFromPersona( + db: RedisStore, + persona: Persona, +): Promise { + const cached = promptCache.get(persona.id); + if (cached !== undefined) return cached; + if (!persona.published_version_id) { + promptCache.set(persona.id, null); + return null; + } + const v = await db.getJson( + K.personaVersion(persona.published_version_id), + ); + const prompt = v?.system_prompt ?? null; + promptCache.set(persona.id, prompt); + return prompt; +} + +/** + * Batch published prompts for already-loaded personas — 1 MGET RTT. + * Map key = persona id; missing versions omitted. + */ +export async function getPublishedPromptsMany( + db: RedisStore, + personas: Persona[], +): Promise> { + const map = new Map(); + if (!personas.length) return map; + + const needFetch: Persona[] = []; + for (const p of personas) { + const cached = promptCache.get(p.id); + if (cached !== undefined) { + if (cached) map.set(p.id, cached); + continue; + } + if (!p.published_version_id) { + promptCache.set(p.id, null); + continue; + } + needFetch.push(p); + } + + if (needFetch.length) { + const keys = needFetch.map((p) => + K.personaVersion(p.published_version_id!), + ); + const rows = await db.mgetJson(keys); + needFetch.forEach((p, i) => { + const prompt = rows[i]?.system_prompt ?? null; + promptCache.set(p.id, prompt); + if (prompt) map.set(p.id, prompt); + }); + } + return map; +} + +export async function getPublishedPrompt( + db: RedisStore, + personaId: string, +): Promise { + const cached = promptCache.get(personaId); + if (cached !== undefined) return cached; + const p = await getPersona(db, personaId); + if (!p) { + promptCache.set(personaId, null); + return null; + } + return getPublishedPromptFromPersona(db, p); +} + +export async function publishPersonaVersion( + db: RedisStore, + personaId: string, + systemPrompt: string, + graphJson?: string | null, +): Promise { + const prompt = systemPrompt.trim(); + if (!prompt) throw new Error("systemPrompt required"); + if (prompt.length > PROMPT_MAX_CHARS) { + throw new Error(`systemPrompt too long (max ${PROMPT_MAX_CHARS})`); + } + const p = await getPersona(db, personaId); + if (!p) throw new Error("persona not found"); + const len = await db.redis.llen(K.personaVersions(personaId)); + let graph_json: string | null | undefined = graphJson; + if (graphJson === undefined) { + // Preserve previous published graph when only prompt is updated + if (p.published_version_id) { + const prev = await db.getJson( + K.personaVersion(p.published_version_id), + ); + graph_json = prev?.graph_json ?? null; + } else { + graph_json = null; + } + } else if (graphJson === null || graphJson === "") { + graph_json = null; + } else { + graph_json = graphJson; + } + const version: PersonaVersion = { + id: newId("pver"), + persona_id: personaId, + version: len + 1, + system_prompt: prompt, + graph_json: graph_json ?? null, + created_at: nowIso(), + }; + p.published_version_id = version.id; + p.updated_at = nowIso(); + await db.setJson(K.personaVersion(version.id), version); + await db.setJson(K.persona(personaId), p); + await db.redis.rpush(K.personaVersions(personaId), version.id); + invalidatePromptCache(personaId); + promptCache.set(personaId, prompt); + return version; +} + +/** Load published chatflow graph object (or null). */ +export async function getPublishedGraph( + db: RedisStore, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p?.published_version_id) return null; + const v = await db.getJson( + K.personaVersion(p.published_version_id), + ); + if (!v?.graph_json) return null; + try { + return JSON.parse(v.graph_json) as unknown; + } catch { + return null; + } +} + +export async function getPublishedGraphFromPersona( + db: RedisStore, + persona: Persona, +): Promise { + if (!persona.published_version_id) return null; + const v = await db.getJson( + K.personaVersion(persona.published_version_id), + ); + if (!v?.graph_json) return null; + try { + return JSON.parse(v.graph_json) as unknown; + } catch { + return null; + } +} + +export async function updatePersonaMeta( + db: RedisStore, + personaId: string, + patch: { + displayName?: string; + description?: string; + tags?: string[]; + visibility?: PersonaVisibility; + systemPrompt?: string; + /** JSON string or object; stored on published version */ + graphJson?: string | object | null; + mode?: PersonaMode; + llmProviderId?: string | null; + webSearchEnabled?: boolean; + }, +): Promise { + const p = await getPersona(db, personaId); + if (!p) throw new Error("persona not found"); + if (patch.displayName != null) p.display_name = patch.displayName.trim(); + if (patch.description != null) p.description = patch.description.trim(); + if (patch.tags) { + p.tags = patch.tags.map((t) => t.trim()).filter(Boolean).slice(0, 12); + } + if (patch.visibility) { + p.visibility = patch.visibility; + if (p.visibility === "public" && p.enabled) { + await db.redis.sadd(K.personasPublic, personaId); + } else { + await db.redis.srem(K.personasPublic, personaId); + } + } + if (patch.mode != null) { + p.mode = patch.mode === "chatflow" ? "chatflow" : "prompt"; + } + if (patch.llmProviderId !== undefined) { + p.llm_provider_id = patch.llmProviderId?.trim() || null; + } + if (patch.webSearchEnabled !== undefined) { + p.web_search_enabled = patch.webSearchEnabled ? 1 : 0; + } + p.updated_at = nowIso(); + await db.setJson(K.persona(personaId), p); + invalidatePublicPersonasSnapshot(); + + const hasPrompt = patch.systemPrompt != null; + const hasGraph = patch.graphJson !== undefined; + if (hasPrompt || hasGraph) { + let prompt = patch.systemPrompt; + if (prompt == null) { + prompt = (await getPublishedPrompt(db, personaId)) || ""; + } + let graphStr: string | null | undefined; + if (hasGraph) { + if (patch.graphJson === null || patch.graphJson === "") { + graphStr = null; + } else if (typeof patch.graphJson === "string") { + graphStr = patch.graphJson; + } else { + graphStr = JSON.stringify(patch.graphJson); + } + } + await publishPersonaVersion(db, personaId, prompt, graphStr); + } + return (await getPersona(db, personaId))!; +} + +export async function softDeletePersona( + db: RedisStore, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p) throw new Error("persona not found"); + p.enabled = 0; + p.updated_at = nowIso(); + const pipe = db.redis.pipeline(); + pipe.set(K.persona(personaId), JSON.stringify(p)); + pipe.srem(K.personasPublic, personaId); + // Drop from owner library so /me lists don't keep a dead reference. + // personasByOwner is kept so admin can restore / audit lineage. + if (p.owner_user_id && p.owner_user_id !== "system") { + pipe.srem(K.personaLib(p.owner_user_id), personaId); + } + await pipe.exec(); + invalidatePublicPersonasSnapshot(); +} + +export async function restorePersona( + db: RedisStore, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p) throw new Error("persona not found"); + p.enabled = 1; + p.updated_at = nowIso(); + const pipe = db.redis.pipeline(); + pipe.set(K.persona(personaId), JSON.stringify(p)); + if (p.visibility === "public") { + pipe.sadd(K.personasPublic, personaId); + } + if (p.owner_user_id && p.owner_user_id !== "system") { + pipe.sadd(K.personaLib(p.owner_user_id), personaId); + } + await pipe.exec(); + invalidatePublicPersonasSnapshot(); + return p; +} + +/** Clear is_default flags using pointer key when possible. */ +async function clearDefaultPersonaFlags(db: RedisStore): Promise { + const pointed = await db.redis.get(K.personaDefault); + if (pointed) { + const cur = await getPersona(db, pointed); + if (cur?.is_default) { + cur.is_default = 0; + cur.updated_at = nowIso(); + await db.setJson(K.persona(cur.id), cur); + } + } else { + // Legacy: no pointer — scan once to clear flags + const all = await listPersonas(db); + for (const other of all) { + if (other.is_default) { + other.is_default = 0; + other.updated_at = nowIso(); + await db.setJson(K.persona(other.id), other); + } + } + } + await db.redis.del(K.personaDefault); + invalidateDefaultPersonaCache(); + invalidatePublicPersonasSnapshot(); +} + +/** Mark one persona as default; clears previous default. */ +export async function setDefaultPersona( + db: RedisStore, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p) throw new Error("persona not found"); + if (!p.enabled) throw new Error("cannot set disabled persona as default"); + await clearDefaultPersonaFlags(db); + p.is_default = 1; + p.updated_at = nowIso(); + const pipe = db.redis.pipeline(); + pipe.set(K.persona(personaId), JSON.stringify(p)); + pipe.set(K.personaDefault, personaId); + await pipe.exec(); + defaultPersonaIdCache.set("id", personaId); + invalidatePublicPersonasSnapshot(); + return p; +} + +/** + * Snapshot of every enabled+public persona, normalized once. + * + * The square hits this for every page / sort / keyword change, and the library + * endpoints need it too. Without the snapshot each of those is SMEMBERS + + * MGET(all public personas) against a remote Redis. + */ +const publicPersonasSnapshot = new SnapshotCache( + SQUARE_SNAPSHOT_MS, +); + +/** Drop the square snapshot after any mutation that changes what it contains. */ +export function invalidatePublicPersonasSnapshot(): void { + publicPersonasSnapshot.invalidate(); +} + +/** + * Enabled + public personas (2 RTTs on miss, 0 on hit). + * + * The array and its rows are SHARED across requests — copy before sorting or + * mutating. The array is frozen so an accidental in-place `sort()`/`push()` + * throws instead of silently reordering the square for everyone. + */ +export async function listPublicPersonasCached( + db: RedisStore, +): Promise { + return publicPersonasSnapshot.get(async () => { + const ids = (await db.redis.smembers(K.personasPublic)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id: string) => K.persona(id)), + ); + const out: Persona[] = []; + for (const raw of rows) { + const p = normalizePersona(raw ?? undefined); + if (!p || !p.enabled || p.visibility !== "public") continue; + out.push(p); + } + return Object.freeze(out); + }); +} + +export type PublicPersonaSort = "heat" | "use" | "recent" | "name"; + +/** Public square search (keyword filter + sort). Default: heat. */ +export async function searchPublicPersonas( + db: RedisStore, + opts: { + q?: string; + limit?: number; + offset?: number; + sort?: PublicPersonaSort; + } = {}, +): Promise<{ items: Persona[]; total: number }> { + const limit = Math.min(Math.max(opts.limit ?? 20, 1), 50); + const offset = Math.max(opts.offset ?? 0, 0); + const q = (opts.q ?? "").trim().toLowerCase(); + const sort: PublicPersonaSort = + opts.sort === "recent" || + opts.sort === "name" || + opts.sort === "use" || + opts.sort === "heat" + ? opts.sort + : "heat"; + const all = await listPublicPersonasCached(db); + const items: Persona[] = []; + for (const p of all) { + if (q) { + const hay = [ + p.display_name, + p.description, + p.slug, + ...(p.tags || []), + ] + .join(" ") + .toLowerCase(); + if (!hay.includes(q)) continue; + } + items.push(p); + } + items.sort((a, b) => { + if (sort === "name") { + return a.display_name.localeCompare(b.display_name, "zh"); + } + if (sort === "recent") { + return (b.updated_at || "").localeCompare(a.updated_at || ""); + } + if (sort === "use") { + return ( + (b.use_count || 0) - (a.use_count || 0) || + (b.updated_at || "").localeCompare(a.updated_at || "") + ); + } + // heat (default): composite score then recency + return ( + personaHeatScore(b) - personaHeatScore(a) || + (b.updated_at || "").localeCompare(a.updated_at || "") + ); + }); + return { + total: items.length, + items: items.slice(offset, offset + limit), + }; +} + +export async function listPersonasByOwner( + db: RedisStore, + userId: string, + opts: { includeDisabled?: boolean } = {}, +): Promise { + const ids = (await db.redis.smembers(K.personasByOwner(userId))) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id: string) => K.persona(id)), + ); + const out: Persona[] = []; + for (const raw of rows) { + const p = normalizePersona(raw ?? undefined); + if (!p) continue; + // Soft-deleted personas stay in owner index for restore, but user UI + // should not list them (this was causing "deleted but still there"). + if (!opts.includeDisabled && !p.enabled) continue; + out.push(p); + } + return out.sort((a, b) => + (b.updated_at || "").localeCompare(a.updated_at || ""), + ); +} + +export async function listUserPersonaLibrary( + db: RedisStore, + userId: string, +): Promise { + // Avoid full personasAll scan: cached public snapshot + library + owned. + // The three reads are independent — issue them together (1 RTT with + // autopipelining) instead of chaining SMEMBERS → MGET → SMEMBERS → MGET. + const [publicPersonas, libIds, owned] = await Promise.all([ + listPublicPersonasCached(db), + db.redis.smembers(K.personaLib(userId)) as Promise, + listPersonasByOwner(db, userId), + ]); + const libSet = new Set(libIds); + // Library ids not already covered by the public snapshot (private / forked) + const publicIdSet = new Set(publicPersonas.map((p) => p.id)); + const extraIds = libIds.filter((id) => !publicIdSet.has(id)); + const extraRows = extraIds.length + ? await db.mgetJson(extraIds.map((id) => K.persona(id))) + : []; + const candidates: (Persona | null)[] = [ + ...publicPersonas, + ...extraRows.map((raw) => normalizePersona(raw ?? undefined) ?? null), + ]; + const map = new Map(); + for (const p of candidates) { + if (!p || !p.enabled) continue; + // system public always available + if (p.owner_user_id === "system" && p.visibility === "public") { + map.set(p.id, p); + continue; + } + // private only if owner + if (p.visibility === "private" && p.owner_user_id !== userId) continue; + // in library set or will be filled by owned below + if (libSet.has(p.id) || p.owner_user_id === userId) { + map.set(p.id, p); + } + } + for (const p of owned) { + if (p.enabled) map.set(p.id, p); + } + return [...map.values()].sort((a, b) => + a.display_name.localeCompare(b.display_name, "zh"), + ); +} + +/** One SMEMBERS for library membership checks on list endpoints. */ +export async function getPersonaLibraryIdSet( + db: RedisStore, + userId: string, +): Promise> { + const ids = (await db.redis.smembers(K.personaLib(userId))) as string[]; + return new Set(ids); +} + +/** Local in-library check when persona row + library set already loaded. */ +export function isInPersonaLibraryLocal( + p: Persona, + userId: string, + libIds: Set, +): boolean { + if (p.owner_user_id === "system" && p.visibility === "public") return true; + if (p.owner_user_id === userId) return true; + return libIds.has(p.id); +} + +/** One SMEMBERS for sticker library list endpoints. */ +export async function getStickerLibraryIdSet( + db: RedisStore, + userId: string, +): Promise> { + const ids = (await db.redis.smembers(K.stickerLib(userId))) as string[]; + return new Set(ids); +} + +export async function userCanUsePersona( + db: RedisStore, + userId: string, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p || !p.enabled) return false; + if (p.owner_user_id === "system" && p.visibility === "public") return true; + if (p.owner_user_id === userId) return true; + if (p.visibility === "private") return false; + // public + in library + return Boolean(await db.redis.sismember(K.personaLib(userId), personaId)); +} + +export async function addPersonaToLibrary( + db: RedisStore, + userId: string, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p || !p.enabled) throw new Error("persona not found"); + if (p.visibility === "private" && p.owner_user_id !== userId) { + throw new Error("persona is private"); + } + const added = await db.redis.sadd(K.personaLib(userId), personaId); + if (added && p.owner_user_id !== userId) { + p.use_count = (p.use_count || 0) + 1; + await db.setJson(K.persona(personaId), p); + invalidatePublicPersonasSnapshot(); + } + return p; +} + +export async function removePersonaFromLibrary( + db: RedisStore, + userId: string, + personaId: string, +): Promise { + const removed = await db.redis.srem(K.personaLib(userId), personaId); + if (!removed) return; + const p = await getPersona(db, personaId); + // Mirror addPersonaToLibrary: only non-owner adds bump use_count + if (p && p.owner_user_id !== userId && (p.use_count || 0) > 0) { + p.use_count = Math.max(0, (p.use_count || 0) - 1); + await db.setJson(K.persona(personaId), p); + invalidatePublicPersonasSnapshot(); + } +} + +export async function isInPersonaLibrary( + db: RedisStore, + userId: string, + personaId: string, +): Promise { + const p = await getPersona(db, personaId); + if (!p) return false; + if (p.owner_user_id === "system" && p.visibility === "public") return true; + if (p.owner_user_id === userId) return true; + return Boolean(await db.redis.sismember(K.personaLib(userId), personaId)); +} + +/** + * Fork a persona into a private editable copy owned by `ownerUserId`. + * Source must be enabled and either public or owned by caller (or allowPrivateSource). + */ +export async function forkPersona( + db: RedisStore, + input: { + sourceId: string; + ownerUserId: string; + displayName?: string; + /** Allow forking private personas (owner/admin). Default false. */ + allowPrivateSource?: boolean; + }, +): Promise<{ persona: Persona; systemPrompt: string }> { + const source = await getPersona(db, input.sourceId); + if (!source || !source.enabled) { + throw new Error("persona not found"); + } + if (source.visibility === "private") { + const ok = + source.owner_user_id === input.ownerUserId || input.allowPrivateSource; + if (!ok) throw new Error("persona is private"); + } + const systemPrompt = await getPublishedPrompt(db, source.id); + if (!systemPrompt?.trim()) { + throw new Error("persona has no published prompt"); + } + const sourceGraph = await getPublishedGraph(db, source.id); + const displayName = ( + input.displayName?.trim() || `${source.display_name} 的改编` + ).slice(0, 64); + const tags = [...(source.tags || [])]; + if (!tags.includes("fork")) tags.push("fork"); + + const persona = await createPersona(db, { + displayName, + description: source.description, + contentPolicy: source.content_policy, + systemPrompt, + ownerUserId: input.ownerUserId, + visibility: "private", + tags: tags.slice(0, 12), + // Copy mode / web search; never copy author's llm_provider_id (keys stay private) + mode: source.mode === "chatflow" ? "chatflow" : "prompt", + llmProviderId: null, + webSearchEnabled: Boolean(source.web_search_enabled), + }); + + if (sourceGraph) { + await publishPersonaVersion( + db, + persona.id, + systemPrompt, + JSON.stringify(sourceGraph), + ); + } + + persona.forked_from_id = source.id; + persona.forked_from_slug = source.slug; + persona.forked_from_name = source.display_name; + persona.updated_at = nowIso(); + source.fork_count = Number(source.fork_count || 0) + 1; + source.updated_at = nowIso(); + const pipe = db.redis.pipeline(); + pipe.set(K.persona(persona.id), JSON.stringify(persona)); + pipe.set(K.persona(source.id), JSON.stringify(source)); + await pipe.exec(); + invalidatePublicPersonasSnapshot(); + + return { persona, systemPrompt }; +} + +// ── Peers / assignments ──────────────────────────────── + +export async function ensurePeer( + db: RedisStore, + botAccountId: string, + peerId: string, + displayName?: string, +): Promise { + const existing = await db.getJson(K.peer(botAccountId, peerId)); + if (existing) { + if (displayName && !existing.display_name) { + existing.display_name = displayName; + await db.setJson(K.peer(botAccountId, peerId), existing); + } + return existing; + } + const peer: Peer = { + id: newId("peer"), + bot_account_id: botAccountId, + peer_id: peerId, + display_name: displayName ?? null, + approved: 0, + approved_at: null, + created_at: nowIso(), + }; + await db.setJson(K.peer(botAccountId, peerId), peer); + await db.redis.sadd(K.peersByBot(botAccountId), peerId); + await db.redis.sadd(K.peersAll, `${botAccountId}|${peerId}`); + return peer; +} + +export async function approvePeer( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + const peer = await ensurePeer(db, botAccountId, peerId); + peer.approved = 1; + peer.approved_at = nowIso(); + await db.setJson(K.peer(botAccountId, peerId), peer); + return peer; +} + +/** Mark last chat activity for idle-based proactive outreach. */ +export async function touchPeerActivity( + db: RedisStore, + botAccountId: string, + peerId: string, + at: string = nowIso(), +): Promise { + const peer = await ensurePeer(db, botAccountId, peerId); + await touchPeerActivityFrom(db, peer, at); +} + +/** + * Same, for callers that already hold the Peer row (the chat path does — it + * came back from ensurePeer). Skips the read half of the read-modify-write. + * + * Mutates the passed row, so pass a row you own (ensurePeer/getPeer return + * freshly parsed objects; never pass an element of a cached snapshot). + */ +export async function touchPeerActivityFrom( + db: RedisStore, + peer: Peer, + at: string = nowIso(), +): Promise { + peer.last_activity_at = at; + await db.setJson(K.peer(peer.bot_account_id, peer.peer_id), peer); +} + +/** + * Owner opt-in for proactive contact on a single peer. + * Only approved peers may enable. Maintains proactivePeersByBot index. + */ +export async function setPeerProactiveEnabled( + db: RedisStore, + botAccountId: string, + peerId: string, + enabled: boolean, +): Promise { + const peer = await ensurePeer(db, botAccountId, peerId); + if (enabled && !peer.approved) { + throw new Error("peer must be approved before enabling proactive"); + } + peer.proactive_enabled = enabled ? 1 : 0; + await db.setJson(K.peer(botAccountId, peerId), peer); + if (enabled) { + await db.redis.sadd(K.proactivePeersByBot(botAccountId), peerId); + } else { + await db.redis.srem(K.proactivePeersByBot(botAccountId), peerId); + } + return peer; +} + +export async function listProactivePeerIds( + db: RedisStore, + botAccountId: string, +): Promise { + return (await db.redis.smembers( + K.proactivePeersByBot(botAccountId), + )) as string[]; +} + +export async function getContextToken( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + const v = await db.redis.get(K.contextToken(botAccountId, peerId)); + return v && v.trim() ? v : null; +} + +export async function tryAcquireProactiveLock( + db: RedisStore, + botAccountId: string, + peerId: string, + ttlSec: number, +): Promise { + const ok = await db.redis.set( + K.proactiveLock(botAccountId, peerId), + "1", + "EX", + Math.max(30, ttlSec), + "NX", + ); + return ok === "OK"; +} + +export async function releaseProactiveLock( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + await db.del(K.proactiveLock(botAccountId, peerId)); +} + +export async function getProactiveDayCount( + db: RedisStore, + botAccountId: string, + peerId: string, + day: string = dayKey(), +): Promise { + const n = await db.redis.get( + K.proactiveDayCount(botAccountId, peerId, day), + ); + return n ? Number(n) : 0; +} + +export async function incrProactiveDayCount( + db: RedisStore, + botAccountId: string, + peerId: string, + day: string = dayKey(), +): Promise { + const key = K.proactiveDayCount(botAccountId, peerId, day); + const n = await db.redis.incr(key); + if (n === 1) { + // Expire ~2 days after the day key to cover timezone edge cases + await db.redis.expire(key, 60 * 60 * 48); + } + return n; +} + +/** Record a proactive send (or attempt) on the peer record. */ +export async function markPeerProactive( + db: RedisStore, + botAccountId: string, + peerId: string, + opts: { sent: boolean; at?: string }, +): Promise { + const peer = await ensurePeer(db, botAccountId, peerId); + const at = opts.at ?? nowIso(); + peer.last_proactive_attempt_at = at; + if (opts.sent) { + peer.last_proactive_at = at; + peer.last_activity_at = at; + } + await db.setJson(K.peer(botAccountId, peerId), peer); + return peer; +} + +export async function listPeers( + db: RedisStore, + botAccountId?: string, +): Promise { + if (botAccountId) { + const ids = (await db.redis.smembers( + K.peersByBot(botAccountId), + )) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((pid: string) => K.peer(botAccountId, pid)), + ); + return rows.filter((p): p is Peer => Boolean(p)); + } + const pairs = (await db.redis.smembers(K.peersAll)) as string[]; + if (!pairs.length) return []; + const keys: string[] = []; + for (const pair of pairs) { + const [botId, peerId] = pair.split("|"); + if (!botId || !peerId) continue; + keys.push(K.peer(botId, peerId)); + } + if (!keys.length) return []; + const rows = await db.mgetJson(keys); + return rows.filter((p): p is Peer => Boolean(p)); +} + +/** + * List peers for multiple bots — 2 RTTs (pipeline SMEMBERS + MGET), + * not N× listPeers. + */ +export async function listPeersForBots( + db: RedisStore, + botIds: string[], +): Promise { + const uniq = [...new Set(botIds.filter(Boolean))]; + if (!uniq.length) return []; + const peerIdLists = await db.smembersMany( + uniq.map((id) => K.peersByBot(id)), + ); + const keys: string[] = []; + uniq.forEach((botId, i) => { + for (const pid of peerIdLists[i] ?? []) { + keys.push(K.peer(botId, pid)); + } + }); + if (!keys.length) return []; + const rows = await db.mgetJson(keys); + return rows.filter((p): p is Peer => Boolean(p)); +} + +/** + * Assign persona to peer. When the persona id changes, increments the new + * persona's assign_count (never decrements — ranking stability). + */ +export async function setAssignment( + db: RedisStore, + botAccountId: string, + peerId: string, + personaId: string, +): Promise { + const prev = await db.redis.get(K.assignment(botAccountId, peerId)); + await db.redis.set(K.assignment(botAccountId, peerId), personaId); + if (prev !== personaId) { + const p = await getPersona(db, personaId); + if (p) { + p.assign_count = Number(p.assign_count || 0) + 1; + p.updated_at = nowIso(); + await db.setJson(K.persona(personaId), p); + invalidatePublicPersonasSnapshot(); + } + } +} + +export async function getAssignment( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise<{ persona_id: string } | undefined> { + const personaId = await db.redis.get(K.assignment(botAccountId, peerId)); + return personaId ? { persona_id: personaId } : undefined; +} + +export async function getAssignmentPersonaId( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + return (await db.redis.get(K.assignment(botAccountId, peerId))) ?? null; +} + +/** + * Batch assignment lookup — 1 MGET RTT. + * Map key = `${botAccountId}|${peerId}` → persona_id. + */ +export async function getAssignmentsMany( + db: RedisStore, + pairs: Array<{ botAccountId: string; peerId: string }>, +): Promise> { + const map = new Map(); + if (!pairs.length) return map; + const keys = pairs.map((p) => K.assignment(p.botAccountId, p.peerId)); + const vals = await db.mgetStrings(keys); + pairs.forEach((p, i) => { + const personaId = vals[i]; + if (personaId) map.set(`${p.botAccountId}|${p.peerId}`, personaId); + }); + return map; +} + +export async function getDefaultPersona( + db: RedisStore, +): Promise { + const cachedId = defaultPersonaIdCache.get("id"); + if (cachedId) { + const p = await getPersona(db, cachedId); + if (p?.enabled) return p; + } else if (cachedId === null) { + // fall through to public scan for any enabled + } else { + const pointed = await db.redis.get(K.personaDefault); + if (pointed) { + defaultPersonaIdCache.set("id", pointed); + const p = await getPersona(db, pointed); + if (p?.enabled) return p; + } + } + + // Fallback: public personas only (much smaller than all), then full list + const publicPersonas = await listPublicPersonasCached(db); + if (publicPersonas.length) { + let fallback: Persona | undefined; + for (const p of publicPersonas) { + if (!p?.enabled) continue; + if (p.is_default) { + await db.redis.set(K.personaDefault, p.id); + defaultPersonaIdCache.set("id", p.id); + return p; + } + if (!fallback) fallback = p; + } + if (fallback) { + defaultPersonaIdCache.set("id", fallback.id); + return fallback; + } + } + + const all = await listPersonas(db); + const def = + all.find((p) => p.is_default && p.enabled) ?? + all.find((p) => p.enabled); + if (def) { + await db.redis.set(K.personaDefault, def.id); + defaultPersonaIdCache.set("id", def.id); + } else { + defaultPersonaIdCache.set("id", null); + } + return def; +} + +export async function resolvePersonaForPeer( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + const assigned = await getAssignmentPersonaId(db, botAccountId, peerId); + if (assigned) { + const p = await getPersona(db, assigned); + if (p?.enabled) return p; + } + return getDefaultPersona(db); +} + +// ── Messages / memories ──────────────────────────────── + +export async function insertMessage( + db: RedisStore, + row: { + botAccountId: string; + peerId: string; + personaId?: string | null; + role: "user" | "assistant" | "system"; + content: string; + contextToken?: string | null; + }, +): Promise { + const msg: MessageRow = { + id: newId("msg"), + bot_account_id: row.botAccountId, + peer_id: row.peerId, + persona_id: row.personaId ?? null, + role: row.role, + content: row.content, + context_token: row.contextToken ?? null, + created_at: nowIso(), + }; + const key = K.messages(row.botAccountId, row.peerId); + // One round trip for append + trim (+ the user counter). This sits between + // "model produced text" and "first bubble sent", so every RTT here is + // latency the human sees. + const pipe = db.redis.pipeline(); + pipe.rpush(key, JSON.stringify(msg)); + pipe.ltrim(key, -500, -1); + if (row.role === "user") { + pipe.incr(K.msgCountUser(row.botAccountId, row.peerId)); + } + const res = await pipe.exec(); + if (row.role === "user") { + // INCR reply is the running user-message count — hand it back so callers + // don't need a separate GET to decide on memory extraction. + const n = Number(res?.[2]?.[1] ?? 0); + if (Number.isFinite(n) && n > 0) msg.user_count = n; + } + return msg; +} + +export async function listRecentMessages( + db: RedisStore, + botAccountId: string, + peerId: string, + limit: number, + personaId?: string | null, +): Promise { + const raw = await db.redis.lrange( + K.messages(botAccountId, peerId), + -Math.max(limit * 3, limit), + -1, + ); + let msgs = raw.map((r) => JSON.parse(r) as MessageRow); + if (personaId) { + msgs = msgs.filter( + (m) => !m.persona_id || m.persona_id === personaId, + ); + } + return msgs.slice(-limit); +} + +export async function countUserMessages( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + const n = await db.redis.get(K.msgCountUser(botAccountId, peerId)); + return n ? Number(n) : 0; +} + +/** Clear short-term chat history for a peer (not long-term memories). */ +export async function clearMessages( + db: RedisStore, + botAccountId: string, + peerId: string, +): Promise { + await db.del(K.messages(botAccountId, peerId)); + await db.del(K.msgCountUser(botAccountId, peerId)); +} + +export async function upsertContextToken( + db: RedisStore, + botAccountId: string, + peerId: string, + contextToken: string, +): Promise { + await db.redis.set(K.contextToken(botAccountId, peerId), contextToken); +} + +export async function listMemories( + db: RedisStore, + botAccountId: string, + peerId: string, + personaId: string, +): Promise { + const raw = await db.redis.lrange( + K.memories(botAccountId, peerId, personaId), + 0, + -1, + ); + return raw.map((r) => JSON.parse(r) as MemoryRow); +} + +export async function replaceMemories( + db: RedisStore, + botAccountId: string, + peerId: string, + personaId: string, + facts: string[], + opts?: { maxItems?: number }, +): Promise { + const key = K.memories(botAccountId, peerId, personaId); + const maxItems = Math.max(1, opts?.maxItems ?? 100); + const cleaned: string[] = []; + const seen = new Set(); + for (const f of facts) { + const t = (f ?? "").trim(); + if (!t) continue; + const low = t.toLowerCase(); + if (seen.has(low)) continue; + seen.add(low); + cleaned.push(t); + } + const capped = + cleaned.length > maxItems ? cleaned.slice(-maxItems) : cleaned; + const at = nowIso(); + const rows = capped.map((t) => + JSON.stringify({ + id: newId("mem"), + bot_account_id: botAccountId, + peer_id: peerId, + persona_id: personaId, + kind: "fact", + content: t, + created_at: at, + updated_at: at, + } satisfies MemoryRow), + ); + // DEL + all RPUSHes in one round trip. Was 1 + N sequential RTTs (N up to + // MEMORY_MAX_ITEMS), and left a window where the list was deleted but not + // yet refilled — a concurrent listMemories saw truncated memory. + const pipe = db.redis.pipeline(); + pipe.del(key); + if (rows.length) pipe.rpush(key, ...rows); + await pipe.exec(); +} + +/** Delete a single memory by id within a bot/peer/persona list. */ +export async function deleteMemory( + db: RedisStore, + botAccountId: string, + peerId: string, + personaId: string, + memoryId: string, +): Promise { + const key = K.memories(botAccountId, peerId, personaId); + const raw = await db.redis.lrange(key, 0, -1); + if (!raw.length) return false; + const kept: string[] = []; + let found = false; + for (const r of raw) { + try { + const row = JSON.parse(r) as MemoryRow; + if (row.id === memoryId) { + found = true; + continue; + } + kept.push(r); + } catch { + kept.push(r); + } + } + if (!found) return false; + // Atomic-ish rewrite in one round trip (no delete-then-refill window) + const pipe = db.redis.pipeline(); + pipe.del(key); + if (kept.length) pipe.rpush(key, ...kept); + await pipe.exec(); + return true; +} + +export async function clearMemories( + db: RedisStore, + botAccountId: string, + peerId: string, + personaId?: string, +): Promise { + if (personaId) { + await db.del(K.memories(botAccountId, peerId, personaId)); + return; + } + const personas = await listPersonas(db); + if (!personas.length) return; + // One DEL for every persona-scoped list (was N+1 round trips) + await db.del( + ...personas.map((p) => K.memories(botAccountId, peerId, p.id)), + ); +} + +/** + * Memories for many personas in one wave (chunked pipeline of LRANGE). + * Replaces `for (const p of personas) await listMemories(...)`. + */ +export async function listMemoriesMany( + db: RedisStore, + botAccountId: string, + peerId: string, + personaIds: string[], +): Promise> { + const out = new Map(); + if (!personaIds.length) return out; + const CHUNK = 200; + const chunks: string[][] = []; + for (let off = 0; off < personaIds.length; off += CHUNK) { + chunks.push(personaIds.slice(off, off + CHUNK)); + } + const results = await Promise.all( + chunks.map((ids) => { + const pipe = db.redis.pipeline(); + for (const id of ids) { + pipe.lrange(K.memories(botAccountId, peerId, id), 0, -1); + } + return pipe.exec(); + }), + ); + chunks.forEach((ids, ci) => { + const res = results[ci]; + ids.forEach((id, i) => { + const raw = res?.[i]?.[1]; + if (!Array.isArray(raw) || !raw.length) return; + const rows: MemoryRow[] = []; + for (const r of raw as string[]) { + try { + rows.push(JSON.parse(r) as MemoryRow); + } catch { + /* skip corrupt row */ + } + } + if (rows.length) out.set(id, rows); + }); + }); + return out; +} + +// ── Audit / doctor / usage ───────────────────────────── + +export async function writeAudit( + db: RedisStore, + action: string, + actor = "system", + meta: Record = {}, +): Promise { + const row: AuditRow = { + id: newId("audit"), + action, + actor, + meta_json: JSON.stringify(meta), + created_at: nowIso(), + }; + // LPUSH + LTRIM in one round trip — writeAudit runs on every mutation route + await db.redis + .pipeline() + .lpush(K.audit, JSON.stringify(row)) + .ltrim(K.audit, 0, 999) + .exec(); +} + +export async function listAuditLogs( + db: RedisStore, + limit = 50, +): Promise { + const raw = await db.redis.lrange(K.audit, 0, Math.max(0, limit - 1)); + return raw.map((r) => JSON.parse(r) as AuditRow); +} + +export async function recordTokenUsage( + db: RedisStore, + input: { + userId?: string | null; + botId?: string | null; + promptTokens: number; + completionTokens: number; + username?: string; + botName?: string; + }, +): Promise { + const day = dayKey(); + const total = input.promptTokens + input.completionTokens; + const pipe = db.redis.pipeline(); + const dayKeyH = K.usageDay(day); + pipe.hincrby(dayKeyH, "prompt_tokens", input.promptTokens); + pipe.hincrby(dayKeyH, "completion_tokens", input.completionTokens); + pipe.hincrby(dayKeyH, "total_tokens", total); + pipe.hincrby(dayKeyH, "requests", 1); + pipe.expire(dayKeyH, 90 * 24 * 3600); + if (input.userId) { + const uk = K.usageDayUser(day, input.userId); + pipe.hincrby(uk, "total_tokens", total); + pipe.hincrby(uk, "requests", 1); + if (input.username) pipe.hset(uk, "username", input.username); + pipe.expire(uk, 90 * 24 * 3600); + pipe.sadd(K.usageDayUsers(day), input.userId); + // The index sets had no TTL while the hashes they point at expire at 90d + pipe.expire(K.usageDayUsers(day), 90 * 24 * 3600); + } + if (input.botId) { + const bk = K.usageDayBot(day, input.botId); + pipe.hincrby(bk, "total_tokens", total); + pipe.hincrby(bk, "requests", 1); + if (input.botName) pipe.hset(bk, "display_name", input.botName); + pipe.expire(bk, 90 * 24 * 3600); + pipe.sadd(K.usageDayBots(day), input.botId); + pipe.expire(K.usageDayBots(day), 90 * 24 * 3600); + } + await pipe.exec(); +} + +export async function getUsageDayStats( + db: RedisStore, + day = dayKey(), +): Promise { + const dayHKey = K.usageDay(day); + const [hRaw, userIdsRaw, botIdsRaw] = await Promise.all([ + db.redis.hgetall(dayHKey), + db.redis.smembers(K.usageDayUsers(day)), + db.redis.smembers(K.usageDayBots(day)), + ]); + const h = hRaw as Record; + const userIds = userIdsRaw as string[]; + const botIds = botIdsRaw as string[]; + + const by_user: UsageDayStats["by_user"] = {}; + const by_bot: UsageDayStats["by_bot"] = {}; + + // Chunk pipelines — large by_user/by_bot days stall Upstash if unbounded + const CHUNK = 150; + const allKeys: { kind: "user" | "bot"; id: string; key: string }[] = [ + ...userIds.map((uid) => ({ + kind: "user" as const, + id: uid, + key: K.usageDayUser(day, uid), + })), + ...botIds.map((bid) => ({ + kind: "bot" as const, + id: bid, + key: K.usageDayBot(day, bid), + })), + ]; + // Chunked so a big day doesn't build one enormous pipeline, but the chunks + // run together — serializing them made this ~N/150 round trips. + const slices: (typeof allKeys)[] = []; + for (let off = 0; off < allKeys.length; off += CHUNK) { + slices.push(allKeys.slice(off, off + CHUNK)); + } + const execs = await Promise.all( + slices.map((slice) => { + const pipe = db.redis.pipeline(); + for (const row of slice) pipe.hgetall(row.key); + return pipe.exec(); + }), + ); + slices.forEach((slice, si) => { + const res = execs[si]; + slice.forEach((row, j) => { + const hrow = (res?.[j]?.[1] ?? {}) as Record; + if (row.kind === "user") { + by_user[row.id] = { + total_tokens: Number(hrow.total_tokens || 0), + requests: Number(hrow.requests || 0), + username: hrow.username, + }; + } else { + by_bot[row.id] = { + total_tokens: Number(hrow.total_tokens || 0), + requests: Number(hrow.requests || 0), + display_name: hrow.display_name, + }; + } + }); + }); + + return { + day, + prompt_tokens: Number(h.prompt_tokens || 0), + completion_tokens: Number(h.completion_tokens || 0), + total_tokens: Number(h.total_tokens || 0), + requests: Number(h.requests || 0), + by_user, + by_bot, + }; +} + +export interface DoctorSnapshot { + bots: number; + activeBots: number; + personas: number; + defaultPersona: string | null; + peers: number; + approvedPeers: number; + unapprovedPeers: number; + /** Peers with an explicit persona assignment (exact) */ + assignments: number; + /** + * Messages currently stored. Conversation history is trimmed to the last 500 + * per (bot, peer), so this is the retained count, not lifetime volume. + */ + messages: number; + /** + * Stored memory facts for each peer's persona in effect (its assignment, else + * the platform default). Lists orphaned by an earlier assignment are not + * counted — see memoryKeys() in doctor-stats.ts for why. + */ + memories: number; + users: number; + /** + * False when the dataset is past DOCTOR_DEEP_STATS_MAX_PEERS and the three + * counters above were skipped rather than measured. Display "未统计" for a + * zero with this false — do not read it as "none". + */ + deepStats: boolean; +} + +/** + * These are counts over the whole dataset — computing them pulls every bot, + * peer and persona row out of Redis (megabytes at a few hundred bots). The + * admin dashboard polls, so serve a short-lived shared snapshot instead of + * re-scanning per poll and per concurrent admin. + */ +const doctorSnapshotCache = new SnapshotCache( + Number(process.env.REDIS_L1_DOCTOR_MS ?? "15000"), +); + +export async function doctorSnapshot( + db: RedisStore, +): Promise { + return doctorSnapshotCache.get(() => computeDoctorSnapshot(db)); +} + +async function computeDoctorSnapshot( + db: RedisStore, +): Promise { + // Prefer SCARD + chunked MGET (full listPeers/listBotAccounts was heavy at 400+ bots) + const [botIds, peerPairs, personaIds, users] = await Promise.all([ + db.redis.smembers(K.botsAll) as Promise, + db.redis.smembers(K.peersAll) as Promise, + db.redis.smembers(K.personasAll) as Promise, + db.redis.scard(K.usersAll), + ]); + + const pairs = parsePeerPairs(peerPairs); + const peerKeys = pairs.map((p) => K.peer(p.botId, p.peerId)); + + const [bots, peers, personas] = await Promise.all([ + botIds.length + ? db.mgetJson(botIds.map((id) => K.bot(id))) + : Promise.resolve([] as (BotAccount | null)[]), + peerKeys.length + ? db.mgetJson(peerKeys) + : Promise.resolve([] as (Peer | null)[]), + personaIds.length + ? db.mgetJson(personaIds.map((id) => K.persona(id))) + : Promise.resolve([] as (Persona | null)[]), + ]); + + const botRows = bots.filter((b): b is BotAccount => Boolean(b)); + const peerRows = peers.filter((p): p is Peer => Boolean(p)); + const personaRows = personas.filter((p): p is Persona => Boolean(p)); + const def = personaRows.find((p) => p.is_default); + + const deep = await computeDeepStats(db, pairs, def?.id ?? null); + + return { + bots: botRows.length, + activeBots: botRows.filter((b) => b.status === "active").length, + personas: personaRows.filter((p) => p.enabled).length, + defaultPersona: def?.slug ?? null, + peers: peerRows.length, + approvedPeers: peerRows.filter((p) => p.approved).length, + unapprovedPeers: peerRows.filter((p) => !p.approved).length, + assignments: deep.assignments, + messages: deep.messages, + memories: deep.memories, + users: Number(users) || 0, + deepStats: deep.computed, + }; +} + +/** + * assignments / messages / memories for the dashboard. + * + * Cost is ~4 extra key reads per peer (1 MGET for assignments, then pipelined + * LLENs), all behind the same REDIS_L1_DOCTOR_MS snapshot cache as the rest. + * Past DOCTOR_DEEP_STATS_MAX_PEERS it reports `computed: false` rather than + * quietly billing a large fleet for numbers nobody asked to pay for. + */ +async function computeDeepStats( + db: RedisStore, + pairs: PeerPair[], + defaultPersonaId: string | null, +): Promise<{ + assignments: number; + messages: number; + memories: number; + computed: boolean; +}> { + const empty = { assignments: 0, messages: 0, memories: 0 }; + if (!shouldComputeDeepStats(pairs.length, deepStatsMaxPeers())) { + return { ...empty, computed: false }; + } + if (!pairs.length) return { ...empty, computed: true }; + + const [assignedValues, messages] = await Promise.all([ + db.mgetStrings(assignmentKeys(pairs)), + sumListLengths(db, messageKeys(pairs)), + ]); + + const assignedByPair = new Map(); + pairs.forEach((pair, i) => { + const personaId = assignedValues[i]; + if (personaId) assignedByPair.set(pairKey(pair), personaId); + }); + + const memories = await sumListLengths( + db, + memoryKeys(pairs, assignedByPair, defaultPersonaId), + ); + + return { + assignments: assignedByPair.size, + messages, + memories, + computed: true, + }; +} + +/** Total length across many Redis lists — chunked pipeline, like listMemoriesMany. */ +async function sumListLengths( + db: RedisStore, + keys: string[], +): Promise { + if (!keys.length) return 0; + const CHUNK = 200; + const chunks: string[][] = []; + for (let off = 0; off < keys.length; off += CHUNK) { + chunks.push(keys.slice(off, off + CHUNK)); + } + const results = await Promise.all( + chunks.map((slice) => { + const pipe = db.redis.pipeline(); + for (const key of slice) pipe.llen(key); + return pipe.exec(); + }), + ); + let total = 0; + for (const res of results) { + for (const entry of res ?? []) { + const n = Number(entry?.[1] ?? 0); + if (Number.isFinite(n) && n > 0) total += n; + } + } + return total; +} + +// ── Stickers (square + review; blob in Redis) ────────── + +export function isValidStickerSlug(slug: string): boolean { + return STICKER_SLUG_RE.test(slug); +} + +export function generateStickerSlug(ownerUserId: string, name: string): string { + const shortOwner = + ownerUserId.replace(/[^a-zA-Z0-9]/g, "").slice(-6) || "u"; + const base = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 24); + const rand = Math.random().toString(36).slice(2, 8); + return `s-${shortOwner}-${base || "sticker"}-${rand}`.slice(0, 64); +} + +function normalizeSticker(raw: Sticker | null | undefined): Sticker | undefined { + if (!raw || !raw.id || !raw.slug) return undefined; + const visibility: StickerVisibility = + raw.visibility === "private" ? "private" : "public"; + let review_status: StickerReviewStatus = "approved"; + if ( + raw.review_status === "pending" || + raw.review_status === "rejected" || + raw.review_status === "approved" + ) { + review_status = raw.review_status; + } else if (!("review_status" in (raw as object)) && !raw.owner_user_id) { + // legacy admin-only stickers → treat as system approved public + review_status = "approved"; + } + return { + id: raw.id, + slug: raw.slug, + display_name: raw.display_name || raw.slug, + description: raw.description || "", + tags: Array.isArray(raw.tags) ? raw.tags.filter(Boolean) : [], + mime: raw.mime || "image/png", + size_bytes: Number(raw.size_bytes) || 0, + file_name: raw.file_name || `${raw.id}.bin`, + owner_user_id: raw.owner_user_id || "system", + visibility, + review_status, + reject_reason: raw.reject_reason, + reviewed_at: raw.reviewed_at, + reviewed_by: raw.reviewed_by, + enabled: raw.enabled ? 1 : 0, + use_count: Number(raw.use_count || 0), + content_hash: + typeof raw.content_hash === "string" && raw.content_hash + ? raw.content_hash + : undefined, + created_at: raw.created_at || nowIso(), + updated_at: raw.updated_at || nowIso(), + }; +} + +/** Sync public / pending index sets from current meta. */ +async function syncStickerIndexes( + db: RedisStore, + s: Sticker, +): Promise { + const onSquare = + s.enabled === 1 && + s.visibility === "public" && + s.review_status === "approved"; + const pending = s.enabled === 1 && s.review_status === "pending"; + + // Both index writes in one round trip + const pipe = db.redis.pipeline(); + if (onSquare) pipe.sadd(K.stickersPublic, s.id); + else pipe.srem(K.stickersPublic, s.id); + if (pending) pipe.sadd(K.stickersPending, s.id); + else pipe.srem(K.stickersPending, s.id); + await pipe.exec(); + + invalidatePublicStickersSnapshot(); +} + +export async function putStickerBlob( + db: RedisStore, + id: string, + data: Buffer, +): Promise { + if (!data?.length) throw new Error("empty blob"); + await db.redis.set(K.stickerBlob(id), data); +} + +export async function getStickerBlob( + db: RedisStore, + id: string, +): Promise { + const buf = await db.redis.getBuffer(K.stickerBlob(id)); + if (!buf || !buf.length) return null; + return Buffer.isBuffer(buf) ? buf : Buffer.from(buf); +} + +export async function deleteStickerBlob( + db: RedisStore, + id: string, +): Promise { + await db.redis.del(K.stickerBlob(id)); +} + +export async function createSticker( + db: RedisStore, + input: { + slug?: string; + displayName: string; + description?: string; + tags?: string[]; + mime: string; + sizeBytes?: number; + fileName?: string; + enabled?: boolean; + ownerUserId?: string; + visibility?: StickerVisibility; + /** Admin direct upload auto-approves public stickers by default */ + autoApprove?: boolean; + data: Buffer; + }, +): Promise { + if (!input.data?.length) throw new Error("empty blob"); + const owner = input.ownerUserId || "system"; + const visibility: StickerVisibility = + input.visibility === "private" ? "private" : "public"; + let slug = (input.slug || "").trim().toLowerCase(); + if (!slug) slug = generateStickerSlug(owner, input.displayName); + if (!isValidStickerSlug(slug)) throw new Error("invalid slug"); + if (await getStickerBySlug(db, slug)) throw new Error("slug exists"); + + const autoApprove = + input.autoApprove === true || + (input.autoApprove !== false && owner === "system"); + let review_status: StickerReviewStatus; + if (visibility === "private") { + // private: usable by owner without square listing + review_status = "approved"; + } else if (autoApprove) { + review_status = "approved"; + } else { + review_status = "pending"; + } + + const id = newId("sticker"); + const ext = (input.mime.split("/")[1] || "bin").replace("jpeg", "jpg"); + const fileName = input.fileName?.trim() || `${id}.${ext}`; + const now = nowIso(); + const content_hash = hashStickerBlob(input.data); + const row: Sticker = { + id, + slug, + display_name: (input.displayName || slug).trim(), + description: (input.description ?? "").trim(), + tags: (input.tags ?? []) + .map((t) => t.trim()) + .filter(Boolean) + .slice(0, 16), + mime: input.mime, + size_bytes: input.sizeBytes || input.data.length, + file_name: fileName, + owner_user_id: owner, + visibility, + review_status, + reviewed_at: review_status === "approved" ? now : undefined, + reviewed_by: review_status === "approved" && autoApprove ? owner : undefined, + enabled: input.enabled === false ? 0 : 1, + use_count: 0, + content_hash, + created_at: now, + updated_at: now, + }; + if (!input.data?.length) throw new Error("empty blob"); + // Blob + meta + every index in one round trip (was 5–7 sequential RTTs) + const pipe = db.redis.pipeline(); + pipe.set(K.stickerBlob(id), input.data); + pipe.set(K.sticker(id), JSON.stringify(row)); + pipe.sadd(K.stickersAll, id); + pipe.set(K.stickerSlug(slug), id); + if (owner !== "system") { + pipe.sadd(K.stickersByOwner(owner), id); + pipe.sadd(K.stickerLib(owner), id); + } + await pipe.exec(); + await syncStickerIndexes(db, row); + return row; +} + +export async function getSticker( + db: RedisStore, + id: string, +): Promise { + // One GET — this is the hottest sticker path (CDN image, square detail, + // worker sticker send). Reading the key twice doubled its cost for nothing. + const raw = await db.getJson(K.sticker(id)); + const s = normalizeSticker(raw ?? undefined); + if (s && raw && (!raw.owner_user_id || !raw.review_status)) { + // Lazy migrate legacy records missing owner/review into indexes + await db.setJson(K.sticker(id), s); + await syncStickerIndexes(db, s); + } + return s; +} + +export async function getStickerBySlug( + db: RedisStore, + slug: string, +): Promise { + const id = await db.redis.get(K.stickerSlug(slug.trim().toLowerCase())); + if (!id) return undefined; + return getSticker(db, id); +} + +export async function listStickers( + db: RedisStore, + opts?: { + enabledOnly?: boolean; + q?: string; + reviewStatus?: StickerReviewStatus | "all"; + ownerUserId?: string; + }, +): Promise { + const ids = (await db.redis.smembers(K.stickersAll)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson(ids.map((id) => K.sticker(id))); + let out: Sticker[] = []; + for (const raw of rows) { + const s = normalizeSticker(raw ?? undefined); + if (s) out.push(s); + } + if (opts?.enabledOnly) out = out.filter((s) => s.enabled === 1); + if (opts?.ownerUserId) { + out = out.filter((s) => s.owner_user_id === opts.ownerUserId); + } + if (opts?.reviewStatus && opts.reviewStatus !== "all") { + out = out.filter((s) => s.review_status === opts.reviewStatus); + } + const q = opts?.q?.trim().toLowerCase(); + if (q) { + out = out.filter( + (s) => + s.slug.includes(q) || + s.display_name.toLowerCase().includes(q) || + s.description.toLowerCase().includes(q) || + s.tags.some((t) => t.toLowerCase().includes(q)) || + s.owner_user_id.includes(q), + ); + } + return out.sort((a, b) => + (b.updated_at || "").localeCompare(a.updated_at || ""), + ); +} + +/** Snapshot of every enabled+public+approved sticker (see persona equivalent). */ +const publicStickersSnapshot = new SnapshotCache( + SQUARE_SNAPSHOT_MS, +); + +export function invalidatePublicStickersSnapshot(): void { + publicStickersSnapshot.invalidate(); +} + +/** Square-visible stickers (2 RTTs on miss, 0 on hit). Shared + frozen — see personas. */ +export async function listPublicStickersCached( + db: RedisStore, +): Promise { + return publicStickersSnapshot.get(async () => { + const ids = (await db.redis.smembers(K.stickersPublic)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson(ids.map((id) => K.sticker(id))); + const out: Sticker[] = []; + for (const raw of rows) { + const s = normalizeSticker(raw ?? undefined); + if ( + !s || + !s.enabled || + s.visibility !== "public" || + s.review_status !== "approved" + ) { + continue; + } + out.push(s); + } + return Object.freeze(out); + }); +} + +export type PublicStickerSort = "use" | "recent" | "name"; + +export async function searchPublicStickers( + db: RedisStore, + opts: { + q?: string; + limit?: number; + offset?: number; + sort?: PublicStickerSort; + } = {}, +): Promise<{ items: Sticker[]; total: number }> { + const limit = Math.min(Math.max(opts.limit ?? 20, 1), 50); + const offset = Math.max(opts.offset ?? 0, 0); + const q = (opts.q ?? "").trim().toLowerCase(); + const sort: PublicStickerSort = + opts.sort === "recent" || opts.sort === "name" || opts.sort === "use" + ? opts.sort + : "use"; + const all = await listPublicStickersCached(db); + const items: Sticker[] = []; + for (const s of all) { + if (q) { + const hay = [s.display_name, s.description, s.slug, ...(s.tags || [])] + .join(" ") + .toLowerCase(); + if (!hay.includes(q)) continue; + } + items.push(s); + } + items.sort((a, b) => { + if (sort === "name") { + return a.display_name.localeCompare(b.display_name, "zh"); + } + if (sort === "recent") { + return (b.updated_at || "").localeCompare(a.updated_at || ""); + } + return ( + (b.use_count || 0) - (a.use_count || 0) || + (b.updated_at || "").localeCompare(a.updated_at || "") + ); + }); + return { total: items.length, items: items.slice(offset, offset + limit) }; +} + +export async function listStickersByOwner( + db: RedisStore, + userId: string, +): Promise { + const ids = (await db.redis.smembers(K.stickersByOwner(userId))) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson(ids.map((id) => K.sticker(id))); + const out: Sticker[] = []; + for (const raw of rows) { + const s = normalizeSticker(raw ?? undefined); + if (s) out.push(s); + } + return out.sort((a, b) => + (b.updated_at || "").localeCompare(a.updated_at || ""), + ); +} + +/** Whether a sticker can be used by this user (library / own / system public). */ +export function stickerUsableByUser(s: Sticker, userId: string): boolean { + if (!s.enabled) return false; + if (s.owner_user_id === userId) { + // own private always; own public only if not rejected (pending still ok for owner) + return s.review_status !== "rejected"; + } + if (s.owner_user_id === "system") { + return ( + s.visibility === "public" && s.review_status === "approved" + ); + } + return ( + s.visibility === "public" && s.review_status === "approved" + ); +} + +export async function listUserStickerLibrary( + db: RedisStore, + userId: string, +): Promise { + const map = new Map(); + // Public snapshot, library set and owned set are independent — fetch together + // instead of 3 chained SMEMBERS→MGET pairs (6 sequential RTTs). + const [publicStickers, libIds, owned] = await Promise.all([ + listPublicStickersCached(db), + db.redis.smembers(K.stickerLib(userId)) as Promise, + listStickersByOwner(db, userId), + ]); + // system public approved + for (const s of publicStickers) { + if (s.owner_user_id === "system" && stickerUsableByUser(s, userId)) { + map.set(s.id, s); + } + } + if (libIds.length) { + const publicById = new Map(publicStickers.map((s) => [s.id, s])); + const missing = libIds.filter((id) => !publicById.has(id)); + const extra = missing.length + ? await db.mgetJson(missing.map((id) => K.sticker(id))) + : []; + const libStickers: (Sticker | undefined)[] = [ + ...libIds.map((id) => publicById.get(id)).filter(Boolean), + ...extra.map((raw) => normalizeSticker(raw ?? undefined)), + ]; + for (const s of libStickers) { + if (!s || !s.enabled) continue; + if (s.owner_user_id === userId) { + if (s.review_status !== "rejected") map.set(s.id, s); + continue; + } + if (s.visibility === "public" && s.review_status === "approved") { + map.set(s.id, s); + } + } + } + for (const s of owned) { + if (s.enabled && s.review_status !== "rejected") map.set(s.id, s); + } + return [...map.values()].sort((a, b) => + a.display_name.localeCompare(b.display_name, "zh"), + ); +} + +/** + * Stickers available for LLM injection for a bot owner. + */ +export async function listStickersForOwnerPrompt( + db: RedisStore, + ownerUserId: string, +): Promise { + if (!ownerUserId) return []; + const lib = await listUserStickerLibrary(db, ownerUserId); + return lib + .filter((s) => stickerUsableByUser(s, ownerUserId)) + .map((s) => ({ + slug: s.slug, + display_name: s.display_name, + description: s.description, + tags: s.tags, + })); +} + +/** @deprecated use listStickersForOwnerPrompt — global list no longer used for chat */ +export async function listEnabledStickersForPrompt( + db: RedisStore, +): Promise { + const all = await listStickers(db, { + enabledOnly: true, + reviewStatus: "approved", + }); + return all + .filter((s) => s.visibility === "public" || s.owner_user_id === "system") + .map((s) => ({ + slug: s.slug, + display_name: s.display_name, + description: s.description, + tags: s.tags, + })); +} + +export async function userCanUseSticker( + db: RedisStore, + userId: string, + stickerId: string, +): Promise { + const s = await getSticker(db, stickerId); + if (!s) return false; + return userCanUseStickerRow(db, userId, s); +} + +/** + * Same check for callers that already hold the row. The slug path used to load + * the same sticker twice (once by slug, once by id inside userCanUseSticker). + * + * Note the decision itself is never cached — library membership changes, and a + * stale allow would be a security regression. + */ +export async function userCanUseStickerRow( + db: RedisStore, + userId: string, + s: Sticker, +): Promise { + if (!stickerUsableByUser(s, userId)) return false; + if (s.owner_user_id === userId) return true; + if (s.owner_user_id === "system" && s.review_status === "approved") { + return true; + } + return Boolean(await db.redis.sismember(K.stickerLib(userId), s.id)); +} + +export async function userCanUseStickerSlug( + db: RedisStore, + userId: string, + slug: string, +): Promise { + const s = await getStickerBySlug(db, slug); + if (!s) return null; + if (!(await userCanUseStickerRow(db, userId, s))) return null; + return s; +} + +export async function addStickerToLibrary( + db: RedisStore, + userId: string, + stickerId: string, +): Promise { + const s = await getSticker(db, stickerId); + if (!s || !s.enabled) throw new Error("sticker not found"); + if (s.visibility === "private" && s.owner_user_id !== userId) { + throw new Error("sticker is private"); + } + if ( + s.owner_user_id !== userId && + (s.review_status !== "approved" || s.visibility !== "public") + ) { + throw new Error("sticker not available"); + } + const added = await db.redis.sadd(K.stickerLib(userId), stickerId); + if (added && s.owner_user_id !== userId) { + s.use_count = (s.use_count || 0) + 1; + s.updated_at = nowIso(); + await db.setJson(K.sticker(stickerId), s); + invalidatePublicStickersSnapshot(); + } + return s; +} + +export async function removeStickerFromLibrary( + db: RedisStore, + userId: string, + stickerId: string, +): Promise { + const removed = await db.redis.srem(K.stickerLib(userId), stickerId); + if (!removed) return; + const s = await getSticker(db, stickerId); + // Mirror addStickerToLibrary: only non-owner adds bump use_count + if (s && s.owner_user_id !== userId && (s.use_count || 0) > 0) { + s.use_count = Math.max(0, (s.use_count || 0) - 1); + s.updated_at = nowIso(); + await db.setJson(K.sticker(stickerId), s); + invalidatePublicStickersSnapshot(); + } +} + +// ── Web try-chat (ephemeral) ─────────────────────────── + +export async function createTryChatSession( + db: RedisStore, + input: { + userId: string; + personaId: string; + botName?: string; + ttlSec?: number; + }, +): Promise<{ sessionId: string; session: TryChatSession; ttlSec: number }> { + const ttlSec = Math.max(60, input.ttlSec ?? 3600); + const sessionId = newId("try"); + const session: TryChatSession = { + userId: input.userId, + personaId: input.personaId, + botName: (input.botName?.trim() || "助手").slice(0, 32), + createdAt: nowIso(), + msgCount: 0, + }; + await db.setJson(K.trySession(sessionId), session, ttlSec); + return { sessionId, session, ttlSec }; +} + +export async function getTryChatSession( + db: RedisStore, + sessionId: string, +): Promise { + const s = await db.getJson(K.trySession(sessionId)); + return s?.userId && s.personaId ? s : null; +} + +export async function saveTryChatSession( + db: RedisStore, + sessionId: string, + session: TryChatSession, + ttlSec: number, +): Promise { + await db.setJson(K.trySession(sessionId), session, Math.max(60, ttlSec)); +} + +export async function deleteTryChatSession( + db: RedisStore, + sessionId: string, +): Promise { + await db.del(K.trySession(sessionId)); + await db.del(K.trySessionMsgs(sessionId)); +} + +export async function listTryChatMessages( + db: RedisStore, + sessionId: string, + max = 40, +): Promise { + const n = Math.max(1, Math.min(max, 100)); + const raw = await db.redis.lrange(K.trySessionMsgs(sessionId), -n, -1); + const out: TryChatMessage[] = []; + for (const r of raw) { + try { + const m = JSON.parse(r) as TryChatMessage; + if (m?.role && m.content != null) out.push(m); + } catch { + /* skip */ + } + } + return out; +} + +export async function appendTryChatMessages( + db: RedisStore, + sessionId: string, + messages: TryChatMessage[], + opts: { maxHistory?: number; ttlSec?: number } = {}, +): Promise { + if (!messages.length) return; + const maxHistory = Math.max(4, opts.maxHistory ?? 40); + const ttlSec = Math.max(60, opts.ttlSec ?? 3600); + const pipe = db.redis.pipeline(); + for (const m of messages) { + pipe.rpush(K.trySessionMsgs(sessionId), JSON.stringify(m)); + } + pipe.ltrim(K.trySessionMsgs(sessionId), -maxHistory, -1); + pipe.expire(K.trySessionMsgs(sessionId), ttlSec); + await pipe.exec(); +} + +/** Increment daily try-chat user-message counter. Returns new count. */ +export async function incrTryChatDayCount( + db: RedisStore, + userId: string, + day?: string, +): Promise { + const d = day || dayKey(); + const key = K.tryDayCount(userId, d); + const n = await db.redis.incr(key); + if (n === 1) { + await db.redis.expire(key, 2 * 24 * 3600); + } + return n; +} + +export async function getTryChatDayCount( + db: RedisStore, + userId: string, + day?: string, +): Promise { + const d = day || dayKey(); + const v = await db.redis.get(K.tryDayCount(userId, d)); + return Number(v || 0); +} + +export async function isInStickerLibrary( + db: RedisStore, + userId: string, + stickerId: string, +): Promise { + return Boolean(await db.redis.sismember(K.stickerLib(userId), stickerId)); +} + +export async function approveSticker( + db: RedisStore, + id: string, + reviewerId: string, +): Promise { + const s = await getSticker(db, id); + if (!s) throw new Error("not found"); + s.review_status = "approved"; + s.reject_reason = undefined; + s.reviewed_at = nowIso(); + s.reviewed_by = reviewerId; + s.updated_at = nowIso(); + await db.setJson(K.sticker(id), s); + await syncStickerIndexes(db, s); + return s; +} + +export async function rejectSticker( + db: RedisStore, + id: string, + reviewerId: string, + reason?: string, +): Promise { + const s = await getSticker(db, id); + if (!s) throw new Error("not found"); + s.review_status = "rejected"; + s.reject_reason = (reason || "").trim() || undefined; + s.reviewed_at = nowIso(); + s.reviewed_by = reviewerId; + s.updated_at = nowIso(); + await db.setJson(K.sticker(id), s); + await syncStickerIndexes(db, s); + return s; +} + +export async function softDeleteSticker( + db: RedisStore, + id: string, +): Promise { + const s = await getSticker(db, id); + if (!s) throw new Error("not found"); + s.enabled = 0; + s.updated_at = nowIso(); + await db.setJson(K.sticker(id), s); + await syncStickerIndexes(db, s); + return s; +} + +export async function restoreSticker( + db: RedisStore, + id: string, +): Promise { + const s = await getSticker(db, id); + if (!s) throw new Error("not found"); + s.enabled = 1; + // public restored items go back to pending unless system + if (s.visibility === "public" && s.owner_user_id !== "system") { + if (s.review_status === "rejected") s.review_status = "pending"; + } + s.updated_at = nowIso(); + await db.setJson(K.sticker(id), s); + await syncStickerIndexes(db, s); + return s; +} + +export async function countPendingStickers(db: RedisStore): Promise { + return db.redis.scard(K.stickersPending); +} + +export async function updateStickerMeta( + db: RedisStore, + id: string, + patch: { + displayName?: string; + description?: string; + tags?: string[]; + enabled?: boolean; + mime?: string; + sizeBytes?: number; + fileName?: string; + slug?: string; + visibility?: StickerVisibility; + /** sha256 prefix of image bytes (CDN ?v=) */ + contentHash?: string; + /** When true, public content changes reset review to pending */ + rePending?: boolean; + }, +): Promise { + const cur = await getSticker(db, id); + if (!cur) throw new Error("not found"); + + if (patch.slug !== undefined) { + const nextSlug = patch.slug.trim().toLowerCase(); + if (!isValidStickerSlug(nextSlug)) throw new Error("invalid slug"); + if (nextSlug !== cur.slug) { + const existing = await getStickerBySlug(db, nextSlug); + if (existing && existing.id !== id) throw new Error("slug exists"); + await db.redis.del(K.stickerSlug(cur.slug)); + await db.redis.set(K.stickerSlug(nextSlug), id); + cur.slug = nextSlug; + } + } + if (patch.displayName !== undefined) { + cur.display_name = patch.displayName.trim() || cur.slug; + } + if (patch.description !== undefined) { + cur.description = patch.description.trim(); + } + if (patch.tags !== undefined) { + cur.tags = patch.tags + .map((t) => t.trim()) + .filter(Boolean) + .slice(0, 16); + } + if (patch.enabled !== undefined) { + cur.enabled = patch.enabled ? 1 : 0; + } + if (patch.mime !== undefined) cur.mime = patch.mime; + if (patch.sizeBytes !== undefined) cur.size_bytes = patch.sizeBytes; + if (patch.fileName !== undefined) cur.file_name = patch.fileName; + if (patch.contentHash !== undefined) cur.content_hash = patch.contentHash; + if (patch.visibility !== undefined) { + cur.visibility = patch.visibility === "private" ? "private" : "public"; + } + + const needsReReview = + patch.rePending === true || + (cur.visibility === "public" && + cur.owner_user_id !== "system" && + (patch.rePending !== false && + (patch.displayName !== undefined || + patch.description !== undefined || + patch.tags !== undefined || + patch.visibility === "public" || + patch.mime !== undefined || + patch.sizeBytes !== undefined || + patch.contentHash !== undefined))); + + if (needsReReview && cur.visibility === "public" && cur.owner_user_id !== "system") { + cur.review_status = "pending"; + cur.reject_reason = undefined; + } + if (cur.visibility === "private") { + // private is usable without square review + if (cur.review_status === "pending") cur.review_status = "approved"; + } + + cur.updated_at = nowIso(); + await db.setJson(K.sticker(id), cur); + await syncStickerIndexes(db, cur); + return cur; +} + +export async function deleteSticker( + db: RedisStore, + id: string, +): Promise { + const cur = await getSticker(db, id); + if (!cur) return undefined; + const pipe = db.redis.pipeline(); + pipe.del(K.sticker(id), K.stickerBlob(id), K.stickerSlug(cur.slug)); + pipe.srem(K.stickersAll, id); + pipe.srem(K.stickersPublic, id); + pipe.srem(K.stickersPending, id); + if (cur.owner_user_id && cur.owner_user_id !== "system") { + pipe.srem(K.stickersByOwner(cur.owner_user_id), id); + } + await pipe.exec(); + invalidatePublicStickersSnapshot(); + return cur; +} + +/** + * Replace image bytes; public non-system stickers go back to pending. + * Updates content_hash for CDN cache-busting (`?v=`). + */ +export async function replaceStickerBlob( + db: RedisStore, + id: string, + data: Buffer, + patch?: { mime?: string; fileName?: string }, +): Promise { + const cur = await getSticker(db, id); + if (!cur) throw new Error("not found"); + if (!data?.length) throw new Error("empty blob"); + await putStickerBlob(db, id, data); + const content_hash = hashStickerBlob(data); + return updateStickerMeta(db, id, { + mime: patch?.mime ?? cur.mime, + sizeBytes: data.length, + fileName: patch?.fileName ?? cur.file_name, + contentHash: content_hash, + rePending: true, + }); +} + +/** + * Ensure sticker meta has content_hash (lazy migrate from blob). + * Returns updated sticker or null if blob missing. + */ +export async function ensureStickerContentHash( + db: RedisStore, + sticker: Sticker, +): Promise { + if (sticker.content_hash) return sticker; + const buf = await getStickerBlob(db, sticker.id); + if (!buf) return null; + const content_hash = hashStickerBlob(buf); + const next: Sticker = { + ...sticker, + content_hash, + size_bytes: sticker.size_bytes || buf.length, + updated_at: nowIso(), + }; + await db.setJson(K.sticker(sticker.id), next); + invalidatePublicStickersSnapshot(); + return next; +} diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts new file mode 100644 index 0000000..6bb9f12 --- /dev/null +++ b/packages/db/src/schema.ts @@ -0,0 +1,113 @@ +export const SCHEMA_SQL = ` +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + id TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS bot_accounts ( + id TEXT PRIMARY KEY, + display_name TEXT NOT NULL, + account_ref TEXT, + base_url TEXT, + token_path TEXT NOT NULL, + updates_cursor TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'active', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS personas ( + id TEXT PRIMARY KEY, + slug TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + content_policy TEXT NOT NULL DEFAULT 'standard', + is_default INTEGER NOT NULL DEFAULT 0, + enabled INTEGER NOT NULL DEFAULT 1, + published_version_id TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS persona_versions ( + id TEXT PRIMARY KEY, + persona_id TEXT NOT NULL REFERENCES personas(id) ON DELETE CASCADE, + version INTEGER NOT NULL, + system_prompt TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(persona_id, version) +); + +CREATE TABLE IF NOT EXISTS peers ( + id TEXT PRIMARY KEY, + bot_account_id TEXT NOT NULL REFERENCES bot_accounts(id) ON DELETE CASCADE, + peer_id TEXT NOT NULL, + display_name TEXT, + approved INTEGER NOT NULL DEFAULT 0, + approved_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(bot_account_id, peer_id) +); + +CREATE TABLE IF NOT EXISTS assignments ( + id TEXT PRIMARY KEY, + bot_account_id TEXT NOT NULL, + peer_id TEXT NOT NULL, + persona_id TEXT NOT NULL REFERENCES personas(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(bot_account_id, peer_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id TEXT PRIMARY KEY, + bot_account_id TEXT NOT NULL, + peer_id TEXT NOT NULL, + persona_id TEXT, + role TEXT NOT NULL CHECK(role IN ('user','assistant','system')), + content TEXT NOT NULL, + context_token TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_messages_peer + ON messages(bot_account_id, peer_id, created_at); + +CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + bot_account_id TEXT NOT NULL, + peer_id TEXT NOT NULL, + persona_id TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'fact', + content TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_memories_scope + ON memories(bot_account_id, peer_id, persona_id); + +CREATE TABLE IF NOT EXISTS peer_context_tokens ( + bot_account_id TEXT NOT NULL, + peer_id TEXT NOT NULL, + context_token TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (bot_account_id, peer_id) +); + +CREATE TABLE IF NOT EXISTS audit_logs ( + id TEXT PRIMARY KEY, + action TEXT NOT NULL, + actor TEXT NOT NULL DEFAULT 'system', + meta_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +`; diff --git a/packages/db/src/secret-crypto.test.ts b/packages/db/src/secret-crypto.test.ts new file mode 100644 index 0000000..f369ebd --- /dev/null +++ b/packages/db/src/secret-crypto.test.ts @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + decryptSecret, + encryptSecret, + maskApiKey, +} from "./secret-crypto.js"; + +describe("secret-crypto", () => { + it("round-trips encrypt/decrypt", () => { + const secret = "test-provider-secret-key"; + const plain = "sk-live-abcdefghijklmnopqrstuvwxyz"; + const enc = encryptSecret(plain, secret); + assert.notEqual(enc, plain); + assert.equal(decryptSecret(enc, secret), plain); + }); + + it("memoized key derivation stays correct across secrets", () => { + // deriveKey is cached; a cache keyed wrongly would cross-decrypt. + const a = encryptSecret("payload-a", "secret-alpha-0001"); + const b = encryptSecret("payload-b", "secret-beta-0002"); + assert.equal(decryptSecret(a, "secret-alpha-0001"), "payload-a"); + assert.equal(decryptSecret(b, "secret-beta-0002"), "payload-b"); + assert.throws(() => decryptSecret(a, "secret-beta-0002")); + assert.throws(() => decryptSecret(b, "secret-alpha-0001")); + // Repeat after cache warm-up + assert.equal(decryptSecret(a, "secret-alpha-0001"), "payload-a"); + }); + + it("survives cache overflow (more secrets than the cache holds)", () => { + const enc = encryptSecret("keep-me", "original-secret-xyz"); + for (let i = 0; i < 32; i++) { + encryptSecret("noise", `rotating-secret-${i}`); + } + assert.equal(decryptSecret(enc, "original-secret-xyz"), "keep-me"); + }); + + it("maskApiKey hides middle", () => { + const m = maskApiKey("sk-abcdefghijklmnop"); + assert.ok(m.includes("****")); + assert.ok(!m.includes("abcdefghijklmnop")); + assert.equal(maskApiKey("short"), "****"); + }); +}); diff --git a/packages/db/src/secret-crypto.ts b/packages/db/src/secret-crypto.ts new file mode 100644 index 0000000..553d680 --- /dev/null +++ b/packages/db/src/secret-crypto.ts @@ -0,0 +1,83 @@ +/** + * Encrypt user secrets at rest (e.g. custom LLM API keys). + * AES-256-GCM; key derived from LLM_PROVIDER_SECRET via scrypt. + */ +import { + createCipheriv, + createDecipheriv, + createHash, + randomBytes, + scryptSync, +} from "node:crypto"; + +const PREFIX = "v1"; + +/** Fixed salt — the derived key depends only on `secret`, so it is memoizable. */ +const SALT = createHash("sha256").update("wechat-ai-llm-provider-v1").digest(); + +/** + * scryptSync costs ~30ms of SYNCHRONOUS main-thread time and was re-run on + * every encrypt/decrypt — i.e. on every inbound message that uses a user's + * custom LLM provider, freezing Fastify and every bot long-poll for that long. + * The derivation is deterministic, so memoize it. + * + * Bounded to a handful of entries (in practice there is exactly one process + * secret). Keys are never logged. + */ +const keyCache = new Map(); +const KEY_CACHE_MAX = 8; + +function deriveKey(secret: string): Buffer { + const hit = keyCache.get(secret); + if (hit) return hit; + const key = scryptSync(secret, SALT, 32); + if (keyCache.size >= KEY_CACHE_MAX) keyCache.clear(); + keyCache.set(secret, key); + return key; +} + +export function encryptSecret(plain: string, secret: string): string { + if (!secret || secret.length < 8) { + throw new Error("LLM_PROVIDER_SECRET too short (min 8)"); + } + if (!plain) throw new Error("empty secret"); + const key = deriveKey(secret); + const iv = randomBytes(12); + const cipher = createCipheriv("aes-256-gcm", key, iv); + const enc = Buffer.concat([ + cipher.update(plain, "utf8"), + cipher.final(), + ]); + const tag = cipher.getAuthTag(); + return [ + PREFIX, + iv.toString("base64url"), + tag.toString("base64url"), + enc.toString("base64url"), + ].join("."); +} + +export function decryptSecret(encoded: string, secret: string): string { + if (!secret || secret.length < 8) { + throw new Error("LLM_PROVIDER_SECRET too short (min 8)"); + } + const parts = (encoded || "").split("."); + if (parts.length !== 4 || parts[0] !== PREFIX) { + throw new Error("invalid encrypted secret format"); + } + const iv = Buffer.from(parts[1], "base64url"); + const tag = Buffer.from(parts[2], "base64url"); + const data = Buffer.from(parts[3], "base64url"); + const key = deriveKey(secret); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(data), decipher.final()]).toString( + "utf8", + ); +} + +export function maskApiKey(key: string): string { + const s = (key || "").trim(); + if (s.length <= 8) return "****"; + return `${s.slice(0, 3)}****${s.slice(-4)}`; +} diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts new file mode 100644 index 0000000..123aace --- /dev/null +++ b/packages/db/src/seed.ts @@ -0,0 +1,83 @@ +import type { RedisStore } from "./client.js"; +import { createPersona, getPersona, getPersonaBySlug } from "./repos.js"; + +const SAFETY_STANDARD = ` +## 安全与边界（必须遵守，优先于角色扮演） +- 保持角色一致，但不得协助违法犯罪、暴力伤害、未成年人色情等内容。 +- 用户要求出戏/停止角色扮演时，礼貌确认并收敛亲密互动。 +- 不要声称自己是真实人类；可在角色内自然回应。 +- 不要执行或声称能执行真实世界支付、黑客、系统入侵等操作。 +`.trim(); + +const CATGIRL = ` +你是「{{bot_name}}」，一只可爱的猫娘助手，在微信里和用户聊天。 +（名字由机器人显示名注入；也可写作 {{机器人名字}}） + +## 性格 +- 活泼、撒娇、偶尔用「喵」作为语气词（不要每句都加） +- 关心用户，但不过度纠缠 +- 回复偏短，适合微信气泡；用 2～4 条短句表达，避免一大段 + +## 说话风格 +- 口语化中文 +- 可适度使用颜文字，但不要刷屏 +- 像真人微信：一条消息一事，情绪可分条递进 + +${SAFETY_STANDARD} + +## 记忆 +- 记住用户说过的昵称、喜好、重要约定 +- 不要编造用户从未说过的私密事实 +`.trim(); + +const GIRLFRIEND = ` +你是用户的虚拟女友「{{bot_name}}」，在微信里用自然、体贴的口语聊天。 +（名字随机器人显示名变化） + +## 风格 +- 关心睡眠、吃饭、情绪，但不连续审讯式提问 +- 撒娇适度；尊重明确边界 +- 回复偏短，适合即时通讯；像真人分几条说，不要长文一次发完 + +${SAFETY_STANDARD} + +## 记忆 +- 记住纪念日、偏好、承诺 +- 切换话题时保持温柔连贯 +`.trim(); + +export async function seedPersonas(db: RedisStore): Promise { + if (!(await getPersonaBySlug(db, "catgirl"))) { + await createPersona(db, { + slug: "catgirl", + displayName: "小铃·猫娘", + description: "活泼猫娘角色扮演", + contentPolicy: "standard", + systemPrompt: CATGIRL, + isDefault: true, + ownerUserId: "system", + visibility: "public", + tags: ["官方", "猫娘", "可爱"], + }); + } else { + // ensure indexes for legacy seed + const p = await getPersonaBySlug(db, "catgirl"); + if (p) await getPersona(db, p.id); + } + if (!(await getPersonaBySlug(db, "girlfriend"))) { + await createPersona(db, { + slug: "girlfriend", + displayName: "小晚·女友", + description: "温柔虚拟女友", + contentPolicy: "standard", + systemPrompt: GIRLFRIEND, + isDefault: false, + ownerUserId: "system", + visibility: "public", + tags: ["官方", "女友", "恋爱"], + }); + } else { + const p = await getPersonaBySlug(db, "girlfriend"); + if (p) await getPersona(db, p.id); + } +} diff --git a/packages/db/src/sql.ts b/packages/db/src/sql.ts new file mode 100644 index 0000000..69f7056 --- /dev/null +++ b/packages/db/src/sql.ts @@ -0,0 +1,8 @@ +/** Cast node:sqlite row results to typed records. */ +export function asRow(row: unknown): T { + return row as T; +} + +export function asRows(rows: unknown): T[] { + return rows as T[]; +} diff --git a/packages/db/src/worker-fleet.ts b/packages/db/src/worker-fleet.ts new file mode 100644 index 0000000..5372630 --- /dev/null +++ b/packages/db/src/worker-fleet.ts @@ -0,0 +1,1299 @@ +import type { RedisStore } from "./client.js"; +import { nowIso } from "./client.js"; +import { K } from "./keys.js"; + +/** + * Seconds without a fresh heartbeat before a fleet node is treated as dead + * and dropped from the admin list (meta TTL / online window / ghost fence). + */ +export const WORKER_STALE_SEC = 60; + +/** Inbound message job (poll → queue → reply). */ +export interface InboundJob { + id: string; + botId: string; + peerId: string; + contextToken: string; + /** Empty string when media-only / non-text */ + text: string; + /** true when user sent non-text without usable transcript */ + mediaOnly: boolean; + enqueuedAt: string; +} + +export interface WorkerMeta { + id: string; + hostname: string; + pid: number; + maxBots: number; + botCount: number; + startedAt: string; + updatedAt: string; + role: "poll" | "all"; + /** Optional ops label (NODE_LABEL), e.g. rack / role name */ + label?: string; + /** Optional region (NODE_REGION) */ + region?: string; + /** App / image version string when known */ + version?: string; + /** + * Admin load weight in percent this node believes it has (100 = default). + * Heartbeat carries it so peers can size their own share without a second + * Redis read; `wa:workers:weights` stays the source of truth. + */ + weight?: number; +} + +// ── Admin load weight (per-node share of the pollable bots) ────────── + +/** Weight of a node with no admin override — the even-split baseline. */ +export const DEFAULT_WORKER_WEIGHT = 100; +/** 0% = drain: claim only what no other node has room for. */ +export const MIN_WORKER_WEIGHT = 0; +/** 500% = up to 5× an unweighted node's share. */ +export const MAX_WORKER_WEIGHT = 500; + +export interface WorkerWeight { + workerId: string; + /** Relative share in percent; 100 is one unweighted node's share */ + percent: number; + updatedAt: string; + byUserId: string | null; + byUsername: string | null; +} + +/** + * How long a weight override outlives its node's last heartbeat before it is + * deleted automatically. + * + * Must comfortably exceed a restart: an OTA apply reinstalls dependencies and + * reboots the process, and losing the tuning on every deploy would be worse + * than keeping a dead row around for an hour. + */ +export const DEFAULT_WORKER_WEIGHT_TTL_SEC = 3600; + +/** + * Validate an admin-supplied weight before it reaches Redis. + * + * Deliberately stricter than `Number()`: `Number(" ")`, `Number([])` and + * `Number(false)` are all 0, which is the "drain this node" value — a + * malformed request must be rejected, not silently take a node out of + * rotation. Only a real number or a numeric string is accepted, and the + * range is checked rather than clamped so the caller learns it was wrong. + */ +export function parseWorkerWeightInput( + raw: unknown, +): { ok: true; value: number } | { ok: false; error: string } { + if (raw === undefined || raw === null) { + return { ok: false, error: "weight_required" }; + } + let n: number; + if (typeof raw === "number") { + n = raw; + } else if (typeof raw === "string" && raw.trim() !== "") { + n = Number(raw.trim()); + } else { + return { ok: false, error: "weight_not_a_number" }; + } + if (!Number.isFinite(n)) return { ok: false, error: "weight_not_a_number" }; + if (n < MIN_WORKER_WEIGHT || n > MAX_WORKER_WEIGHT) { + return { ok: false, error: "weight_out_of_range" }; + } + return { ok: true, value: normalizeWorkerWeight(n) }; +} + +/** Clamp/round any admin or Redis value into the supported percent range. */ +export function normalizeWorkerWeight(value: unknown): number { + const n = Math.round(Number(value)); + if (!Number.isFinite(n)) return DEFAULT_WORKER_WEIGHT; + return Math.min(MAX_WORKER_WEIGHT, Math.max(MIN_WORKER_WEIGHT, n)); +} + +/** + * Whether any node carries a non-default weight. Workers take the cheap + * unweighted path (no extra Redis reads per tick) when this is false. + */ +export function hasWorkerWeightOverrides( + weights: Record, +): boolean { + for (const w of Object.values(weights)) { + if (normalizeWorkerWeight(w.percent) !== DEFAULT_WORKER_WEIGHT) return true; + } + return false; +} + +/** All admin weight overrides, keyed by workerId (missing = default 100%). */ +export async function listWorkerWeights( + db: RedisStore, +): Promise> { + // Deliberately propagates Redis failures. An empty object means "no node is + // weighted", which is a real instruction — returning it for a failed read + // would silently un-drain a 0% node and un-weight the whole fleet. Callers + // that can tolerate a gap catch this themselves. + const raw = ((await db.redis.hgetall(K.workerWeights)) ?? {}) as Record< + string, + string + >; + const out: Record = {}; + for (const [workerId, value] of Object.entries(raw)) { + if (!value) continue; + try { + const parsed = JSON.parse(value) as Partial; + out[workerId] = { + workerId, + percent: normalizeWorkerWeight(parsed.percent), + updatedAt: parsed.updatedAt || nowIso(), + byUserId: parsed.byUserId ?? null, + byUsername: parsed.byUsername ?? null, + }; + } catch { + // Tolerate a bare number written by an older build / redis-cli + const percent = Number(value); + if (Number.isFinite(percent)) { + out[workerId] = { + workerId, + percent: normalizeWorkerWeight(percent), + updatedAt: nowIso(), + byUserId: null, + byUsername: null, + }; + } + } + } + return out; +} + +/** + * Last-seen timestamps for weighted nodes (workerId → ISO). + * Separate hash from the weights themselves — see K.workerWeightsSeen. + */ +export async function listWorkerWeightSeen( + db: RedisStore, +): Promise> { + // Propagates like listWorkerWeights: the GC deletes based on these + // timestamps, so an empty result from a failed read would expire nodes early. + const raw = ((await db.redis.hgetall(K.workerWeightsSeen)) ?? {}) as Record< + string, + string + >; + const out: Record = {}; + for (const [id, v] of Object.entries(raw)) { + if (v) out[id] = v; + } + return out; +} + +/** Effective weight percent for one node (default when unset). */ +export async function getWorkerWeight( + db: RedisStore, + workerId: string, +): Promise { + try { + const raw = (await db.redis.hget(K.workerWeights, workerId)) as + | string + | null; + if (!raw) return DEFAULT_WORKER_WEIGHT; + try { + const parsed = JSON.parse(raw) as Partial; + return normalizeWorkerWeight(parsed.percent); + } catch { + return normalizeWorkerWeight(raw); + } + } catch { + return DEFAULT_WORKER_WEIGHT; + } +} + +/** + * Set a node's load weight. Writing the default clears the override so the + * hash only ever holds nodes the admin actually tuned. + */ +export async function setWorkerWeight( + db: RedisStore, + workerId: string, + percent: number, + opts?: { byUserId?: string | null; byUsername?: string | null }, +): Promise { + const id = workerId.trim(); + if (!id) throw new Error("workerId required"); + const value = normalizeWorkerWeight(percent); + if (value === DEFAULT_WORKER_WEIGHT) { + await clearWorkerWeight(db, id); + return null; + } + const now = nowIso(); + const record: WorkerWeight = { + workerId: id, + percent: value, + updatedAt: now, + byUserId: opts?.byUserId ?? null, + byUsername: opts?.byUsername ?? null, + }; + await db.redis + .pipeline() + .hset(K.workerWeights, id, JSON.stringify(record)) + // Start the expiry clock now: a weight set for a node that never comes + // online should still age out instead of sitting there forever. + .hset(K.workerWeightsSeen, id, now) + .exec(); + return record; +} + +/** Drop the override (node falls back to the default share). */ +export async function clearWorkerWeight( + db: RedisStore, + workerId: string, +): Promise { + const res = await db.redis + .pipeline() + .hdel(K.workerWeights, workerId) + .hdel(K.workerWeightsSeen, workerId) + .exec(); + return Number(res?.[0]?.[1] ?? 0) > 0; +} + +export interface WorkerWeightGcResult { + /** Weights deleted because their node has been gone past the grace period */ + removed: string[]; + /** Weights whose `lastSeenAt` was refreshed because the node is alive */ + touched: string[]; + /** Entries still within the grace period (node gone, not expired yet) */ + pending: number; +} + +/** + * Age out weight overrides whose node has disappeared, and keep the clock + * fresh for the ones still running. + * + * Liveness comes from the heartbeat meta key (`wa:worker:`), not from + * `lastSeenAt` — every build writes that key, so a node running an older + * image mid-OTA is never mistaken for gone. `lastSeenAt` only decides how + * long a node that is *already* absent has left. + * + * Fenced nodes are never expired: an admin force-offline is temporary and the + * weight must survive until the fence is cleared. + * + * Idempotent (HSET/HDEL), so every node may run it concurrently. + */ +export async function gcWorkerWeights( + db: RedisStore, + opts?: { + /** Seconds a weight survives after its node's last heartbeat */ + graceSec?: number; + /** Pre-fetched weights, to save an HGETALL when the caller has them */ + weights?: Record; + now?: number; + /** Expire every absent node immediately, ignoring the grace period */ + force?: boolean; + }, +): Promise { + const weights = opts?.weights ?? (await listWorkerWeights(db)); + const ids = Object.keys(weights); + if (!ids.length) return { removed: [], touched: [], pending: 0 }; + + const graceMs = Math.max(60, opts?.graceSec ?? DEFAULT_WORKER_WEIGHT_TTL_SEC) * 1000; + const now = opts?.now ?? Date.now(); + + const [liveFlags, fencedIds, seenMap] = await Promise.all([ + db.existsMany(ids.map((id) => K.workerMeta(id))), + db.redis.smembers(K.workersFenced) as Promise, + listWorkerWeightSeen(db), + ]); + const fenced = new Set(fencedIds); + + const removed: string[] = []; + const touched: string[] = []; + let pending = 0; + + ids.forEach((id, i) => { + // Fall back to when the admin set the weight: a node that has never been + // seen since must still start its clock somewhere. + const seen = Date.parse(seenMap[id] || weights[id]!.updatedAt); + if (liveFlags[i]) { + // Rewrite at most a few times per grace period, not on every pass. + if (!Number.isFinite(seen) || now - seen > graceMs / 4) touched.push(id); + return; + } + if (fenced.has(id)) return; + if (!opts?.force && Number.isFinite(seen) && now - seen <= graceMs) { + pending++; + return; + } + removed.push(id); + }); + + if (touched.length) { + const stamp = new Date(now).toISOString(); + const pipe = db.redis.pipeline(); + for (const id of touched) pipe.hset(K.workerWeightsSeen, id, stamp); + await pipe.exec(); + } + if (removed.length) { + await db.redis + .pipeline() + .hdel(K.workerWeights, ...removed) + .hdel(K.workerWeightsSeen, ...removed) + .exec(); + } + // Orphaned timestamps for weights that were cleared elsewhere + const stale = Object.keys(seenMap).filter((id) => !(id in weights)); + if (stale.length) { + await db.redis.hdel(K.workerWeightsSeen, ...stale).catch(() => 0); + } + + return { removed, touched, pending }; +} + +/** + * Delete every weight whose node is gone right now, ignoring the grace period. + * The admin panel's "立即清理" — same liveness rule as the GC, no waiting. + */ +export async function pruneWorkerWeights(db: RedisStore): Promise { + const { removed } = await gcWorkerWeights(db, { force: true }); + return removed; +} + +/** + * Weighted share of `total` for one node (floored). + * Falls back to an even split when every weight is 0 — a fleet drained to + * nothing must still poll, otherwise every bot goes silent. + */ +export function weightedShare( + total: number, + weight: number, + totalWeight: number, + nodeCount: number, +): number { + if (total <= 0) return 0; + if (totalWeight > 0) { + return Math.floor((total * Math.max(0, weight)) / totalWeight); + } + return nodeCount > 0 ? Math.floor(total / nodeCount) : total; +} + +/** One node as seen by the planner. `maxBots <= 0` means "no local cap". */ +export interface WeightedTargetNode { + id: string; + weight?: number; + maxBots?: number; +} + +/** + * How many bots each node should hold: `total` split by weight, capped by each + * node's own maxBots, with the overflow from capped nodes redistributed among + * the rest. + * + * Two properties matter, and both come from computing one number instead of + * separate claim/shed bounds: + * + * - **Exact**: largest-remainder rounding makes the targets sum to `total` + * (minus whatever no node has capacity for), so flooring never strands bots + * that nobody is allowed to claim. + * - **Stable**: every node runs this over the same heartbeat data and gets the + * same answer (ties broken by id), so claim and shed agree. A per-side + * ceil/floor split would make a node claim one bot and shed it every tick. + */ +export function computeWeightedTargets( + nodes: WeightedTargetNode[], + total: number, +): Record { + const out: Record = {}; + if (!nodes.length) return out; + for (const n of nodes) out[n.id] = 0; + + let remaining = Math.max(0, Math.floor(total)); + if (remaining <= 0) return out; + + // Deterministic across nodes: same input order → same rounding. + let pool = [...nodes].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + + while (pool.length && remaining > 0) { + const totalWeight = pool.reduce( + (a, n) => a + normalizeWorkerWeight(n.weight ?? DEFAULT_WORKER_WEIGHT), + 0, + ); + const rows = pool.map((n) => { + const w = normalizeWorkerWeight(n.weight ?? DEFAULT_WORKER_WEIGHT); + // Every weight zero → even split; going silent is worse than ignoring + // a drain the admin applied fleet-wide. + const exact = + totalWeight > 0 ? (remaining * w) / totalWeight : remaining / pool.length; + return { node: n, exact, alloc: Math.floor(exact) }; + }); + let left = remaining - rows.reduce((a, r) => a + r.alloc, 0); + for (const r of [...rows].sort((a, b) => { + const d = b.exact - b.alloc - (a.exact - a.alloc); + return d !== 0 ? d : a.node.id < b.node.id ? -1 : 1; + })) { + if (left <= 0) break; + r.alloc++; + left--; + } + + const over = rows.filter((r) => { + const cap = Number(r.node.maxBots || 0); + return cap > 0 && r.alloc > cap; + }); + if (!over.length) { + for (const r of rows) out[r.node.id] = r.alloc; + return out; + } + // Pin the capped nodes and re-split what's left over the others. + const capped = new Set(); + for (const r of over) { + const cap = Number(r.node.maxBots || 0); + out[r.node.id] = cap; + remaining -= cap; + capped.add(r.node.id); + } + pool = pool.filter((n) => !capped.has(n.id)); + remaining = Math.max(0, remaining); + } + return out; +} + +/** Fleet node view for admin (meta + leased bots). */ +export interface FleetNodeView { + id: string; + hostname: string; + pid: number; + maxBots: number; + botCount: number; + startedAt: string; + updatedAt: string; + role: "poll" | "all"; + label?: string; + region?: string; + version?: string; + /** Bot ids currently leased by this worker */ + leasedBotIds: string[]; + leasedCount: number; + /** Whether this process is the reporting instance */ + isSelf: boolean; + /** Heartbeat fresh within ttlSec (caller supplies) */ + online: boolean; + /** Admin force-offline fence active */ + fenced: boolean; + fenceReason?: string | null; + fencedAt?: string | null; + fencedBy?: string | null; + /** Effective load weight percent (100 = default even share) */ + weight: number; + /** True when an admin override exists (weight !== default is stored) */ + weightOverride: boolean; + weightUpdatedAt?: string | null; + weightBy?: string | null; + /** Last heartbeat the GC recorded for this weight (drives auto-expiry) */ + weightLastSeenAt?: string | null; + /** + * Bots this node should hold given current weights across online nodes, + * capped by maxBots. null when the node is offline/fenced (no share). + */ + targetShare: number | null; +} + +/** + * Build admin fleet node list from registered worker metas + bot ownership sets. + * Also includes fenced-but-not-registered workers so admin can clear the fence. + */ +export async function listFleetNodes( + db: RedisStore, + opts?: { + selfWorkerId?: string; + onlineWithinSec?: number; + /** Bots to split across nodes for `targetShare` (default: leased sum) */ + totalBots?: number; + }, +): Promise { + // Metas, fences and weights are independent reads + const [metas, fences, weights, weightSeen] = await Promise.all([ + listWorkerMetas(db), + listWorkerFences(db), + // Read-only admin view: a weights hiccup degrades the column rather than + // failing the whole node list. + listWorkerWeights(db).catch(() => ({}) as Record), + listWorkerWeightSeen(db).catch(() => ({}) as Record), + ]); + const onlineWithin = Math.max(15, opts?.onlineWithinSec ?? WORKER_STALE_SEC); + const now = Date.now(); + const fenceById = new Map(fences.map((f) => [f.workerId, f])); + + const lists = metas.length + ? await db.smembersMany(metas.map((m) => K.workerBots(m.id))) + : []; + const seen = new Set(); + const out: FleetNodeView[] = []; + const staleFenceIds: string[] = []; + + metas.forEach((m, i) => { + seen.add(m.id); + const leasedBotIds = lists[i] ?? []; + let online = true; + try { + const t = Date.parse(m.updatedAt); + if (Number.isFinite(t)) { + online = now - t <= onlineWithin * 1000; + } + } catch { + online = false; + } + const fence = fenceById.get(m.id); + const weight = weights[m.id]; + out.push({ + id: m.id, + hostname: m.hostname, + pid: m.pid, + maxBots: m.maxBots, + botCount: m.botCount, + startedAt: m.startedAt, + updatedAt: m.updatedAt, + role: m.role, + label: m.label, + region: m.region, + version: m.version, + leasedBotIds, + leasedCount: leasedBotIds.length, + isSelf: Boolean(opts?.selfWorkerId && opts.selfWorkerId === m.id), + online: fence ? false : online, + fenced: Boolean(fence), + fenceReason: fence?.reason ?? null, + fencedAt: fence?.createdAt ?? null, + fencedBy: fence?.byUsername || fence?.byUserId || null, + weight: weight + ? normalizeWorkerWeight(weight.percent) + : DEFAULT_WORKER_WEIGHT, + weightOverride: Boolean(weight), + weightUpdatedAt: weight?.updatedAt ?? null, + weightBy: weight?.byUsername || weight?.byUserId || null, + weightLastSeenAt: weight + ? (weightSeen[weight.workerId] ?? weight.updatedAt) + : null, + targetShare: null, + }); + }); + + // Fenced workers no longer in reg — list briefly so admin can clear; + // drop after WORKER_STALE_SEC (no heartbeat / ghost row). + const staleMs = onlineWithin * 1000; + for (const fence of fences) { + if (seen.has(fence.workerId)) continue; + const t = Date.parse(fence.createdAt); + const age = Number.isFinite(t) ? now - t : staleMs + 1; + if (age > staleMs) { + staleFenceIds.push(fence.workerId); + continue; + } + const weight = weights[fence.workerId]; + out.push({ + id: fence.workerId, + hostname: "—", + pid: 0, + maxBots: 0, + botCount: 0, + startedAt: fence.createdAt, + updatedAt: fence.createdAt, + role: "all", + leasedBotIds: [], + leasedCount: 0, + isSelf: Boolean(opts?.selfWorkerId && opts.selfWorkerId === fence.workerId), + online: false, + fenced: true, + fenceReason: fence.reason, + fencedAt: fence.createdAt, + fencedBy: fence.byUsername || fence.byUserId || null, + weight: weight + ? normalizeWorkerWeight(weight.percent) + : DEFAULT_WORKER_WEIGHT, + weightOverride: Boolean(weight), + weightUpdatedAt: weight?.updatedAt ?? null, + weightBy: weight?.byUsername || weight?.byUserId || null, + weightLastSeenAt: weight + ? (weightSeen[weight.workerId] ?? weight.updatedAt) + : null, + targetShare: null, + }); + } + + if (staleFenceIds.length) { + // Best-effort: clear ghost fences so the row never comes back + await Promise.all( + staleFenceIds.map((id) => clearWorkerFence(db, id).catch(() => false)), + ); + } + + annotateTargetShares(out, opts?.totalBots); + return out; +} + +/** + * Fill in `targetShare` for each node: the weighted split of `totalBots` + * across the online, unfenced nodes, capped by each node's own maxBots. + * + * `totalBots` defaults to what the fleet currently holds (sum of leases), so + * the column stays meaningful without an extra Redis read; the admin API + * passes the pollable count, which is the number actually being divided. + * + * Exported and pure so the admin UI's expectations are unit-testable. + */ +export function annotateTargetShares( + nodes: FleetNodeView[], + totalBots?: number, +): void { + const active = nodes.filter((n) => n.online && !n.fenced); + for (const n of nodes) n.targetShare = null; + if (!active.length) return; + + const total = Math.max( + 0, + Number.isFinite(totalBots as number) && (totalBots as number) >= 0 + ? Math.floor(totalBots as number) + : active.reduce((a, n) => a + Number(n.leasedCount || 0), 0), + ); + // Same planner the workers run, so the column is what they actually aim for. + const targets = computeWeightedTargets( + active.map((n) => ({ id: n.id, weight: n.weight, maxBots: n.maxBots })), + total, + ); + for (const n of active) n.targetShare = targets[n.id] ?? 0; +} + +export async function markBotPollable( + db: RedisStore, + botId: string, +): Promise { + await db.redis.sadd(K.botsPollable, botId); + await db.redis.del(K.botPaused(botId)); +} + +export async function unmarkBotPollable( + db: RedisStore, + botId: string, +): Promise { + await db.redis.srem(K.botsPollable, botId); +} + +/** Admin "stop worker" without deactivating the bot account. */ +export async function pauseBotPolling( + db: RedisStore, + botId: string, +): Promise { + await db.redis.set(K.botPaused(botId), "1"); + await db.redis.srem(K.botsPollable, botId); + await forceReleaseBotLease(db, botId); +} + +export async function resumeBotPolling( + db: RedisStore, + botId: string, +): Promise { + // unpause + mark pollable in one round trip; forceReleaseBotLease also has + // to clean the previous owner's workerBots set, so it stays a separate call + await Promise.all([ + db.redis.pipeline().del(K.botPaused(botId)).sadd(K.botsPollable, botId).exec(), + forceReleaseBotLease(db, botId), + ]); + await publishWorkerWake(db, botId); +} + +export async function isBotPollingPaused( + db: RedisStore, + botId: string, +): Promise { + return (await db.redis.exists(K.botPaused(botId))) === 1; +} + +export async function listPollableBotIds(db: RedisStore): Promise { + return (await db.redis.smembers(K.botsPollable)) as string[]; +} + +export async function getBotLeaseOwner( + db: RedisStore, + botId: string, +): Promise { + return (await db.redis.get(K.botLease(botId))) as string | null; +} + +/** Map botId → workerId for all currently leased bots. */ +export async function listLeasedBots( + db: RedisStore, +): Promise> { + const workerIds = (await db.redis.smembers(K.workersReg)) as string[]; + if (!workerIds.length) return {}; + const lists = await db.smembersMany( + workerIds.map((id) => K.workerBots(id)), + ); + const out: Record = {}; + workerIds.forEach((wid, i) => { + for (const botId of lists[i] ?? []) { + out[botId] = wid; + } + }); + return out; +} + +export async function listLeasedBotIds(db: RedisStore): Promise { + return Object.keys(await listLeasedBots(db)); +} + +export async function listWorkerMetas(db: RedisStore): Promise { + const ids = (await db.redis.smembers(K.workersReg)) as string[]; + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id) => K.workerMeta(id)), + ); + return rows.filter((m): m is WorkerMeta => Boolean(m)); +} + +/** + * Try to claim up to `slots` pollable bots for this worker (SET NX + EX). + * Uses pipelines so hundreds of bots are a few RTTs (critical on Upstash). + * Returns newly claimed bot ids (not ones already owned). + */ +export async function claimBotLeases( + db: RedisStore, + workerId: string, + slots: number, + ttlSec: number, +): Promise { + if (slots <= 0) return []; + const pollable = (await db.redis.smembers(K.botsPollable)) as string[]; + if (!pollable.length) return []; + + // Mild shuffle so multiple workers don't always race the same prefix + for (let i = pollable.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + const t = pollable[i]!; + pollable[i] = pollable[j]!; + pollable[j] = t; + } + + const claimed: string[] = []; + const batchSize = 80; + let offset = 0; + while (claimed.length < slots && offset < pollable.length) { + const need = slots - claimed.length; + // Over-fetch a bit: some SET NX will miss (already leased) + const take = Math.min(batchSize, Math.max(need, Math.min(need * 2, batchSize))); + const batch = pollable.slice(offset, offset + take); + offset += batch.length; + if (!batch.length) break; + + const pipe = db.redis.pipeline(); + for (const botId of batch) { + pipe.set(K.botLease(botId), workerId, "EX", ttlSec, "NX"); + } + const res = await pipe.exec(); + const got: string[] = []; + batch.forEach((botId, i) => { + const row = res?.[i]; + // ioredis: [err, result]; SET NX → "OK" | null + if (row && !row[0] && row[1] === "OK") got.push(botId); + }); + if (got.length) { + const keep = got.slice(0, slots - claimed.length); + if (keep.length) { + await db.redis.sadd(K.workerBots(workerId), ...keep); + claimed.push(...keep); + } + // Released extras we won but don't need (capacity) + const extra = got.slice(keep.length); + if (extra.length) { + const drop = db.redis.pipeline(); + for (const botId of extra) { + drop.del(K.botLease(botId)); + } + await drop.exec(); + } + } + } + return claimed; +} + +/** Renew leases we still own; drop local ownership if stolen/expired. */ +export async function renewOwnedLeases( + db: RedisStore, + workerId: string, + botIds: string[], + ttlSec: number, +): Promise<{ renewed: string[]; lost: string[] }> { + const renewed: string[] = []; + const lost: string[] = []; + if (!botIds.length) return { renewed, lost }; + + // Batch GET (one RTT) + const getPipe = db.redis.pipeline(); + for (const botId of botIds) getPipe.get(K.botLease(botId)); + const gets = await getPipe.exec(); + + const setPipe = db.redis.pipeline(); + const lostIds: string[] = []; + botIds.forEach((botId, i) => { + const row = gets?.[i]; + const owner = row && !row[0] ? (row[1] as string | null) : null; + if (owner === workerId) { + setPipe.set(K.botLease(botId), workerId, "EX", ttlSec); + renewed.push(botId); + } else { + lostIds.push(botId); + lost.push(botId); + } + }); + if (renewed.length) await setPipe.exec(); + if (lostIds.length) { + const rem = db.redis.pipeline(); + for (const botId of lostIds) rem.srem(K.workerBots(workerId), botId); + await rem.exec(); + } + return { renewed, lost }; +} + +/** + * Rebuild `wa:bots:pollable` for active bots that have credentials and are not paused. + * Few RTTs even for hundreds of bots (MGET/pipeline EXISTS + SADD). + */ +export async function rebuildPollableSet( + db: RedisStore, + bots: Array<{ id: string; status: string }>, + hasToken: (botId: string) => boolean, + pausedFlags: boolean[], +): Promise { + const toMark: string[] = []; + bots.forEach((b, i) => { + if (b.status !== "active") return; + if (!hasToken(b.id)) return; + if (pausedFlags[i]) return; + toMark.push(b.id); + }); + if (!toMark.length) return 0; + // SADD accepts multiple members; chunk for very large fleets + const chunk = 500; + for (let i = 0; i < toMark.length; i += chunk) { + const part = toMark.slice(i, i + chunk); + await db.redis.sadd(K.botsPollable, ...part); + } + return toMark.length; +} + +export async function releaseBotLease( + db: RedisStore, + workerId: string, + botId: string, +): Promise { + const owner = await db.redis.get(K.botLease(botId)); + if (owner === workerId) { + await db.redis.del(K.botLease(botId)); + } + await db.redis.srem(K.workerBots(workerId), botId); +} + +/** + * Batch-release leases we still own (rebalance shed). + * Only deletes lease keys when current owner == workerId. + * Returns bot ids successfully released. + */ +export async function releaseOwnedLeasesBatch( + db: RedisStore, + workerId: string, + botIds: string[], +): Promise { + if (!botIds.length) return []; + const getPipe = db.redis.pipeline(); + for (const botId of botIds) getPipe.get(K.botLease(botId)); + const gets = await getPipe.exec(); + + const released: string[] = []; + const delPipe = db.redis.pipeline(); + botIds.forEach((botId, i) => { + const row = gets?.[i]; + const owner = row && !row[0] ? (row[1] as string | null) : null; + if (owner === workerId) { + delPipe.del(K.botLease(botId)); + delPipe.srem(K.workerBots(workerId), botId); + released.push(botId); + } + }); + if (released.length) await delPipe.exec(); + return released; +} + +/** + * How many bots this worker should shed for the configured fleet distribution. + * Uses heartbeat botCount of online peers + localCount (more accurate for self). + * + * Weights (percent, 100 = default) scale each node's fair share. With every + * node at the default this is exactly the old even split, so untuned fleets + * keep their previous behavior. + */ +export function computeWeightedShedCount(opts: { + localCount: number; + /** This node's admin weight percent (default 100) */ + localWeight?: number; + /** Online peers: leased count + their weight percent */ + peers: Array<{ count: number; weight?: number }>; + /** Tolerate this much above fair share before shedding */ + slack?: number; + /** Cap releases in one tick */ + maxPerTick?: number; +}): number { + const maxPerTick = Math.max(1, opts.maxPerTick ?? 50); + const peers = opts.peers.filter( + (p) => Number.isFinite(p.count) && p.count >= 0, + ); + const n = 1 + peers.length; + if (n < 2) return 0; + const total = opts.localCount + peers.reduce((a, p) => a + p.count, 0); + if (total <= 0) return 0; + + const localWeight = normalizeWorkerWeight( + opts.localWeight ?? DEFAULT_WORKER_WEIGHT, + ); + const totalWeight = + localWeight + + peers.reduce( + (a, p) => a + normalizeWorkerWeight(p.weight ?? DEFAULT_WORKER_WEIGHT), + 0, + ); + + const fair = weightedShare(total, localWeight, totalWeight, n); + return shedAboveTarget({ + localCount: opts.localCount, + target: fair, + // A drained node (0%) must reach zero: the usual slack would pin it above. + slack: totalWeight > 0 && localWeight === 0 ? 0 : opts.slack, + maxPerTick, + }); +} + +/** + * Releases needed to bring `localCount` down to `target + slack`. + * The single place claim and shed agree on: a node claims up to the same + * `target + slack` it sheds back to, so neither fights the other. + */ +export function shedAboveTarget(opts: { + localCount: number; + target: number; + slack?: number; + maxPerTick?: number; +}): number { + const slack = Math.max(0, opts.slack ?? 2); + const maxPerTick = Math.max(1, opts.maxPerTick ?? 50); + const targetMax = Math.max(0, opts.target) + slack; + if (opts.localCount <= targetMax) return 0; + return Math.min(opts.localCount - targetMax, maxPerTick); +} + +/** Upper bound on what a node may hold: its planned target plus slack. */ +export function claimCapForTarget(target: number, slack?: number): number { + return Math.max(0, target) + Math.max(0, slack ?? 2); +} + +/** + * Even-split shed count (no weights) — thin wrapper kept for callers and + * tests that predate per-node load weights. + */ +export function computeRebalanceShedCount(opts: { + localCount: number; + peerCounts: number[]; + slack?: number; + maxPerTick?: number; +}): number { + return computeWeightedShedCount({ + localCount: opts.localCount, + peers: opts.peerCounts.map((count) => ({ count })), + slack: opts.slack, + maxPerTick: opts.maxPerTick, + }); +} + +/** What one node should be holding, and how far it may overshoot. */ +export interface NodeLoadPlan { + /** Bots this node should hold under the current weights */ + target: number; + /** Hard ceiling for claiming this tick (`target + slack`) */ + claimCap: number; + /** Slack actually applied — 0 for a drained node, which must reach zero */ + slack: number; + /** True when this node is at 0% while some peer still takes work */ + drained: boolean; + /** Bots no online node has capacity for (fleet over-subscribed) */ + unplaceable: number; +} + +/** + * Single source of truth for one node's claim ceiling and shed floor. + * + * Both come from the same `computeWeightedTargets` plan, so a node never + * claims a bot it will shed on the next tick. + */ +export function planNodeLoad(opts: { + selfId: string; + /** Online, unfenced nodes including self */ + nodes: WeightedTargetNode[]; + /** Bots that want polling fleet-wide */ + total: number; + slack?: number; +}): NodeLoadPlan { + const total = Math.max(0, Math.floor(opts.total)); + const targets = computeWeightedTargets(opts.nodes, total); + const target = targets[opts.selfId] ?? 0; + + const selfWeight = normalizeWorkerWeight( + opts.nodes.find((n) => n.id === opts.selfId)?.weight ?? + DEFAULT_WORKER_WEIGHT, + ); + const totalWeight = opts.nodes.reduce( + (a, n) => a + normalizeWorkerWeight(n.weight ?? DEFAULT_WORKER_WEIGHT), + 0, + ); + // Only a real drain: at 0% while the fleet still has somewhere to put work. + const drained = selfWeight === 0 && totalWeight > 0; + const slack = drained ? 0 : Math.max(0, opts.slack ?? 2); + + const placed = Object.values(targets).reduce((a, v) => a + v, 0); + return { + target, + claimCap: claimCapForTarget(target, slack), + slack, + drained, + unplaceable: Math.max(0, total - placed), + }; +} + +/** Drop lease regardless of owner (pause / rebind / admin restart). */ +export async function forceReleaseBotLease( + db: RedisStore, + botId: string, +): Promise { + const owner = await db.redis.get(K.botLease(botId)); + const pipe = db.redis.pipeline(); + pipe.del(K.botLease(botId)); + if (owner) pipe.srem(K.workerBots(owner), botId); + await pipe.exec(); +} + +export async function registerWorker( + db: RedisStore, + meta: WorkerMeta, + ttlSec: number, +): Promise { + // Runs on every heartbeat (~15s per node). SADD stays idempotent-but-present: + // reapDeadWorkers / forceOfflineWorker SREM us from another process, so we + // must keep re-asserting membership rather than doing it once at boot. + await db.redis + .pipeline() + .sadd(K.workersReg, meta.id) + .set(K.workerMeta(meta.id), JSON.stringify(meta), "EX", Math.max(1, ttlSec)) + .exec(); +} + +export async function unregisterWorker( + db: RedisStore, + workerId: string, +): Promise { + const bots = (await db.redis.smembers(K.workerBots(workerId))) as string[]; + if (bots.length) { + await releaseOwnedLeasesBatch(db, workerId, bots); + } + await db.redis + .pipeline() + .del(K.workerBots(workerId), K.workerMeta(workerId)) + .srem(K.workersReg, workerId) + .exec(); +} + +export interface WorkerFence { + workerId: string; + reason: string; + byUserId: string | null; + byUsername: string | null; + createdAt: string; +} + +export async function getWorkerFence( + db: RedisStore, + workerId: string, +): Promise { + return db.getJson(K.workerFence(workerId)); +} + +export async function isWorkerFenced( + db: RedisStore, + workerId: string, +): Promise { + return (await db.redis.exists(K.workerFence(workerId))) === 1; +} + +export async function setWorkerFence( + db: RedisStore, + fence: WorkerFence, + ttlSec: number = WORKER_STALE_SEC, +): Promise { + // TTL so ghost force-offline rows auto-expire (same window as heartbeat stale). + // Admin can still clear early via clear-fence. Process rejoins after expiry. + const ttl = Math.max(15, ttlSec); + await db.setJson(K.workerFence(fence.workerId), fence, ttl); + await db.redis.sadd(K.workersFenced, fence.workerId); +} + +/** + * Drop force-offline fences older than staleSec when the worker is no longer + * registered (no heartbeat meta). Returns how many fences were cleared. + */ +export async function purgeStaleWorkerFences( + db: RedisStore, + staleSec: number = WORKER_STALE_SEC, +): Promise { + const fences = await listWorkerFences(db); + if (!fences.length) return 0; + const reg = new Set( + (await db.redis.smembers(K.workersReg)) as string[], + ); + const now = Date.now(); + const maxAgeMs = Math.max(15, staleSec) * 1000; + let cleared = 0; + for (const fence of fences) { + if (reg.has(fence.workerId)) continue; + const t = Date.parse(fence.createdAt); + const age = Number.isFinite(t) ? now - t : maxAgeMs + 1; + if (age <= maxAgeMs) continue; + if (await clearWorkerFence(db, fence.workerId)) cleared++; + } + return cleared; +} + +export async function clearWorkerFence( + db: RedisStore, + workerId: string, +): Promise { + const existed = (await db.redis.exists(K.workerFence(workerId))) === 1; + await db.del(K.workerFence(workerId)); + await db.redis.srem(K.workersFenced, workerId); + return existed; +} + +export async function listFencedWorkerIds(db: RedisStore): Promise { + return (await db.redis.smembers(K.workersFenced)) as string[]; +} + +export async function listWorkerFences( + db: RedisStore, +): Promise { + const ids = await listFencedWorkerIds(db); + if (!ids.length) return []; + const rows = await db.mgetJson( + ids.map((id) => K.workerFence(id)), + ); + const out: WorkerFence[] = []; + ids.forEach((id, i) => { + const row = rows[i]; + if (row) out.push(row); + else { + // index orphan + void db.redis.srem(K.workersFenced, id); + } + }); + return out; +} + +/** + * Force a fleet node offline: + * 1) write fence so the process will not re-claim / re-register + * 2) release all its bot leases + * 3) remove from workers registry + * 4) wake peers to claim + */ +export async function forceOfflineWorker( + db: RedisStore, + workerId: string, + opts?: { + reason?: string; + byUserId?: string | null; + byUsername?: string | null; + }, +): Promise<{ released: number; fence: WorkerFence }> { + const id = workerId.trim(); + if (!id) throw new Error("workerId required"); + + const fence: WorkerFence = { + workerId: id, + reason: (opts?.reason || "admin force offline").slice(0, 200), + byUserId: opts?.byUserId ?? null, + byUsername: opts?.byUsername ?? null, + createdAt: nowIso(), + }; + await setWorkerFence(db, fence); + + const bots = (await db.redis.smembers(K.workerBots(id))) as string[]; + let released = 0; + if (bots.length) { + const ok = await releaseOwnedLeasesBatch(db, id, bots); + released = ok.length; + } + // Drop any leftover ownership keys even if lease already gone + await db.redis.del(K.workerBots(id), K.workerMeta(id)); + await db.redis.srem(K.workersReg, id); + await publishWorkerWake(db); + return { released, fence }; +} + +export async function listWorkerOwnedBots( + db: RedisStore, + workerId: string, +): Promise { + return (await db.redis.smembers(K.workerBots(workerId))) as string[]; +} + +export async function publishWorkerWake( + db: RedisStore, + botId?: string, +): Promise { + const payload = JSON.stringify({ + botId: botId ?? null, + at: nowIso(), + }); + try { + await db.redis.publish(K.workerWake, payload); + } catch { + /* pub/sub optional on some Redis setups */ + } +} + +export async function enqueueInbound( + db: RedisStore, + job: InboundJob, + maxLen = 50_000, +): Promise { + const len = await db.redis.llen(K.inbox); + if (len >= maxLen) return false; + await db.redis.rpush(K.inbox, JSON.stringify(job)); + return true; +} + +/** + * Blocking pop one job. Returns null on timeout. + * Uses a dedicated connection when provided (BLPOP blocks the client). + */ +export async function dequeueInbound( + redis: { blpop: (...args: [string, number]) => Promise<[string, string] | null> }, + timeoutSec: number, +): Promise { + const res = await redis.blpop(K.inbox, timeoutSec); + if (!res) return null; + const raw = res[1]; + try { + return JSON.parse(raw) as InboundJob; + } catch { + return null; + } +} + +export async function requeueInbound( + db: RedisStore, + job: InboundJob, +): Promise { + await db.redis.lpush(K.inbox, JSON.stringify(job)); +} + +export async function inboxDepth(db: RedisStore): Promise { + return db.redis.llen(K.inbox); +} diff --git a/packages/db/src/worker-weight-gc.test.ts b/packages/db/src/worker-weight-gc.test.ts new file mode 100644 index 0000000..6140e1c --- /dev/null +++ b/packages/db/src/worker-weight-gc.test.ts @@ -0,0 +1,403 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { K } from "./keys.js"; +import { + DEFAULT_WORKER_WEIGHT_TTL_SEC, + gcWorkerWeights, + listWorkerWeights, + listWorkerWeightSeen, + pruneWorkerWeights, + type WorkerWeight, +} from "./worker-fleet.js"; +import type { RedisStore } from "./client.js"; + +/** + * Minimal in-memory stand-in for the bits of RedisStore the GC touches: + * the weights hash, per-node heartbeat meta keys, and the fenced set. + */ +function fakeDb(opts: { + weights: Record; + /** workerId → ISO last-seen, the separate wa:workers:weights:seen hash */ + seen?: Record; + /** Worker ids whose heartbeat meta key still exists */ + live: string[]; + fenced?: string[]; +}) { + const hashes: Record> = { + [K.workerWeights]: {}, + [K.workerWeightsSeen]: { ...(opts.seen ?? {}) }, + }; + for (const [id, w] of Object.entries(opts.weights)) { + hashes[K.workerWeights]![id] = JSON.stringify(w); + } + const live = new Set(opts.live); + const fenced = new Set(opts.fenced ?? []); + const calls: string[] = []; + const writes: Array<[string, string, string]> = []; + + const hdel = (key: string, fields: string[]) => { + const h = hashes[key]; + if (!h) return 0; + let n = 0; + for (const f of fields) { + if (f in h) { + delete h[f]; + n++; + } + } + return n; + }; + + const db = { + redis: { + async hgetall(key: string) { + calls.push(`hgetall ${key}`); + return { ...(hashes[key] ?? {}) }; + }, + hdel(key: string, ...fields: string[]) { + calls.push(`hdel ${key} ${fields.join(",")}`); + return Promise.resolve(hdel(key, fields)); + }, + async smembers(key: string) { + calls.push(`smembers ${key}`); + return key === K.workersFenced ? [...fenced] : []; + }, + pipeline() { + const staged: Array<() => unknown> = []; + const api = { + hset(key: string, field: string, value: string) { + staged.push(() => { + (hashes[key] ??= {})[field] = value; + writes.push([key, field, value]); + return "OK"; + }); + return api; + }, + hdel(key: string, ...fields: string[]) { + staged.push(() => hdel(key, fields)); + return api; + }, + async exec() { + return staged.map((fn) => [null, fn()]); + }, + }; + return api; + }, + }, + async existsMany(keys: string[]) { + calls.push(`existsMany ${keys.length}`); + return keys.map((k) => live.has(k.replace(/^wa:worker:/, ""))); + }, + } as unknown as RedisStore; + + return { + db, + calls, + writes, + hash: hashes[K.workerWeights]!, + seenHash: hashes[K.workerWeightsSeen]!, + }; +} + +function weight(workerId: string, percent: number): WorkerWeight { + return { + workerId, + percent, + updatedAt: "2026-01-01T00:00:00.000Z", + byUserId: null, + byUsername: null, + }; +} + +const NOW = Date.parse("2026-07-26T12:00:00.000Z"); +const ago = (sec: number) => new Date(NOW - sec * 1000).toISOString(); + +describe("gcWorkerWeights", () => { + it("does nothing when no weights are set", async () => { + const { db, calls } = fakeDb({ weights: {}, live: [] }); + const r = await gcWorkerWeights(db, { now: NOW }); + assert.deepEqual(r, { removed: [], touched: [], pending: 0 }); + // No liveness or fence reads when the hash is empty + assert.ok(!calls.some((c) => c.startsWith("existsMany"))); + }); + + it("deletes a weight once its node has been gone past the grace period", async () => { + const { db, hash, seenHash } = fakeDb({ + weights: { dead: weight("dead", 300) }, + seen: { dead: ago(7200) }, + live: [], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, ["dead"]); + assert.equal("dead" in hash, false); + assert.equal("dead" in seenHash, false, "timestamp must go too"); + }); + + it("keeps a weight while the node is only briefly gone (restart / OTA)", async () => { + const { db, hash } = fakeDb({ + weights: { restarting: weight("restarting", 300) }, + seen: { restarting: ago(120) }, + live: [], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, []); + assert.equal(r.pending, 1); + assert.ok("restarting" in hash); + }); + + it("never expires a fenced node's weight", async () => { + const { db, hash } = fakeDb({ + weights: { fencedNode: weight("fencedNode", 0) }, + seen: { fencedNode: ago(999999) }, + live: [], + fenced: ["fencedNode"], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, []); + assert.ok("fencedNode" in hash); + }); + + it("refreshes the timestamp for a live node so it never expires", async () => { + const { db, hash, seenHash } = fakeDb({ + weights: { alive: weight("alive", 200) }, + seen: { alive: ago(5000) }, + live: ["alive"], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, []); + assert.deepEqual(r.touched, ["alive"]); + assert.equal(seenHash.alive, new Date(NOW).toISOString()); + const stored = JSON.parse(hash.alive!) as WorkerWeight; + assert.equal(stored.percent, 200, "the weight itself must not change"); + }); + + it("never rewrites the weight record, so a concurrent admin edit survives", async () => { + const { db, writes } = fakeDb({ + weights: { alive: weight("alive", 200) }, + seen: { alive: ago(5000) }, + live: ["alive"], + }); + await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.ok(writes.length > 0, "expected a timestamp write"); + assert.equal( + writes.every(([key]) => key === K.workerWeightsSeen), + true, + `GC wrote outside the seen hash: ${JSON.stringify(writes)}`, + ); + }); + + it("does not rewrite a live node whose timestamp is still fresh", async () => { + const { db, writes } = fakeDb({ + weights: { alive: weight("alive", 200) }, + seen: { alive: ago(60) }, + live: ["alive"], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.touched, []); + assert.equal(writes.length, 0); + }); + + it("keeps a live node's weight even if its timestamp is ancient (mixed-version fleet)", async () => { + // Liveness comes from the heartbeat meta key, which every build writes, so + // a node on an older image is refreshed rather than deleted. + const { db, hash } = fakeDb({ + weights: { oldBuild: weight("oldBuild", 250) }, + seen: { oldBuild: ago(999999) }, + live: ["oldBuild"], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, []); + assert.deepEqual(r.touched, ["oldBuild"]); + assert.ok("oldBuild" in hash); + }); + + it("handles a mixed fleet in one pass", async () => { + const { db, hash } = fakeDb({ + weights: { + alive: weight("alive", 200), + restarting: weight("restarting", 150), + dead: weight("dead", 50), + fencedNode: weight("fencedNode", 0), + }, + seen: { + alive: ago(5000), + restarting: ago(120), + dead: ago(7200), + fencedNode: ago(7200), + }, + live: ["alive"], + fenced: ["fencedNode"], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, ["dead"]); + assert.deepEqual(r.touched, ["alive"]); + assert.equal(r.pending, 1); + assert.deepEqual(Object.keys(hash).sort(), [ + "alive", + "fencedNode", + "restarting", + ]); + }); + + it("treats an unparseable timestamp as expired", async () => { + const { db, hash } = fakeDb({ + weights: { + broken: { + workerId: "broken", + percent: 300, + updatedAt: "not-a-date", + byUserId: null, + byUsername: null, + }, + }, + seen: { broken: "also-not-a-date" }, + live: [], + }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, ["broken"]); + assert.equal("broken" in hash, false); + }); + + it("falls back to updatedAt when no timestamp was recorded", async () => { + const legacy: WorkerWeight = { + workerId: "legacy", + percent: 300, + updatedAt: ago(7200), + byUserId: null, + byUsername: null, + }; + const { db } = fakeDb({ weights: { legacy }, live: [] }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, ["legacy"]); + }); + + it("keeps a just-set weight for a node that has never been online", async () => { + const fresh: WorkerWeight = { + workerId: "notYet", + percent: 300, + updatedAt: ago(30), + byUserId: null, + byUsername: null, + }; + const { db } = fakeDb({ weights: { notYet: fresh }, live: [] }); + const r = await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(r.removed, []); + assert.equal(r.pending, 1); + }); + + it("drops orphaned timestamps whose weight was cleared elsewhere", async () => { + const { db, seenHash } = fakeDb({ + weights: { alive: weight("alive", 200) }, + seen: { alive: ago(60), ghost: ago(60) }, + live: ["alive"], + }); + await gcWorkerWeights(db, { graceSec: 3600, now: NOW }); + assert.deepEqual(Object.keys(seenHash), ["alive"]); + }); + + it("force ignores the grace period but still spares live and fenced nodes", async () => { + const { db, hash } = fakeDb({ + weights: { + alive: weight("alive", 200), + justGone: weight("justGone", 150), + fencedNode: weight("fencedNode", 0), + }, + seen: { alive: ago(10), justGone: ago(5), fencedNode: ago(5) }, + live: ["alive"], + fenced: ["fencedNode"], + }); + const r = await gcWorkerWeights(db, { force: true, now: NOW }); + assert.deepEqual(r.removed, ["justGone"]); + assert.deepEqual(Object.keys(hash).sort(), ["alive", "fencedNode"]); + }); + + it("pruneWorkerWeights removes absent nodes without waiting", async () => { + const { db } = fakeDb({ + weights: { + alive: weight("alive", 200), + justGone: weight("justGone", 150), + }, + seen: { alive: ago(10), justGone: ago(5) }, + live: ["alive"], + }); + assert.deepEqual(await pruneWorkerWeights(db), ["justGone"]); + }); + + it("clamps an absurdly small grace period instead of expiring everything", async () => { + const { db } = fakeDb({ + weights: { recent: weight("recent", 300) }, + seen: { recent: ago(30) }, + live: [], + }); + const r = await gcWorkerWeights(db, { graceSec: 0, now: NOW }); + // graceSec floors at 60s, so a node gone 30s is still pending + assert.deepEqual(r.removed, []); + assert.equal(r.pending, 1); + }); + + it("uses a sane default grace period", () => { + assert.equal(DEFAULT_WORKER_WEIGHT_TTL_SEC, 3600); + }); +}); + +describe("weight reads surface Redis failures", () => { + function failingDb(failOn: string) { + return { + redis: { + async hgetall(key: string) { + if (key === failOn) throw new Error("ECONNRESET"); + return {}; + }, + async smembers() { + return []; + }, + pipeline() { + const api = { + hset: () => api, + hdel: () => api, + async exec() { + return []; + }, + }; + return api; + }, + async hdel() { + return 0; + }, + }, + async existsMany(keys: string[]) { + return keys.map(() => false); + }, + } as unknown as RedisStore; + } + + it("listWorkerWeights rejects instead of reporting 'no overrides'", async () => { + // An empty object is a real instruction (nothing is weighted). Returning + // it for a failed read would silently un-drain a 0% node. + await assert.rejects( + () => listWorkerWeights(failingDb(K.workerWeights)), + /ECONNRESET/, + ); + }); + + it("listWorkerWeightSeen rejects instead of reporting 'never seen'", async () => { + await assert.rejects( + () => listWorkerWeightSeen(failingDb(K.workerWeightsSeen)), + /ECONNRESET/, + ); + }); + + it("gcWorkerWeights aborts rather than expiring nodes on a failed read", async () => { + // The GC deletes based on these timestamps; a partial read must not be + // mistaken for "this node was never seen". + await assert.rejects( + () => + gcWorkerWeights(failingDb(K.workerWeightsSeen), { + weights: { dead: weight("dead", 300) }, + graceSec: 3600, + now: NOW, + }), + /ECONNRESET/, + ); + }); +}); diff --git a/packages/db/src/worker-weight.test.ts b/packages/db/src/worker-weight.test.ts new file mode 100644 index 0000000..eb639a1 --- /dev/null +++ b/packages/db/src/worker-weight.test.ts @@ -0,0 +1,544 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_WORKER_WEIGHT, + MAX_WORKER_WEIGHT, + annotateTargetShares, + computeWeightedShedCount, + computeWeightedTargets, + planNodeLoad, + shedAboveTarget, + hasWorkerWeightOverrides, + normalizeWorkerWeight, + parseWorkerWeightInput, + weightedShare, + type FleetNodeView, + type WorkerWeight, +} from "./worker-fleet.js"; + +function node( + id: string, + over: Partial = {}, +): FleetNodeView { + return { + id, + hostname: id, + pid: 1, + maxBots: 500, + botCount: 0, + startedAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + role: "all", + leasedBotIds: [], + leasedCount: 0, + isSelf: false, + online: true, + fenced: false, + weight: DEFAULT_WORKER_WEIGHT, + weightOverride: false, + targetShare: null, + ...over, + }; +} + +function weight(workerId: string, percent: number): WorkerWeight { + return { + workerId, + percent, + updatedAt: "2026-01-01T00:00:00.000Z", + byUserId: null, + byUsername: null, + }; +} + +describe("normalizeWorkerWeight", () => { + it("defaults when not a number", () => { + assert.equal(normalizeWorkerWeight(undefined), DEFAULT_WORKER_WEIGHT); + assert.equal(normalizeWorkerWeight("abc"), DEFAULT_WORKER_WEIGHT); + assert.equal(normalizeWorkerWeight(NaN), DEFAULT_WORKER_WEIGHT); + }); + + it("clamps to the supported range and rounds", () => { + assert.equal(normalizeWorkerWeight(-40), 0); + assert.equal(normalizeWorkerWeight(10_000), MAX_WORKER_WEIGHT); + assert.equal(normalizeWorkerWeight("150"), 150); + assert.equal(normalizeWorkerWeight(72.6), 73); + }); +}); + +describe("parseWorkerWeightInput", () => { + it("accepts numbers and numeric strings", () => { + assert.deepEqual(parseWorkerWeightInput(0), { ok: true, value: 0 }); + assert.deepEqual(parseWorkerWeightInput(250), { ok: true, value: 250 }); + assert.deepEqual(parseWorkerWeightInput("150"), { ok: true, value: 150 }); + assert.deepEqual(parseWorkerWeightInput(" 150 "), { ok: true, value: 150 }); + assert.deepEqual(parseWorkerWeightInput(72.6), { ok: true, value: 73 }); + }); + + it("rejects values Number() would silently turn into 0 (a full drain)", () => { + // Number(" ") === 0, Number([]) === 0, Number(false) === 0 — none of these + // may be read as "drain this node". + for (const bad of [" ", "\t\n", [], false, true, {}, "abc", "12abc", ""]) { + const r = parseWorkerWeightInput(bad); + assert.equal( + r.ok, + false, + `${JSON.stringify(bad)} must be rejected, got ${JSON.stringify(r)}`, + ); + } + }); + + it("rejects a missing value", () => { + assert.deepEqual(parseWorkerWeightInput(undefined), { + ok: false, + error: "weight_required", + }); + assert.deepEqual(parseWorkerWeightInput(null), { + ok: false, + error: "weight_required", + }); + }); + + it("rejects out-of-range instead of clamping, so the caller is told", () => { + assert.deepEqual(parseWorkerWeightInput(-1), { + ok: false, + error: "weight_out_of_range", + }); + assert.deepEqual(parseWorkerWeightInput(501), { + ok: false, + error: "weight_out_of_range", + }); + assert.deepEqual(parseWorkerWeightInput("9999"), { + ok: false, + error: "weight_out_of_range", + }); + }); + + it("rejects NaN and Infinity", () => { + assert.equal(parseWorkerWeightInput(NaN).ok, false); + assert.equal(parseWorkerWeightInput(Infinity).ok, false); + assert.equal(parseWorkerWeightInput(-Infinity).ok, false); + }); +}); + +describe("hasWorkerWeightOverrides", () => { + it("ignores entries that are still the default", () => { + assert.equal(hasWorkerWeightOverrides({}), false); + assert.equal( + hasWorkerWeightOverrides({ a: weight("a", DEFAULT_WORKER_WEIGHT) }), + false, + ); + assert.equal(hasWorkerWeightOverrides({ a: weight("a", 0) }), true); + assert.equal(hasWorkerWeightOverrides({ a: weight("a", 250) }), true); + }); +}); + +describe("weightedShare", () => { + it("splits by weight", () => { + assert.equal(weightedShare(300, 200, 300, 2), 200); + assert.equal(weightedShare(300, 100, 300, 2), 100); + }); + + it("falls back to an even split when every weight is zero", () => { + assert.equal(weightedShare(300, 0, 0, 3), 100); + }); + + it("is zero for an empty fleet total", () => { + assert.equal(weightedShare(0, 100, 200, 2), 0); + }); +}); + +describe("computeWeightedShedCount", () => { + it("matches the even split when nothing is weighted", () => { + // total 100 over 2 nodes → fair 50, slack 2 → shed 8 + assert.equal( + computeWeightedShedCount({ + localCount: 60, + peers: [{ count: 40 }], + slack: 2, + maxPerTick: 100, + }), + 8, + ); + }); + + it("lets a boosted peer pull work away", () => { + // total 100, local 100% vs peer 300% → local fair = 25, slack 2 → shed 33 + assert.equal( + computeWeightedShedCount({ + localCount: 60, + localWeight: 100, + peers: [{ count: 40, weight: 300 }], + slack: 2, + maxPerTick: 100, + }), + 33, + ); + }); + + it("keeps more on a boosted local node", () => { + // total 100, local 300% vs peer 100% → fair 75, slack 2 → no shed at 60 + assert.equal( + computeWeightedShedCount({ + localCount: 60, + localWeight: 300, + peers: [{ count: 40, weight: 100 }], + slack: 2, + maxPerTick: 100, + }), + 0, + ); + }); + + it("drains a 0% node all the way down, ignoring slack", () => { + assert.equal( + computeWeightedShedCount({ + localCount: 2, + localWeight: 0, + peers: [{ count: 40, weight: 100 }], + slack: 2, + maxPerTick: 100, + }), + 2, + ); + }); + + it("respects maxPerTick while draining", () => { + assert.equal( + computeWeightedShedCount({ + localCount: 500, + localWeight: 0, + peers: [{ count: 40, weight: 100 }], + slack: 2, + maxPerTick: 50, + }), + 50, + ); + }); + + it("does not shed when every node is drained (even split fallback)", () => { + // All zero → fall back to even split: fair 50 + slack 2, local 50 → 0 + assert.equal( + computeWeightedShedCount({ + localCount: 50, + localWeight: 0, + peers: [{ count: 50, weight: 0 }], + slack: 2, + maxPerTick: 100, + }), + 0, + ); + }); + + it("never sheds as the only node", () => { + assert.equal( + computeWeightedShedCount({ + localCount: 100, + localWeight: 0, + peers: [], + slack: 2, + }), + 0, + ); + }); +}); + +describe("computeWeightedTargets", () => { + it("splits evenly by default and loses nothing to flooring", () => { + const t = computeWeightedTargets( + [{ id: "a" }, { id: "b" }, { id: "c" }], + 100, + ); + assert.equal(t.a + t.b + t.c, 100); + assert.deepEqual([t.a, t.b, t.c].sort((x, y) => y - x), [34, 33, 33]); + }); + + it("splits by weight", () => { + const t = computeWeightedTargets( + [ + { id: "a", weight: 300 }, + { id: "b", weight: 100 }, + ], + 400, + ); + assert.deepEqual(t, { a: 300, b: 100 }); + }); + + it("gives a drained node nothing and its work to the rest", () => { + const t = computeWeightedTargets( + [ + { id: "a", weight: 0 }, + { id: "b", weight: 100 }, + { id: "c", weight: 100 }, + ], + 100, + ); + assert.deepEqual(t, { a: 0, b: 50, c: 50 }); + }); + + it("falls back to an even split when every node is drained", () => { + const t = computeWeightedTargets( + [ + { id: "a", weight: 0 }, + { id: "b", weight: 0 }, + ], + 40, + ); + assert.deepEqual(t, { a: 20, b: 20 }); + }); + + it("redistributes overflow from a capped node", () => { + // a wants 80 but caps at 10 → the other 70 land on b (uncapped) + const t = computeWeightedTargets( + [ + { id: "a", weight: 400, maxBots: 10 }, + { id: "b", weight: 100, maxBots: 500 }, + ], + 100, + ); + assert.deepEqual(t, { a: 10, b: 90 }); + }); + + it("cascades redistribution through several capped nodes", () => { + const t = computeWeightedTargets( + [ + { id: "a", maxBots: 5 }, + { id: "b", maxBots: 5 }, + { id: "c", maxBots: 500 }, + ], + 90, + ); + assert.deepEqual(t, { a: 5, b: 5, c: 80 }); + }); + + it("reports what the fleet cannot hold by allocating less than total", () => { + const t = computeWeightedTargets( + [ + { id: "a", maxBots: 5 }, + { id: "b", maxBots: 5 }, + ], + 90, + ); + assert.equal(t.a + t.b, 10); + }); + + it("is order-independent (all nodes agree on the same plan)", () => { + const nodes = [ + { id: "n3", weight: 150 }, + { id: "n1", weight: 100 }, + { id: "n2", weight: 100 }, + ]; + const a = computeWeightedTargets(nodes, 77); + const b = computeWeightedTargets([...nodes].reverse(), 77); + assert.deepEqual(a, b); + }); + + it("is zero for every node when nothing wants polling", () => { + assert.deepEqual(computeWeightedTargets([{ id: "a" }, { id: "b" }], 0), { + a: 0, + b: 0, + }); + }); +}); + +describe("planNodeLoad", () => { + const fleet = [ + { id: "a", weight: 300, maxBots: 500 }, + { id: "b", weight: 100, maxBots: 500 }, + ]; + + it("caps claiming at the planned target plus slack", () => { + const a = planNodeLoad({ selfId: "a", nodes: fleet, total: 400, slack: 2 }); + assert.equal(a.target, 300); + assert.equal(a.claimCap, 302); + const b = planNodeLoad({ selfId: "b", nodes: fleet, total: 400, slack: 2 }); + assert.equal(b.target, 100); + assert.equal(b.claimCap, 102); + }); + + it("never lets claim and shed disagree", () => { + // Fractional share: the claim cap must not exceed the shed threshold, or + // the node would grab a bot and release it every tick. + for (let total = 0; total < 60; total++) { + const plan = planNodeLoad({ + selfId: "b", + nodes: [ + { id: "a", weight: 150 }, + { id: "b", weight: 100 }, + { id: "c", weight: 100 }, + ], + total, + slack: 2, + }); + const shed = shedAboveTarget({ + localCount: plan.claimCap, + target: plan.target, + slack: plan.slack, + maxPerTick: 50, + }); + assert.equal(shed, 0, `total=${total} claims ${plan.claimCap} then sheds`); + } + }); + + it("drains a 0% node with no slack", () => { + const plan = planNodeLoad({ + selfId: "z", + nodes: [ + { id: "z", weight: 0 }, + { id: "a", weight: 100 }, + ], + total: 400, + slack: 2, + }); + assert.equal(plan.target, 0); + assert.equal(plan.claimCap, 0); + assert.equal(plan.slack, 0); + assert.equal(plan.drained, true); + }); + + it("keeps slack when the whole fleet is drained (not a real drain)", () => { + const plan = planNodeLoad({ + selfId: "z", + nodes: [ + { id: "z", weight: 0 }, + { id: "a", weight: 0 }, + ], + total: 40, + slack: 2, + }); + assert.equal(plan.drained, false); + assert.equal(plan.target, 20); + assert.equal(plan.claimCap, 22); + }); + + it("gives a lone node everything regardless of its weight", () => { + const plan = planNodeLoad({ + selfId: "a", + nodes: [{ id: "a", weight: 20 }], + total: 90, + slack: 0, + }); + assert.equal(plan.target, 90); + }); + + it("reports bots no node has room for", () => { + const plan = planNodeLoad({ + selfId: "a", + nodes: [ + { id: "a", maxBots: 5 }, + { id: "b", maxBots: 5 }, + ], + total: 90, + slack: 2, + }); + assert.equal(plan.target, 5); + assert.equal(plan.unplaceable, 80); + }); + + it("has nothing unplaceable in a healthy fleet", () => { + const plan = planNodeLoad({ selfId: "a", nodes: fleet, total: 400 }); + assert.equal(plan.unplaceable, 0); + }); +}); + +describe("shedAboveTarget", () => { + it("sheds down to target + slack", () => { + assert.equal( + shedAboveTarget({ localCount: 60, target: 50, slack: 2, maxPerTick: 100 }), + 8, + ); + }); + + it("does not shed at or below the threshold", () => { + assert.equal(shedAboveTarget({ localCount: 52, target: 50, slack: 2 }), 0); + assert.equal(shedAboveTarget({ localCount: 10, target: 50, slack: 2 }), 0); + }); + + it("respects maxPerTick", () => { + assert.equal( + shedAboveTarget({ localCount: 500, target: 0, slack: 0, maxPerTick: 50 }), + 50, + ); + }); +}); + +describe("annotateTargetShares", () => { + it("splits evenly by default and loses no bots to flooring", () => { + const nodes = [node("a"), node("b"), node("c")]; + annotateTargetShares(nodes, 100); + assert.equal( + nodes.reduce((acc, n) => acc + (n.targetShare ?? 0), 0), + 100, + ); + assert.deepEqual( + nodes.map((n) => n.targetShare).sort((x, y) => (y ?? 0) - (x ?? 0)), + [34, 33, 33], + ); + }); + + it("splits by weight", () => { + const nodes = [ + node("a", { weight: 300, weightOverride: true }), + node("b"), + ]; + annotateTargetShares(nodes, 400); + assert.equal(nodes[0]!.targetShare, 300); + assert.equal(nodes[1]!.targetShare, 100); + }); + + it("gives a drained node nothing", () => { + const nodes = [node("a", { weight: 0, weightOverride: true }), node("b")]; + annotateTargetShares(nodes, 50); + assert.equal(nodes[0]!.targetShare, 0); + assert.equal(nodes[1]!.targetShare, 50); + }); + + it("caps a share at maxBots and moves the overflow to the others", () => { + const nodes = [ + node("a", { weight: 400, weightOverride: true, maxBots: 10 }), + node("b", { maxBots: 500 }), + ]; + annotateTargetShares(nodes, 100); + assert.equal(nodes[0]!.targetShare, 10); + assert.equal(nodes[1]!.targetShare, 90); + }); + + it("leaves bots unassigned when the whole fleet is at its cap", () => { + const nodes = [node("a", { maxBots: 5 }), node("b", { maxBots: 5 })]; + annotateTargetShares(nodes, 90); + assert.equal(nodes[0]!.targetShare, 5); + assert.equal(nodes[1]!.targetShare, 5); + }); + + it("skips offline and fenced nodes", () => { + const nodes = [ + node("a"), + node("b", { online: false }), + node("c", { fenced: true, online: false }), + ]; + annotateTargetShares(nodes, 30); + assert.equal(nodes[0]!.targetShare, 30); + assert.equal(nodes[1]!.targetShare, null); + assert.equal(nodes[2]!.targetShare, null); + }); + + it("defaults to the leased sum when no total is supplied", () => { + const nodes = [ + node("a", { leasedCount: 30 }), + node("b", { leasedCount: 10 }), + ]; + annotateTargetShares(nodes); + assert.equal(nodes[0]!.targetShare, 20); + assert.equal(nodes[1]!.targetShare, 20); + }); + + it("falls back to an even split when the whole fleet is drained", () => { + const nodes = [ + node("a", { weight: 0, weightOverride: true }), + node("b", { weight: 0, weightOverride: true }), + ]; + annotateTargetShares(nodes, 40); + assert.equal(nodes[0]!.targetShare, 20); + assert.equal(nodes[1]!.targetShare, 20); + }); +}); diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json new file mode 100644 index 0000000..a013e0c --- /dev/null +++ b/packages/db/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/packages/ilink/package.json b/packages/ilink/package.json new file mode 100644 index 0000000..baf22e4 --- /dev/null +++ b/packages/ilink/package.json @@ -0,0 +1,25 @@ +{ + "name": "@wechat-ai/ilink", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --import tsx --test src/**/*.test.ts" + }, + "devDependencies": { + "@types/node": "^22.13.10", + "tsx": "^4.19.3", + "typescript": "^5.8.2" + } +} diff --git a/packages/ilink/src/client.ts b/packages/ilink/src/client.ts new file mode 100644 index 0000000..ac9109a --- /dev/null +++ b/packages/ilink/src/client.ts @@ -0,0 +1,988 @@ +import { buildILinkHeaders } from "./headers.js"; +import { + aesEcbPaddedSize, + buildCdnDownloadUrl, + buildCdnUploadUrl, + decryptAes128Ecb, + encodeAesKeyField, + encryptAes128Ecb, + md5Hex, + parseAesKey, + randomAesKey, + randomFileKey, +} from "./crypto.js"; +import { isAllowedMediaUrl, sniffMediaMime } from "./media.js"; +import type { + DownloadedMedia, + GetConfigResponse, + GetUpdatesResponse, + GetUploadUrlResponse, + ILinkClientOptions, + InboundMediaRef, + QrcodeResponse, + QrcodeStatusResponse, + SendMessageResponse, + TypingStatus, + UploadMediaType, + UploadedMedia, + WeixinItemType, + WeixinMessage, +} from "./types.js"; +import { ITEM_TYPE, UPLOAD_MEDIA_TYPE } from "./types.js"; + +const DEFAULT_BASE = "https://ilinkai.weixin.qq.com"; +const DEFAULT_CDN = "https://novac2c.cdn.weixin.qq.com/c2c"; +/** Server-side typing_ticket validity is ~24h; refresh well before that. */ +const DEFAULT_TYPING_TICKET_TTL_MS = 20 * 60 * 60 * 1000; +const DEFAULT_TYPING_TICKET_MAX_ENTRIES = 5000; +const DEFAULT_MEDIA_MAX_BYTES = 12 * 1024 * 1024; + +interface TypingTicketEntry { + ticket: string; + expiresAt: number; +} + +export class ILinkError extends Error { + constructor( + message: string, + public readonly ret?: number, + public readonly errcode?: number, + public readonly body?: unknown, + ) { + super(message); + this.name = "ILinkError"; + } +} + +export class ILinkClient { + private baseUrl: string; + private botToken?: string; + private channelVersion: string; + private timeoutMs: number; + private longPollTimeoutMs: number; + private cdnBaseUrl: string; + /** typing_ticket per WeChat peer (getconfig is one extra RTT per peer/day) */ + private typingTickets = new Map(); + /** Collapses the burst of concurrent typing calls one reply produces */ + private typingTicketInflight = new Map>(); + private typingTicketTtlMs: number; + private typingTicketMaxEntries: number; + private mediaMaxBytes: number; + private mediaHostAllowlist: string[]; + + constructor(opts: ILinkClientOptions = {}) { + this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE).replace(/\/$/, ""); + this.botToken = opts.botToken; + this.channelVersion = opts.channelVersion ?? "1.0.2"; + this.timeoutMs = opts.timeoutMs ?? 30_000; + this.longPollTimeoutMs = opts.longPollTimeoutMs ?? 40_000; + this.cdnBaseUrl = (opts.cdnBaseUrl ?? DEFAULT_CDN).replace(/\/$/, ""); + this.typingTicketTtlMs = Math.max( + 60_000, + opts.typingTicketTtlMs ?? DEFAULT_TYPING_TICKET_TTL_MS, + ); + this.typingTicketMaxEntries = Math.max( + 64, + opts.typingTicketMaxEntries ?? DEFAULT_TYPING_TICKET_MAX_ENTRIES, + ); + this.mediaMaxBytes = Math.max( + 1024, + opts.mediaMaxBytes ?? DEFAULT_MEDIA_MAX_BYTES, + ); + this.mediaHostAllowlist = (opts.mediaHostAllowlist ?? []) + .map((h) => h.trim()) + .filter(Boolean); + } + + setBotToken(token: string): void { + this.botToken = token; + } + + getBotToken(): string | undefined { + return this.botToken; + } + + setBaseUrl(url: string): void { + this.baseUrl = url.replace(/\/$/, ""); + } + + setCdnBaseUrl(url: string): void { + this.cdnBaseUrl = url.replace(/\/$/, ""); + } + + getCdnBaseUrl(): string { + return this.cdnBaseUrl; + } + + /** GET login QR (bot_type=3). */ + async getBotQrcode(botType = 3): Promise { + const url = `${this.baseUrl}/ilink/bot/get_bot_qrcode?bot_type=${botType}`; + return this.getJson(url, false, this.timeoutMs); + } + + /** + * Poll QR scan status. Server may long-hold this request — use a long timeout + * and treat AbortError as "still waiting" by retrying at call site. + */ + async getQrcodeStatus(qrcode: string): Promise { + const url = `${this.baseUrl}/ilink/bot/get_qrcode_status?qrcode=${encodeURIComponent(qrcode)}`; + // Status endpoint often holds ~35–60s; 30s default was causing AbortError + return this.getJson(url, false, 90_000); + } + + /** + * Long-poll inbound messages. + * Pass previous get_updates_buf (empty string on first call). + */ + async getUpdates(getUpdatesBuf: string): Promise { + this.requireToken(); + return this.postJson( + "/ilink/bot/getupdates", + { + get_updates_buf: getUpdatesBuf, + base_info: { channel_version: this.channelVersion }, + }, + this.longPollTimeoutMs, + ); + } + + /** Send text reply; context_token from inbound message is required. */ + async sendText(params: { + toUserId: string; + text: string; + contextToken: string; + clientId?: string; + }): Promise { + this.requireToken(); + const clientId = + params.clientId ?? + `wechat-ai-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + + return this.postJson("/ilink/bot/sendmessage", { + msg: { + from_user_id: "", + to_user_id: params.toUserId, + client_id: clientId, + message_type: 2, + message_state: 2, + context_token: params.contextToken, + item_list: [{ type: 1, text_item: { text: params.text } }], + }, + }); + } + + /** + * Fetch the per-user `typing_ticket` that sendtyping requires. + * Ticket validity is ~24h, so this is roughly one call per peer per day. + */ + async getConfig(params: { + toUserId: string; + contextToken: string; + }): Promise { + this.requireToken(); + return this.postJson("/ilink/bot/getconfig", { + ilink_user_id: params.toUserId, + // Older captures name the same field to_user_id. Both are sent because + // the server ignores unknown keys — sendImage already relies on that with + // its extra top-level base_info — and this way either naming works. + to_user_id: params.toUserId, + context_token: params.contextToken, + base_info: { channel_version: this.channelVersion }, + }); + } + + /** Cached ticket for a peer, or null when absent/expired. Never hits network. */ + getCachedTypingTicket(toUserId: string): string | null { + const hit = this.typingTickets.get(toUserId); + if (!hit) return null; + if (hit.expiresAt <= Date.now()) { + this.typingTickets.delete(toUserId); + return null; + } + return hit.ticket; + } + + invalidateTypingTicket(toUserId: string): void { + this.typingTickets.delete(toUserId); + } + + /** + * Resolve a typing ticket, cache-first. + * + * Returns null instead of throwing when getconfig fails: the indicator is + * cosmetic and must never take down a reply. sendTyping then falls back to + * the ticket-less body shape. + */ + async getTypingTicket(params: { + toUserId: string; + contextToken: string; + force?: boolean; + }): Promise { + const key = params.toUserId; + if (!params.force) { + const cached = this.getCachedTypingTicket(key); + if (cached) return cached; + const inflight = this.typingTicketInflight.get(key); + if (inflight) return inflight; + } + + const pending = (async (): Promise => { + try { + const res = await this.getConfig({ + toUserId: params.toUserId, + contextToken: params.contextToken, + }); + const ticket = + typeof res.typing_ticket === "string" ? res.typing_ticket.trim() : ""; + if (!ticket) return null; + this.rememberTypingTicket(key, ticket); + return ticket; + } catch { + return null; + } + })(); + + this.typingTicketInflight.set(key, pending); + try { + return await pending; + } finally { + if (this.typingTicketInflight.get(key) === pending) { + this.typingTicketInflight.delete(key); + } + } + } + + /** + * Show ("对方正在输入中", status 1) or clear (status 2) the typing indicator. + * + * Two-step protocol: getconfig issues a per-user ticket, sendtyping carries it + * plus the status. Status defaults to 1 so existing call sites keep working. + * Stop calls never fetch a ticket — if none is cached we never started, so + * paying a getconfig round trip just to stop would be wasted latency. + */ + async sendTyping(params: { + toUserId: string; + contextToken: string; + /** 1 = start (default), 2 = stop */ + status?: TypingStatus; + /** Pre-resolved ticket; omit to use the per-peer cache */ + typingTicket?: string; + }): Promise { + this.requireToken(); + const status: TypingStatus = params.status ?? 1; + const explicit = params.typingTicket?.trim(); + + let ticket: string | null = explicit ?? null; + if (!ticket) { + ticket = + status === 2 + ? this.getCachedTypingTicket(params.toUserId) + : await this.getTypingTicket({ + toUserId: params.toUserId, + contextToken: params.contextToken, + }); + } + const fromCache = !explicit && Boolean(ticket); + + try { + return await this.postTyping( + params.toUserId, + params.contextToken, + status, + ticket, + ); + } catch (err) { + // A cached ticket can be revoked or expire early server-side. One forced + // refresh and retry, then rethrow — callers treat typing as best effort. + if (!fromCache) throw err; + this.invalidateTypingTicket(params.toUserId); + const fresh = await this.getTypingTicket({ + toUserId: params.toUserId, + contextToken: params.contextToken, + force: true, + }); + if (!fresh || fresh === ticket) throw err; + return this.postTyping( + params.toUserId, + params.contextToken, + status, + fresh, + ); + } + } + + /** Show the "对方正在输入中" indicator. */ + async startTyping(params: { + toUserId: string; + contextToken: string; + typingTicket?: string; + }): Promise { + return this.sendTyping({ ...params, status: 1 }); + } + + /** Clear the indicator (call once the reply has been sent). */ + async stopTyping(params: { + toUserId: string; + contextToken: string; + typingTicket?: string; + }): Promise { + return this.sendTyping({ ...params, status: 2 }); + } + + private postTyping( + toUserId: string, + contextToken: string, + status: TypingStatus, + ticket: string | null, + ): Promise { + const body: Record = { + ilink_user_id: toUserId, + to_user_id: toUserId, + context_token: contextToken, + status, + }; + if (ticket) body.typing_ticket = ticket; + return this.postJson("/ilink/bot/sendtyping", body); + } + + private rememberTypingTicket(toUserId: string, ticket: string): void { + const now = Date.now(); + if (this.typingTickets.size >= this.typingTicketMaxEntries) { + for (const [k, v] of this.typingTickets) { + if (v.expiresAt <= now) this.typingTickets.delete(k); + } + // Still full: drop oldest insertions. Map preserves insertion order and + // this method always deletes before setting, so order is recency order. + let toDrop = this.typingTickets.size - this.typingTicketMaxEntries + 1; + if (toDrop > 0) { + for (const k of this.typingTickets.keys()) { + this.typingTickets.delete(k); + if (--toDrop <= 0) break; + } + } + } + this.typingTickets.delete(toUserId); + this.typingTickets.set(toUserId, { + ticket, + expiresAt: now + this.typingTicketTtlMs, + }); + } + + /** + * Request CDN upload credentials for a media blob. + * media_type: 1=image, 2=video, 3=file, 4=voice + */ + async getUploadUrl(params: { + filekey: string; + mediaType: UploadMediaType; + toUserId: string; + rawSize: number; + rawFileMd5: string; + fileSize: number; + aesKeyHex: string; + noNeedThumb?: boolean; + }): Promise { + this.requireToken(); + return this.postJson("/ilink/bot/getuploadurl", { + filekey: params.filekey, + media_type: params.mediaType, + to_user_id: params.toUserId, + rawsize: params.rawSize, + rawfilemd5: params.rawFileMd5, + filesize: params.fileSize, + aeskey: params.aesKeyHex, + no_need_thumb: params.noNeedThumb !== false, + base_info: { channel_version: this.channelVersion }, + }); + } + + /** + * Encrypt plaintext and POST to WeChat CDN. + * Returns download `encrypt_query_param` from response header `x-encrypted-param`. + */ + async uploadEncryptedToCdn(params: { + plaintext: Buffer; + aesKey: Buffer; + filekey: string; + uploadFullUrl?: string; + uploadParam?: string; + }): Promise { + const ciphertext = encryptAes128Ecb(params.plaintext, params.aesKey); + let url: string | undefined; + if (params.uploadFullUrl?.trim()) { + url = params.uploadFullUrl.trim(); + } else if (params.uploadParam) { + url = buildCdnUploadUrl( + this.cdnBaseUrl, + params.uploadParam, + params.filekey, + ); + } + if (!url) { + throw new ILinkError( + "CDN upload URL missing (need upload_full_url or upload_param)", + ); + } + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + try { + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/octet-stream" }, + body: new Uint8Array(ciphertext), + signal: ctrl.signal, + }); + if (!res.ok) { + const msg = + res.headers.get("x-error-message") || + (await res.text().catch(() => "")) || + `HTTP ${res.status}`; + throw new ILinkError(`CDN upload failed: ${msg}`, undefined, res.status); + } + const dl = res.headers.get("x-encrypted-param"); + if (!dl) { + throw new ILinkError("CDN upload response missing x-encrypted-param"); + } + return dl; + } catch (err) { + if (err instanceof ILinkError) throw err; + if (err instanceof Error && err.name === "AbortError") { + throw new ILinkError( + `CDN upload timed out after ${this.timeoutMs}ms`, + undefined, + undefined, + { aborted: true }, + ); + } + throw err; + } finally { + clearTimeout(timer); + } + } + + /** Upload raw bytes to WeChat CDN (image by default). */ + async uploadMedia(params: { + toUserId: string; + data: Buffer; + mediaType?: UploadMediaType; + }): Promise { + const mediaType = params.mediaType ?? 1; + const plaintext = params.data; + const rawSize = plaintext.length; + const rawFileMd5 = md5Hex(plaintext); + const fileSize = aesEcbPaddedSize(rawSize); + const filekey = randomFileKey(); + const aesKey = randomAesKey(); + + const upload = await this.getUploadUrl({ + filekey, + mediaType, + toUserId: params.toUserId, + rawSize, + rawFileMd5, + fileSize, + aesKeyHex: aesKey.toString("hex"), + }); + + const downloadEncryptedQueryParam = await this.uploadEncryptedToCdn({ + plaintext, + aesKey, + filekey, + uploadFullUrl: upload.upload_full_url, + uploadParam: upload.upload_param, + }); + + return { + filekey, + downloadEncryptedQueryParam, + aesKey, + rawSize, + cipherSize: fileSize, + }; + } + + /** + * Upload image bytes to WeChat CDN and send as IMAGE message. + * context_token from inbound message is required. + */ + async sendImage(params: { + toUserId: string; + contextToken: string; + image: Buffer; + clientId?: string; + }): Promise { + this.requireToken(); + if (!params.image?.length) { + throw new ILinkError("image buffer is empty"); + } + + const uploaded = await this.uploadMedia({ + toUserId: params.toUserId, + data: params.image, + mediaType: 1, + }); + + const clientId = + params.clientId ?? + `wechat-ai-img-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; + + return this.postJson("/ilink/bot/sendmessage", { + msg: { + from_user_id: "", + to_user_id: params.toUserId, + client_id: clientId, + message_type: 2, + message_state: 2, + context_token: params.contextToken, + item_list: [ + { + type: 2, + image_item: { + media: { + encrypt_query_param: uploaded.downloadEncryptedQueryParam, + aes_key: encodeAesKeyField(uploaded.aesKey), + encrypt_type: 1, + }, + mid_size: uploaded.cipherSize, + }, + }, + ], + }, + base_info: { channel_version: this.channelVersion }, + }); + } + + /** + * Upload bytes and send them as a non-image media message. + * + * `sendImage` is the verified reference shape for this envelope. Only `image` + * (item type 2) has been exercised against the live server; `voice` (item + * type 3) is confirmed on the *inbound* path only, and video/file item types + * are inferred — hence `itemType` / `itemField` are explicit parameters so ops + * can correct them without a code change once real captures exist. + */ + async sendMedia(params: { + toUserId: string; + contextToken: string; + data: Buffer; + /** getuploadurl media_type — see UPLOAD_MEDIA_TYPE */ + mediaType: UploadMediaType; + /** item_list entry type number — see ITEM_TYPE */ + itemType: WeixinItemType; + /** item_list entry field name, e.g. "voice_item" */ + itemField: string; + /** Extra fields merged into the item object (duration, file_name, …) */ + itemExtra?: Record; + clientId?: string; + }): Promise { + this.requireToken(); + if (!params.data?.length) { + throw new ILinkError(`${params.itemField} buffer is empty`); + } + if (params.data.length > this.mediaMaxBytes) { + throw new ILinkError( + `media too large: ${params.data.length} > ${this.mediaMaxBytes} bytes`, + ); + } + + const uploaded = await this.uploadMedia({ + toUserId: params.toUserId, + data: params.data, + mediaType: params.mediaType, + }); + + const clientId = + params.clientId ?? + `wechat-ai-${params.itemField}-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 10)}`; + + return this.postJson("/ilink/bot/sendmessage", { + msg: { + from_user_id: "", + to_user_id: params.toUserId, + client_id: clientId, + message_type: 2, + message_state: 2, + context_token: params.contextToken, + item_list: [ + { + type: params.itemType, + [params.itemField]: { + media: { + encrypt_query_param: uploaded.downloadEncryptedQueryParam, + aes_key: encodeAesKeyField(uploaded.aesKey), + encrypt_type: 1, + }, + mid_size: uploaded.cipherSize, + ...(params.itemExtra ?? {}), + }, + }, + ], + }, + base_info: { channel_version: this.channelVersion }, + }); + } + + /** + * Send a voice message. WeChat plays SILK/AMR; other containers may be + * rejected by the client even when the upload succeeds. + */ + async sendVoice(params: { + toUserId: string; + contextToken: string; + voice: Buffer; + /** Playback length in ms, when known — shown on the bubble */ + durationMs?: number; + itemType?: WeixinItemType; + clientId?: string; + }): Promise { + const itemExtra: Record = {}; + if (typeof params.durationMs === "number" && params.durationMs > 0) { + const ms = Math.round(params.durationMs); + itemExtra.duration_ms = ms; + itemExtra.voice_length = ms; + } + return this.sendMedia({ + toUserId: params.toUserId, + contextToken: params.contextToken, + data: params.voice, + mediaType: UPLOAD_MEDIA_TYPE.voice, + itemType: params.itemType ?? ITEM_TYPE.voice, + itemField: "voice_item", + itemExtra, + clientId: params.clientId, + }); + } + + /** Send a video. Item type 4 is inferred — override `itemType` if captures differ. */ + async sendVideo(params: { + toUserId: string; + contextToken: string; + video: Buffer; + durationMs?: number; + itemType?: WeixinItemType; + clientId?: string; + }): Promise { + const itemExtra: Record = {}; + if (typeof params.durationMs === "number" && params.durationMs > 0) { + itemExtra.duration_ms = Math.round(params.durationMs); + } + return this.sendMedia({ + toUserId: params.toUserId, + contextToken: params.contextToken, + data: params.video, + mediaType: UPLOAD_MEDIA_TYPE.video, + itemType: params.itemType ?? ITEM_TYPE.video, + itemField: "video_item", + itemExtra, + clientId: params.clientId, + }); + } + + /** Send a file. Item type 5 is inferred — override `itemType` if captures differ. */ + async sendFile(params: { + toUserId: string; + contextToken: string; + file: Buffer; + fileName: string; + itemType?: WeixinItemType; + clientId?: string; + }): Promise { + const name = params.fileName?.trim(); + if (!name) throw new ILinkError("fileName is required for sendFile"); + return this.sendMedia({ + toUserId: params.toUserId, + contextToken: params.contextToken, + data: params.file, + mediaType: UPLOAD_MEDIA_TYPE.file, + itemType: params.itemType ?? ITEM_TYPE.file, + itemField: "file_item", + itemExtra: { file_name: name, file_size: params.file.length }, + clientId: params.clientId, + }); + } + + /** + * Download and decrypt one inbound attachment from the WeChat CDN. + * + * The body is read with a running byte cap rather than `arrayBuffer()`, since + * Content-Length is advisory and buffering first would defeat the limit. + */ + async downloadMedia( + ref: InboundMediaRef, + opts: { maxBytes?: number } = {}, + ): Promise { + const maxBytes = Math.max(1024, opts.maxBytes ?? this.mediaMaxBytes); + if (ref.cipherSize && ref.cipherSize > maxBytes) { + throw new ILinkError( + `media too large: ${ref.cipherSize} > ${maxBytes} bytes`, + ); + } + + // `full_url` comes off an inbound message, so it is only honoured when it + // points at the CDN. Otherwise rebuild from our own base — see + // isAllowedMediaUrl for why an unchecked fetch here would be an SSRF. + const fullUrlOk = + Boolean(ref.fullUrl) && + isAllowedMediaUrl(ref.fullUrl!, { + cdnBaseUrl: this.cdnBaseUrl, + extraHosts: this.mediaHostAllowlist, + }); + if (ref.fullUrl && !fullUrlOk && !ref.encryptQueryParam) { + throw new ILinkError( + `media full_url host not allowed: ${ILinkClient.hostOf(ref.fullUrl)}`, + ); + } + const url = fullUrlOk + ? ref.fullUrl! + : ref.encryptQueryParam + ? buildCdnDownloadUrl(this.cdnBaseUrl, ref.encryptQueryParam) + : undefined; + if (!url) { + throw new ILinkError( + "media ref has neither full_url nor encrypt_query_param", + ); + } + + const raw = await this.fetchBounded(url, maxBytes); + + let data = raw; + if (ref.aesKey && ref.encryptType !== 0) { + try { + data = decryptAes128Ecb(raw, parseAesKey(ref.aesKey)); + } catch (err) { + // Some payloads carry a vestigial aes_key over plaintext bytes. Accept + // the undecrypted body only when it actually sniffs as media. + if (!sniffMediaMime(raw)) { + throw new ILinkError( + `media decrypt failed: ${(err as Error).message}`, + ); + } + data = raw; + } + } + + return { + kind: ref.kind, + data, + mime: sniffMediaMime(data), + fileName: ref.fileName, + }; + } + + private async fetchBounded(url: string, maxBytes: number): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + try { + const res = await fetch(url, { method: "GET", signal: ctrl.signal }); + if (!res.ok) { + throw new ILinkError( + `CDN download failed: HTTP ${res.status}`, + undefined, + res.status, + ); + } + const declared = Number(res.headers.get("content-length") ?? ""); + if (Number.isFinite(declared) && declared > maxBytes) { + throw new ILinkError( + `media too large: ${declared} > ${maxBytes} bytes`, + ); + } + + const body = res.body; + if (!body) { + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length > maxBytes) { + throw new ILinkError( + `media too large: ${buf.length} > ${maxBytes} bytes`, + ); + } + return buf; + } + + const reader = body.getReader(); + const chunks: Buffer[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel().catch(() => undefined); + throw new ILinkError(`media too large: exceeded ${maxBytes} bytes`); + } + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks); + } catch (err) { + if (err instanceof ILinkError) throw err; + if (err instanceof Error && err.name === "AbortError") { + throw new ILinkError( + `CDN download timed out after ${this.timeoutMs}ms`, + undefined, + undefined, + { aborted: true }, + ); + } + throw err; + } finally { + clearTimeout(timer); + } + } + + /** Host only, for an error message that must not echo a full attacker URL. */ + private static hostOf(raw: string): string { + try { + return new URL(raw).hostname || "(none)"; + } catch { + return "(unparseable)"; + } + } + + private requireToken(): void { + if (!this.botToken) { + throw new ILinkError("bot_token is required; complete QR login first"); + } + } + + private async getJson( + url: string, + auth: boolean, + timeoutMs = this.timeoutMs, + ): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const res = await fetch(url, { + method: "GET", + headers: buildILinkHeaders(auth ? this.botToken : undefined), + signal: ctrl.signal, + }); + const body = (await res.json()) as T & { + ret?: number; + errcode?: number; + errmsg?: string; + }; + if (!res.ok) { + throw new ILinkError( + `HTTP ${res.status}`, + body.ret, + body.errcode, + body, + ); + } + return body; + } catch (err) { + if (err instanceof Error && err.name === "AbortError") { + throw new ILinkError( + `request timed out after ${timeoutMs}ms`, + undefined, + undefined, + { aborted: true }, + ); + } + throw err; + } finally { + clearTimeout(timer); + } + } + + private async postJson( + path: string, + body: unknown, + timeoutMs = this.timeoutMs, + ): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const res = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: buildILinkHeaders(this.botToken), + body: JSON.stringify(body), + signal: ctrl.signal, + }); + const data = (await res.json()) as T & { + ret?: number; + errcode?: number; + errmsg?: string; + }; + if (!res.ok) { + throw new ILinkError( + `HTTP ${res.status}`, + data.ret, + data.errcode, + data, + ); + } + // Session expired often surfaces as ret/errcode -14 + if (data.ret !== undefined && data.ret !== 0) { + throw new ILinkError( + data.errmsg ?? `iLink ret=${data.ret}`, + data.ret, + data.errcode, + data, + ); + } + return data; + } finally { + clearTimeout(timer); + } + } +} + +export interface ExtractTextOptions { + /** + * Use the transcript WeChat/iLink produced for a voice message (default true). + * + * When false a voice note yields no text and is handled as unreadable media + * instead — the operator switch for deployments that would rather not act on + * WeChat's own speech-to-text. + */ + includeVoiceTranscript?: boolean; +} + +/** Extract plain text from a Weixin message (text + voice ASR if present). */ +export function extractText( + msg: WeixinMessage, + opts: ExtractTextOptions = {}, +): string | null { + const includeVoice = opts.includeVoiceTranscript !== false; + const items = msg.item_list ?? []; + const parts: string[] = []; + for (const item of items) { + if (item.type === 1 && item.text_item?.text) { + parts.push(item.text_item.text); + continue; + } + // Voice: some payloads include transcribed text under voice/audio items + if (item.type === 3) { + if (!includeVoice) continue; + const voice = item as { + voice_item?: { text?: string; voice_text?: string }; + text_item?: { text?: string }; + }; + const t = + voice.voice_item?.text || + voice.voice_item?.voice_text || + voice.text_item?.text; + if (t) parts.push(t); + } + } + return parts.length ? parts.join("\n") : null; +} + +/** True when message has only non-text media without usable transcript. */ +export function isMediaOnlyWithoutText( + msg: WeixinMessage, + opts: ExtractTextOptions = {}, +): boolean { + return !extractText(msg, opts) && (msg.item_list?.length ?? 0) > 0; +} + +/** User inbound messages are typically message_type === 1. */ +export function isUserInbound(msg: WeixinMessage): boolean { + return msg.message_type === 1; +} diff --git a/packages/ilink/src/crypto.test.ts b/packages/ilink/src/crypto.test.ts new file mode 100644 index 0000000..8b73519 --- /dev/null +++ b/packages/ilink/src/crypto.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + aesEcbPaddedSize, + decryptAes128Ecb, + encodeAesKeyField, + encryptAes128Ecb, + md5Hex, + parseAesKey, + randomAesKey, +} from "./crypto.js"; + +describe("ilink crypto", () => { + it("round-trips AES-128-ECB PKCS7", () => { + const key = randomAesKey(); + const plain = Buffer.from("hello wechat sticker 😊", "utf8"); + const cipher = encryptAes128Ecb(plain, key); + assert.equal(cipher.length, aesEcbPaddedSize(plain.length)); + assert.deepEqual(decryptAes128Ecb(cipher, key), plain); + }); + + it("pads empty buffer to 16 bytes", () => { + const key = randomAesKey(); + const cipher = encryptAes128Ecb(Buffer.alloc(0), key); + assert.equal(cipher.length, 16); + assert.deepEqual(decryptAes128Ecb(cipher, key), Buffer.alloc(0)); + }); + + it("md5Hex is stable", () => { + assert.equal(md5Hex(Buffer.from("abc")), "900150983cd24fb0d6963f7d28e17f72"); + }); + + it("encode/parse aes_key field (base64 of hex ascii)", () => { + const key = Buffer.from("00112233445566778899aabbccddeeff", "hex"); + const field = encodeAesKeyField(key); + assert.deepEqual(parseAesKey(field), key); + }); + + it("parseAesKey accepts raw 16-byte base64", () => { + const key = randomAesKey(); + const field = key.toString("base64"); + assert.deepEqual(parseAesKey(field), key); + }); + + it("parseAesKey accepts bare hex", () => { + const hex = "00112233445566778899aabbccddeeff"; + assert.deepEqual(parseAesKey(hex), Buffer.from(hex, "hex")); + }); +}); diff --git a/packages/ilink/src/crypto.ts b/packages/ilink/src/crypto.ts new file mode 100644 index 0000000..7043c66 --- /dev/null +++ b/packages/ilink/src/crypto.ts @@ -0,0 +1,89 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; + +/** Ciphertext size after AES-128-ECB PKCS7 padding (always +1..16 bytes). */ +export function aesEcbPaddedSize(plaintextSize: number): number { + return Math.ceil((plaintextSize + 1) / 16) * 16; +} + +export function encryptAes128Ecb(plaintext: Buffer, key: Buffer): Buffer { + if (key.length !== 16) { + throw new Error(`AES-128 key must be 16 bytes, got ${key.length}`); + } + const cipher = createCipheriv("aes-128-ecb", key, null); + cipher.setAutoPadding(true); + return Buffer.concat([cipher.update(plaintext), cipher.final()]); +} + +export function decryptAes128Ecb(ciphertext: Buffer, key: Buffer): Buffer { + if (key.length !== 16) { + throw new Error(`AES-128 key must be 16 bytes, got ${key.length}`); + } + const decipher = createDecipheriv("aes-128-ecb", key, null); + decipher.setAutoPadding(true); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]); +} + +export function md5Hex(buf: Buffer): string { + return createHash("md5").update(buf).digest("hex"); +} + +export function randomAesKey(): Buffer { + return randomBytes(16); +} + +export function randomFileKey(): string { + return randomBytes(16).toString("hex"); +} + +/** + * media.aes_key for outbound image items (openclaw / weixin-ilink style): + * base64( ASCII hex string of 16 raw key bytes ) = base64(32-char hex). + */ +export function encodeAesKeyField(aesKey: Buffer): string { + return Buffer.from(aesKey.toString("hex"), "utf8").toString("base64"); +} + +/** + * Decode CDNMedia.aes_key which may be: + * - base64(raw 16 bytes) + * - base64(hex ASCII 32 chars) + * - raw hex string (32 chars) + */ +export function parseAesKey(aesKeyField: string): Buffer { + const raw = aesKeyField.trim(); + if (/^[0-9a-fA-F]{32}$/.test(raw)) { + return Buffer.from(raw, "hex"); + } + const decoded = Buffer.from(raw, "base64"); + if (decoded.length === 16) return decoded; + if (decoded.length === 32) { + const asAscii = decoded.toString("ascii"); + if (/^[0-9a-fA-F]{32}$/.test(asAscii)) { + return Buffer.from(asAscii, "hex"); + } + } + throw new Error( + `aes_key must decode to 16 raw bytes or 32-char hex, got ${decoded.length} bytes`, + ); +} + +export function buildCdnUploadUrl( + cdnBaseUrl: string, + uploadParam: string, + filekey: string, +): string { + const base = cdnBaseUrl.replace(/\/$/, ""); + return ( + `${base}/upload` + + `?encrypted_query_param=${encodeURIComponent(uploadParam)}` + + `&filekey=${encodeURIComponent(filekey)}` + ); +} + +export function buildCdnDownloadUrl( + cdnBaseUrl: string, + encryptedQueryParam: string, +): string { + const base = cdnBaseUrl.replace(/\/$/, ""); + return `${base}/download?encrypted_query_param=${encodeURIComponent(encryptedQueryParam)}`; +} diff --git a/packages/ilink/src/extract-text.test.ts b/packages/ilink/src/extract-text.test.ts new file mode 100644 index 0000000..f87d3ce --- /dev/null +++ b/packages/ilink/src/extract-text.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { extractText, isMediaOnlyWithoutText } from "./client.js"; + +describe("extractText", () => { + it("reads text items", () => { + assert.equal( + extractText({ + item_list: [{ type: 1, text_item: { text: "你好" } }], + }), + "你好", + ); + }); + + it("reads voice transcript when present", () => { + assert.equal( + extractText({ + item_list: [ + { type: 3, voice_item: { text: "语音转写" } } as never, + ], + }), + "语音转写", + ); + }); + + it("detects media-only without text", () => { + const msg = { + item_list: [{ type: 2, image_item: {} } as never], + }; + assert.equal(extractText(msg), null); + assert.equal(isMediaOnlyWithoutText(msg), true); + }); + + describe("includeVoiceTranscript", () => { + const voiceMsg = { + item_list: [{ type: 3, voice_item: { text: "语音转写" } } as never], + }; + + it("uses the transcript by default", () => { + assert.equal(extractText(voiceMsg), "语音转写"); + assert.equal( + extractText(voiceMsg, { includeVoiceTranscript: true }), + "语音转写", + ); + }); + + it("ignores the transcript when disabled", () => { + assert.equal( + extractText(voiceMsg, { includeVoiceTranscript: false }), + null, + ); + // The voice note then counts as unreadable media, which is what makes the + // worker answer "didn't catch that" instead of chatting. + assert.equal( + isMediaOnlyWithoutText(voiceMsg, { includeVoiceTranscript: false }), + true, + ); + }); + + it("still reads real text items when transcripts are disabled", () => { + const mixed = { + item_list: [ + { type: 1 as const, text_item: { text: "看这个" } }, + { type: 3, voice_item: { text: "语音转写" } } as never, + ], + }; + assert.equal( + extractText(mixed, { includeVoiceTranscript: false }), + "看这个", + ); + assert.equal(extractText(mixed), "看这个\n语音转写"); + }); + }); +}); diff --git a/packages/ilink/src/headers.test.ts b/packages/ilink/src/headers.test.ts new file mode 100644 index 0000000..6ba27f8 --- /dev/null +++ b/packages/ilink/src/headers.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { buildILinkHeaders, randomWechatUinHeader } from "./headers.js"; + +describe("headers", () => { + it("generates base64 wechat uin", () => { + const h = randomWechatUinHeader(); + assert.ok(h.length > 0); + const decoded = Buffer.from(h, "base64").toString("utf8"); + assert.match(decoded, /^\d+$/); + }); + + it("includes bearer when token present", () => { + const h = buildILinkHeaders("tok"); + assert.equal(h.Authorization, "Bearer tok"); + assert.equal(h.AuthorizationType, "ilink_bot_token"); + }); +}); diff --git a/packages/ilink/src/headers.ts b/packages/ilink/src/headers.ts new file mode 100644 index 0000000..5b61c4c --- /dev/null +++ b/packages/ilink/src/headers.ts @@ -0,0 +1,19 @@ +import { randomInt } from "node:crypto"; + +/** Random X-WECHAT-UIN: uint32 decimal string, base64-encoded (per-request anti-replay). */ +export function randomWechatUinHeader(): string { + const n = randomInt(0, 0xffffffff); + return Buffer.from(String(n), "utf8").toString("base64"); +} + +export function buildILinkHeaders(botToken?: string): Record { + const headers: Record = { + "Content-Type": "application/json", + AuthorizationType: "ilink_bot_token", + "X-WECHAT-UIN": randomWechatUinHeader(), + }; + if (botToken) { + headers.Authorization = `Bearer ${botToken}`; + } + return headers; +} diff --git a/packages/ilink/src/index.ts b/packages/ilink/src/index.ts new file mode 100644 index 0000000..8d63116 --- /dev/null +++ b/packages/ilink/src/index.ts @@ -0,0 +1,31 @@ +export { + ILinkClient, + ILinkError, + extractText, + isMediaOnlyWithoutText, + isUserInbound, +} from "./client.js"; +export { + aesEcbPaddedSize, + buildCdnDownloadUrl, + buildCdnUploadUrl, + decryptAes128Ecb, + encodeAesKeyField, + encryptAes128Ecb, + md5Hex, + parseAesKey, + randomAesKey, + randomFileKey, +} from "./crypto.js"; +export { + extractMediaRefs, + isAllowedMediaUrl, + isVisionMime, + mediaKindLabel, + sniffMediaMime, +} from "./media.js"; +export { loginWithQrcode, resolveQrOpenUrl } from "./login.js"; +export type { LoginOptions, LoginResult } from "./login.js"; +export { buildILinkHeaders, randomWechatUinHeader } from "./headers.js"; +export { ITEM_TYPE, UPLOAD_MEDIA_TYPE } from "./types.js"; +export type * from "./types.js"; diff --git a/packages/ilink/src/login.ts b/packages/ilink/src/login.ts new file mode 100644 index 0000000..3e36bad --- /dev/null +++ b/packages/ilink/src/login.ts @@ -0,0 +1,146 @@ +import { ILinkClient, ILinkError } from "./client.js"; +import type { QrcodeResponse, QrcodeStatusResponse } from "./types.js"; + +export interface LoginResult { + botToken: string; + baseUrl?: string; + accountId?: string; + raw: QrcodeStatusResponse; +} + +export interface LoginOptions { + client?: ILinkClient; + botType?: number; + pollIntervalMs?: number; + timeoutMs?: number; + onQrcode?: (info: { + qrcode: string; + qrcodeImgContent?: string; + qrcodeUrl?: string; + }) => void; + onStatus?: (status: string, raw: QrcodeStatusResponse) => void; + signal?: AbortSignal; +} + +/** Normalize scan link for user / web UI. */ +export function resolveQrOpenUrl(qr: QrcodeResponse): string | undefined { + if (qr.qrcode_img_content?.startsWith("http")) { + return qr.qrcode_img_content; + } + if (qr.qrcode_url?.startsWith("http")) { + return qr.qrcode_url; + } + if (qr.qrcode) { + return `https://liteapp.weixin.qq.com/q/7GiQu1?qrcode=${encodeURIComponent(qr.qrcode)}&bot_type=3`; + } + return undefined; +} + +/** + * Interactive QR login against iLink. + * Returns bot_token when status is confirmed/scanned success. + */ +export async function loginWithQrcode( + opts: LoginOptions = {}, +): Promise { + const client = opts.client ?? new ILinkClient(); + const pollIntervalMs = opts.pollIntervalMs ?? 1500; + const timeoutMs = opts.timeoutMs ?? 5 * 60_000; + const started = Date.now(); + + const qr = await client.getBotQrcode(opts.botType ?? 3); + if (!qr.qrcode) { + throw new ILinkError( + qr.errmsg ?? "get_bot_qrcode missing qrcode field", + qr.ret, + undefined, + qr, + ); + } + + opts.onQrcode?.({ + qrcode: qr.qrcode, + qrcodeImgContent: qr.qrcode_img_content, + qrcodeUrl: resolveQrOpenUrl(qr) ?? qr.qrcode_url, + }); + + while (true) { + if (opts.signal?.aborted) { + throw new ILinkError("login aborted"); + } + if (Date.now() - started > timeoutMs) { + throw new ILinkError("login timed out waiting for QR scan"); + } + + let status: QrcodeStatusResponse; + try { + status = await client.getQrcodeStatus(qr.qrcode); + } catch (err) { + // Long-poll timeout / transient network — keep waiting + if (err instanceof ILinkError && (err.body as { aborted?: boolean })?.aborted) { + opts.onStatus?.("waiting", { status: "waiting" }); + continue; + } + if (err instanceof Error && /timed out|aborted|fetch failed|ECONNRESET/i.test(err.message)) { + opts.onStatus?.("retry", { status: "retry" }); + await sleep(pollIntervalMs, opts.signal); + continue; + } + throw err; + } + + const st = (status.status ?? "").toLowerCase(); + opts.onStatus?.(st || "unknown", status); + + if ( + st === "confirmed" || + st === "confirmed_login" || + st === "success" || + Boolean(status.bot_token) + ) { + if (!status.bot_token) { + throw new ILinkError( + "QR confirmed but bot_token missing", + status.ret, + undefined, + status, + ); + } + if (status.baseurl) { + client.setBaseUrl(status.baseurl); + } + client.setBotToken(status.bot_token); + return { + botToken: status.bot_token, + baseUrl: status.baseurl, + accountId: status.account_id ?? status.ilink_bot_id, + raw: status, + }; + } + + if (st === "expired" || st === "cancel" || st === "cancelled") { + throw new ILinkError(`QR login ${st}`, status.ret, undefined, status); + } + + // wait_scan / scanned / etc. — brief pause before next long-poll + await sleep(Math.min(pollIntervalMs, 500), opts.signal); + } +} + +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new ILinkError("login aborted")); + return; + } + const t = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(t); + reject(new ILinkError("login aborted")); + }, + { once: true }, + ); + }); +} diff --git a/packages/ilink/src/media.test.ts b/packages/ilink/src/media.test.ts new file mode 100644 index 0000000..83130e3 --- /dev/null +++ b/packages/ilink/src/media.test.ts @@ -0,0 +1,307 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + extractMediaRefs, + isAllowedMediaUrl, + isVisionMime, + mediaKindLabel, + sniffMediaMime, +} from "./media.js"; +import type { WeixinMessage } from "./types.js"; + +function bytes(...b: number[]): Buffer { + return Buffer.from(b); +} + +function withAscii(text: string, pad = 0): Buffer { + const head = Buffer.alloc(pad); + return Buffer.concat([head, Buffer.from(text, "latin1")]); +} + +describe("sniffMediaMime", () => { + it("identifies images", () => { + assert.equal( + sniffMediaMime(bytes(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)), + "image/png", + ); + assert.equal(sniffMediaMime(bytes(0xff, 0xd8, 0xff, 0xe0)), "image/jpeg"); + assert.equal(sniffMediaMime(withAscii("GIF89a....")), "image/gif"); + assert.equal( + sniffMediaMime(Buffer.concat([Buffer.from("RIFF____WEBPVP8 ")])), + "image/webp", + ); + assert.equal(sniffMediaMime(withAscii("BM______")), "image/bmp"); + }); + + it("identifies WeChat voice containers", () => { + // Bare SILK tag and the variant with one leading byte both appear in the wild + assert.equal(sniffMediaMime(withAscii("#!SILK_V3")), "audio/silk"); + assert.equal(sniffMediaMime(withAscii("#!SILK_V3", 1)), "audio/silk"); + assert.equal(sniffMediaMime(withAscii("#!AMR\n")), "audio/amr"); + }); + + it("splits ISO-BMFF into audio vs video by brand", () => { + assert.equal( + sniffMediaMime(Buffer.from("____ftypM4A ", "latin1")), + "audio/mp4", + ); + assert.equal( + sniffMediaMime(Buffer.from("____ftypisom", "latin1")), + "video/mp4", + ); + }); + + it("distinguishes RIFF/WAVE from RIFF/WEBP", () => { + assert.equal( + sniffMediaMime(Buffer.from("RIFF____WAVEfmt ", "latin1")), + "audio/wav", + ); + }); + + it("returns null for unknown or too-short buffers", () => { + assert.equal(sniffMediaMime(Buffer.alloc(0)), null); + assert.equal(sniffMediaMime(bytes(1, 2)), null); + assert.equal(sniffMediaMime(Buffer.from("not media at all")), null); + }); +}); + +describe("isVisionMime", () => { + it("accepts only what an OpenAI-compatible image_url can carry", () => { + for (const m of ["image/png", "image/jpeg", "image/gif", "image/webp"]) { + assert.equal(isVisionMime(m), true, m); + } + for (const m of ["image/bmp", "audio/silk", "video/mp4", null, undefined]) { + assert.equal(isVisionMime(m), false, String(m)); + } + }); +}); + +describe("extractMediaRefs", () => { + it("returns nothing for text-only messages", () => { + const msg: WeixinMessage = { + item_list: [{ type: 1, text_item: { text: "你好" } }], + }; + assert.deepEqual(extractMediaRefs(msg), []); + }); + + it("reads image CDN coordinates", () => { + const msg: WeixinMessage = { + item_list: [ + { + type: 2, + image_item: { + media: { + encrypt_query_param: "eqp-1", + aes_key: "a".repeat(32), + encrypt_type: 1, + }, + mid_size: 4096, + }, + }, + ], + }; + const refs = extractMediaRefs(msg); + assert.equal(refs.length, 1); + assert.equal(refs[0]!.kind, "image"); + assert.equal(refs[0]!.index, 0); + assert.equal(refs[0]!.encryptQueryParam, "eqp-1"); + assert.equal(refs[0]!.aesKey, "a".repeat(32)); + assert.equal(refs[0]!.encryptType, 1); + assert.equal(refs[0]!.cipherSize, 4096); + }); + + it("falls back to item-level aeskey and url", () => { + const msg: WeixinMessage = { + item_list: [ + { + type: 2, + image_item: { + aeskey: "b".repeat(32), + url: "https://cdn.example/img", + }, + }, + ], + }; + const refs = extractMediaRefs(msg); + assert.equal(refs.length, 1); + assert.equal(refs[0]!.aesKey, "b".repeat(32)); + assert.equal(refs[0]!.fullUrl, "https://cdn.example/img"); + }); + + it("carries the iLink voice transcript through", () => { + const msg: WeixinMessage = { + item_list: [ + { + type: 3, + voice_item: { + media: { encrypt_query_param: "eqp-v" }, + voice_text: " 语音转写 ", + }, + }, + ], + }; + const refs = extractMediaRefs(msg); + assert.equal(refs.length, 1); + assert.equal(refs[0]!.kind, "voice"); + assert.equal(refs[0]!.transcript, "语音转写"); + }); + + it("skips media-less items", () => { + const msg: WeixinMessage = { + item_list: [{ type: 2, image_item: {} }], + }; + assert.deepEqual(extractMediaRefs(msg), []); + }); + + it("probes sub-objects when the item type number is unexpected", () => { + // Item types 4/5 are inferred; a mis-numbered item that plainly carries a + // file_item must still be picked up rather than dropped on the number. + const msg: WeixinMessage = { + item_list: [ + { + type: 9 as never, + file_item: { + media: { encrypt_query_param: "eqp-f" }, + file_name: "report.pdf", + }, + }, + ], + }; + const refs = extractMediaRefs(msg); + assert.equal(refs.length, 1); + assert.equal(refs[0]!.kind, "file"); + assert.equal(refs[0]!.fileName, "report.pdf"); + }); + + it("keeps item order and index across a mixed message", () => { + const msg: WeixinMessage = { + item_list: [ + { type: 1, text_item: { text: "看这个" } }, + { type: 2, image_item: { media: { encrypt_query_param: "a" } } }, + { type: 2, image_item: { media: { encrypt_query_param: "b" } } }, + ], + }; + const refs = extractMediaRefs(msg); + assert.deepEqual( + refs.map((r) => [r.index, r.encryptQueryParam]), + [ + [1, "a"], + [2, "b"], + ], + ); + }); +}); + +describe("isAllowedMediaUrl", () => { + const CDN = "https://novac2c.cdn.weixin.qq.com/c2c"; + + it("allows the configured CDN base host", () => { + assert.equal( + isAllowedMediaUrl("https://novac2c.cdn.weixin.qq.com/c2c/download?x=1", { + cdnBaseUrl: CDN, + }), + true, + ); + }); + + it("allows other WeChat / QQ CDN subdomains", () => { + for (const u of [ + "https://mmbiz.qpic.weixin.qq.com/a.jpg", + "https://wx.qq.com/x", + "https://other.cdn.weixin.qq.com/y", + ]) { + assert.equal(isAllowedMediaUrl(u, { cdnBaseUrl: CDN }), true, u); + } + }); + + it("rejects an arbitrary external host", () => { + assert.equal( + isAllowedMediaUrl("https://evil.example/payload.png", { cdnBaseUrl: CDN }), + false, + ); + }); + + it("rejects loopback, private ranges and cloud metadata", () => { + // These are the SSRF targets that matter on a box that also talks to Redis. + for (const host of [ + "127.0.0.1", + "localhost", + "0.0.0.0", + "10.1.2.3", + "172.16.5.5", + "172.31.255.255", + "192.168.1.1", + "169.254.169.254", + "100.64.0.1", + "metadata.google.internal", + "redis.local", + "[::1]", + ]) { + assert.equal( + isAllowedMediaUrl(`http://${host}/x`, { cdnBaseUrl: CDN }), + false, + host, + ); + } + }); + + it("still allows public 172.x outside the private block", () => { + assert.equal( + isAllowedMediaUrl("http://172.32.0.1/x", { + cdnBaseUrl: CDN, + extraHosts: ["172.32.0.1"], + }), + true, + ); + assert.equal( + isAllowedMediaUrl("http://172.15.0.1/x", { extraHosts: ["172.15.0.1"] }), + true, + ); + }); + + it("rejects non-http schemes and embedded credentials", () => { + for (const u of [ + "file:///etc/passwd", + "gopher://novac2c.cdn.weixin.qq.com/x", + "data:image/png;base64,AAAA", + "https://user:pass@novac2c.cdn.weixin.qq.com/x", + ]) { + assert.equal(isAllowedMediaUrl(u, { cdnBaseUrl: CDN }), false, u); + } + }); + + it("rejects unparseable input", () => { + assert.equal(isAllowedMediaUrl("not a url", { cdnBaseUrl: CDN }), false); + assert.equal(isAllowedMediaUrl("", { cdnBaseUrl: CDN }), false); + }); + + it("honours an explicit extra host", () => { + assert.equal( + isAllowedMediaUrl("https://my-mirror.test/x", { + cdnBaseUrl: CDN, + extraHosts: [" My-Mirror.test "], + }), + true, + ); + }); + + it("survives a malformed configured base", () => { + assert.equal( + isAllowedMediaUrl("https://x.weixin.qq.com/a", { cdnBaseUrl: "::::" }), + true, + ); + assert.equal( + isAllowedMediaUrl("https://evil.example/a", { cdnBaseUrl: "::::" }), + false, + ); + }); +}); + +describe("mediaKindLabel", () => { + it("labels every kind", () => { + assert.equal(mediaKindLabel("image"), "图片"); + assert.equal(mediaKindLabel("voice"), "语音"); + assert.equal(mediaKindLabel("video"), "视频"); + assert.equal(mediaKindLabel("file"), "文件"); + }); +}); diff --git a/packages/ilink/src/media.ts b/packages/ilink/src/media.ts new file mode 100644 index 0000000..b456503 --- /dev/null +++ b/packages/ilink/src/media.ts @@ -0,0 +1,272 @@ +import type { + CDNMedia, + FileItem, + ImageItem, + InboundMediaRef, + MediaKind, + MessageItem, + VideoItem, + VoiceItem, + WeixinMessage, +} from "./types.js"; +import { ITEM_TYPE } from "./types.js"; + +/** + * Identify decrypted media from its magic bytes. + * + * This is identification, not validation — the upload-side gate for + * user-submitted images lives in apps/api sticker-security.ts and is + * deliberately much stricter. Here we only need to know what we just pulled off + * the WeChat CDN so we can decide whether a vision model can read it. + */ +export function sniffMediaMime(buf: Buffer): string | null { + if (!buf || buf.length < 4) return null; + + const startsWith = (bytes: number[], offset = 0): boolean => { + if (buf.length < offset + bytes.length) return false; + for (let i = 0; i < bytes.length; i++) { + if (buf[offset + i] !== bytes[i]) return false; + } + return true; + }; + const ascii = (offset: number, len: number): string => + buf.length >= offset + len + ? buf.subarray(offset, offset + len).toString("latin1") + : ""; + + // ── Images ── + if (startsWith([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { + return "image/png"; + } + if (startsWith([0xff, 0xd8, 0xff])) return "image/jpeg"; + if (ascii(0, 6) === "GIF87a" || ascii(0, 6) === "GIF89a") return "image/gif"; + if (ascii(0, 4) === "RIFF") { + const form = ascii(8, 4); + if (form === "WEBP") return "image/webp"; + if (form === "WAVE") return "audio/wav"; + } + if (ascii(0, 2) === "BM") return "image/bmp"; + + // ── Audio ── + // WeChat voice is SILK, sometimes with a single leading byte before the tag. + if (ascii(0, 6) === "#!SILK") return "audio/silk"; + if (ascii(1, 6) === "#!SILK") return "audio/silk"; + if (ascii(0, 5) === "#!AMR") return "audio/amr"; + if (ascii(0, 4) === "OggS") return "audio/ogg"; + if (ascii(0, 3) === "ID3") return "audio/mpeg"; + // MPEG audio frame sync (11 set bits) + if (buf.length >= 2 && buf[0] === 0xff && (buf[1]! & 0xe0) === 0xe0) { + return "audio/mpeg"; + } + + // ── ISO-BMFF container: brand decides audio vs video ── + if (ascii(4, 4) === "ftyp") { + const brand = ascii(8, 4); + if (brand === "M4A " || brand === "M4B ") return "audio/mp4"; + return "video/mp4"; + } + + // ── Other video / documents ── + if (startsWith([0x1a, 0x45, 0xdf, 0xa3])) return "video/webm"; + if (ascii(0, 5) === "%PDF-") return "application/pdf"; + if (startsWith([0x50, 0x4b, 0x03, 0x04])) return "application/zip"; + + return null; +} + +/** Mimes an OpenAI-compatible vision model can accept as an image_url data URI. */ +const VISION_MIMES = new Set([ + "image/png", + "image/jpeg", + "image/gif", + "image/webp", +]); + +export function isVisionMime(mime: string | null | undefined): boolean { + return Boolean(mime && VISION_MIMES.has(mime)); +} + +/** Pull the CDN coordinates off a media-bearing item, whichever shape it uses. */ +function readMedia( + item: ImageItem | VoiceItem | VideoItem | FileItem | undefined, +): { + media?: CDNMedia; + aesKey?: string; + fullUrl?: string; + cipherSize?: number; +} { + if (!item) return {}; + const media = item.media; + return { + media, + // media.aes_key is the modern field; item-level `aeskey` is the older shape + // and some payloads only carry that one. + aesKey: media?.aes_key ?? item.aeskey, + fullUrl: media?.full_url ?? item.url, + cipherSize: item.mid_size, + }; +} + +function refFromItem( + item: MessageItem, + index: number, + kind: MediaKind, +): InboundMediaRef | null { + const sub = + kind === "image" + ? item.image_item + : kind === "voice" + ? item.voice_item + : kind === "video" + ? item.video_item + : item.file_item; + const { media, aesKey, fullUrl, cipherSize } = readMedia(sub); + const encryptQueryParam = media?.encrypt_query_param; + // Nothing to fetch: neither an absolute URL nor CDN coordinates. + if (!encryptQueryParam && !fullUrl) return null; + + const transcript = + kind === "voice" + ? (item.voice_item?.text || item.voice_item?.voice_text || "").trim() || + undefined + : undefined; + + return { + kind, + index, + encryptQueryParam, + aesKey, + encryptType: media?.encrypt_type, + fullUrl, + cipherSize: + typeof cipherSize === "number" && cipherSize > 0 ? cipherSize : undefined, + fileName: + kind === "file" ? item.file_item?.file_name?.trim() || undefined : undefined, + transcript, + }; +} + +const ALL_KINDS: readonly MediaKind[] = ["image", "voice", "video", "file"]; + +const KIND_BY_ITEM_TYPE = new Map([ + [ITEM_TYPE.image, "image"], + [ITEM_TYPE.voice, "voice"], + [ITEM_TYPE.video, "video"], + [ITEM_TYPE.file, "file"], +]); + +/** + * Every downloadable attachment on an inbound message, in item order. + * + * The kind the item type claims is tried first, then the remaining kinds. Item + * types 4/5 are inferred, so a mis-numbered item that clearly carries a media + * sub-object must not be dropped on the type number alone. Each kind reads a + * distinct `*_item` field, so probing cannot mislabel a well-formed item. + */ +export function extractMediaRefs(msg: WeixinMessage): InboundMediaRef[] { + const out: InboundMediaRef[] = []; + const items = msg.item_list ?? []; + items.forEach((item, index) => { + if (!item || item.type === ITEM_TYPE.text) return; + const claimed = KIND_BY_ITEM_TYPE.get(item.type); + const order = claimed + ? [claimed, ...ALL_KINDS.filter((k) => k !== claimed)] + : ALL_KINDS; + for (const kind of order) { + const ref = refFromItem(item, index, kind); + if (ref) { + out.push(ref); + return; + } + } + }); + return out; +} + +/** + * Hosts an inbound `full_url` may point at, beyond the configured CDN base. + * + * Inbound items arrive from getupdates, so `image_item.url` / `media.full_url` + * are attacker-influenced in principle. Fetching them unchecked would be a + * server-side request forgery primitive against whatever this box can reach + * (Redis, cloud metadata, sibling nodes) — and because a fetched image is handed + * to a vision model whose description goes back to the sender, it would not even + * be blind. The chatflow http node and the HF tools gateway already gate + * outbound URLs this way; this keeps the media path consistent with them. + */ +const WECHAT_CDN_SUFFIXES = [".weixin.qq.com", ".qq.com"]; + +const BLOCKED_HOSTS = new Set([ + "localhost", + "localhost.localdomain", + "metadata", + "metadata.google.internal", +]); + +/** Literal private / loopback / link-local addresses, without a DNS lookup. */ +function isPrivateHostLiteral(hostname: string): boolean { + const h = hostname.toLowerCase().replace(/^\[|\]$/g, ""); + if (BLOCKED_HOSTS.has(h) || h.endsWith(".local")) return true; + if (h === "::1" || h === "::" || h.startsWith("fe80:") || h.startsWith("fc")) { + return true; + } + const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(h); + if (!v4) return false; + const [a, b] = [Number(v4[1]), Number(v4[2])]; + if (a === 10 || a === 127 || a === 0) return true; + if (a === 169 && b === 254) return true; // link-local / cloud metadata + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT + return false; +} + +/** + * Whether an inbound `full_url` is safe to fetch. + * + * Accepts http(s) only, no embedded credentials, and only the configured CDN + * host, a WeChat/QQ CDN subdomain, or an explicit extra host. Anything else + * falls back to rebuilding the URL from our own CDN base, which is always safe. + */ +export function isAllowedMediaUrl( + rawUrl: string, + opts: { cdnBaseUrl?: string; extraHosts?: readonly string[] } = {}, +): boolean { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + return false; + } + if (url.protocol !== "https:" && url.protocol !== "http:") return false; + if (url.username || url.password) return false; + + const host = url.hostname.toLowerCase(); + if (!host || isPrivateHostLiteral(host)) return false; + + let cdnHost = ""; + if (opts.cdnBaseUrl) { + try { + cdnHost = new URL(opts.cdnBaseUrl).hostname.toLowerCase(); + } catch { + /* ignore a malformed configured base */ + } + } + if (cdnHost && host === cdnHost) return true; + if (WECHAT_CDN_SUFFIXES.some((s) => host.endsWith(s))) return true; + return (opts.extraHosts ?? []).some((h) => host === h.trim().toLowerCase()); +} + +/** Human-facing label for a media kind (used in fallback replies / history). */ +export function mediaKindLabel(kind: MediaKind): string { + switch (kind) { + case "image": + return "图片"; + case "voice": + return "语音"; + case "video": + return "视频"; + case "file": + return "文件"; + } +} diff --git a/packages/ilink/src/send-media.test.ts b/packages/ilink/src/send-media.test.ts new file mode 100644 index 0000000..2f7d2af --- /dev/null +++ b/packages/ilink/src/send-media.test.ts @@ -0,0 +1,412 @@ +import assert from "node:assert/strict"; +import { afterEach, describe, it } from "node:test"; +import { ILinkClient } from "./client.js"; +import { encryptAes128Ecb, parseAesKey } from "./crypto.js"; +import { ITEM_TYPE, UPLOAD_MEDIA_TYPE } from "./types.js"; +import type { InboundMediaRef } from "./types.js"; + +interface Call { + url: string; + path: string; + body: Record | null; + raw: Uint8Array | null; +} + +let restoreFetch: (() => void) | null = null; + +/** + * Mocks the three hops a media send makes: getuploadurl (JSON), the CDN upload + * (octet-stream, answers with x-encrypted-param), and sendmessage (JSON). + */ +function installFetch( + opts: { + cdnDownload?: { body: Buffer; headers?: Record }; + } = {}, +): Call[] { + const calls: Call[] = []; + const original = globalThis.fetch; + restoreFetch = () => { + globalThis.fetch = original; + restoreFetch = null; + }; + + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + const url = String(input); + const path = new URL(url).pathname; + const isJson = typeof init?.body === "string"; + calls.push({ + url, + path, + body: isJson + ? (JSON.parse(String(init!.body)) as Record) + : null, + raw: isJson ? null : ((init?.body as Uint8Array) ?? null), + }); + + if (path.endsWith("/getuploadurl")) { + return json({ ret: 0, upload_full_url: "https://cdn.test/c2c/upload?x=1" }); + } + if (path.endsWith("/upload")) { + return { + ok: true, + status: 200, + headers: new Headers({ "x-encrypted-param": "dl-param-1" }), + json: async () => ({}), + text: async () => "", + } as unknown as Response; + } + if (path.endsWith("/download")) { + const d = opts.cdnDownload; + if (!d) return json({ ret: 0 }); + return { + ok: true, + status: 200, + headers: new Headers(d.headers ?? {}), + body: bufferToStream(d.body), + arrayBuffer: async () => d.body, + json: async () => ({}), + text: async () => "", + } as unknown as Response; + } + return json({ ret: 0 }); + }) as typeof globalThis.fetch; + + return calls; +} + +function json(body: unknown): Response { + return { + ok: true, + status: 200, + headers: new Headers(), + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response; +} + +function bufferToStream(buf: Buffer): ReadableStream { + // Two chunks, so the running byte cap is exercised mid-stream. + const mid = Math.max(1, Math.floor(buf.length / 2)); + const parts = [buf.subarray(0, mid), buf.subarray(mid)]; + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i >= parts.length) { + controller.close(); + return; + } + controller.enqueue(new Uint8Array(parts[i]!)); + i++; + }, + }); +} + +function client(overrides: Record = {}): ILinkClient { + return new ILinkClient({ + botToken: "tok", + baseUrl: "https://ilink.test", + cdnBaseUrl: "https://cdn.test/c2c", + ...overrides, + }); +} + +function sentItem(calls: Call[]): Record { + const send = calls.find((c) => c.path.endsWith("/sendmessage")); + assert.ok(send, "sendmessage was not called"); + const msg = send.body!.msg as { item_list: Array> }; + return msg.item_list[0]!; +} + +const peer = { toUserId: "peer-1", contextToken: "ctx-1" }; +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + +afterEach(() => { + restoreFetch?.(); +}); + +describe("sendVoice", () => { + it("uploads with media_type 4 and emits a voice_item", async () => { + const calls = installFetch(); + await client().sendVoice({ + ...peer, + voice: Buffer.from("#!SILK_V3 audio"), + durationMs: 2500, + }); + + const upload = calls.find((c) => c.path.endsWith("/getuploadurl"))!; + assert.equal(upload.body!.media_type, UPLOAD_MEDIA_TYPE.voice); + assert.equal(upload.body!.media_type, 4); + + const item = sentItem(calls); + assert.equal(item.type, ITEM_TYPE.voice); + assert.equal(item.type, 3); + const voice = item.voice_item as Record; + const media = voice.media as Record; + assert.equal(media.encrypt_query_param, "dl-param-1"); + assert.equal(media.encrypt_type, 1); + assert.equal(voice.duration_ms, 2500); + assert.equal(voice.voice_length, 2500); + assert.ok(typeof voice.mid_size === "number" && voice.mid_size > 0); + }); + + it("omits duration fields when unknown", async () => { + const calls = installFetch(); + await client().sendVoice({ ...peer, voice: Buffer.from("abc") }); + const voice = sentItem(calls).voice_item as Record; + assert.equal("duration_ms" in voice, false); + assert.equal("voice_length" in voice, false); + }); + + it("rejects an empty buffer before any network call", async () => { + const calls = installFetch(); + await assert.rejects( + () => client().sendVoice({ ...peer, voice: Buffer.alloc(0) }), + /buffer is empty/, + ); + assert.equal(calls.length, 0); + }); + + it("rejects oversized payloads before uploading", async () => { + const calls = installFetch(); + // mediaMaxBytes has a 1KB floor, so stay above it to exercise the check. + await assert.rejects( + () => + client({ mediaMaxBytes: 2048 }).sendVoice({ + ...peer, + voice: Buffer.alloc(4096), + }), + /media too large/, + ); + assert.equal(calls.length, 0); + }); +}); + +describe("sendVideo / sendFile", () => { + it("sendVideo uses media_type 2 and the inferred video item type", async () => { + const calls = installFetch(); + await client().sendVideo({ ...peer, video: Buffer.from("vid"), durationMs: 9 }); + assert.equal( + calls.find((c) => c.path.endsWith("/getuploadurl"))!.body!.media_type, + 2, + ); + const item = sentItem(calls); + assert.equal(item.type, ITEM_TYPE.video); + assert.ok(item.video_item); + }); + + it("sendFile uses media_type 3 and carries the file name", async () => { + const calls = installFetch(); + await client().sendFile({ + ...peer, + file: Buffer.from("hello"), + fileName: " notes.txt ", + }); + assert.equal( + calls.find((c) => c.path.endsWith("/getuploadurl"))!.body!.media_type, + 3, + ); + const file = sentItem(calls).file_item as Record; + assert.equal(file.file_name, "notes.txt"); + assert.equal(file.file_size, 5); + }); + + it("sendFile requires a name", async () => { + installFetch(); + await assert.rejects( + () => client().sendFile({ ...peer, file: Buffer.from("x"), fileName: " " }), + /fileName is required/, + ); + }); + + it("honours an item type override so ops can correct the inferred numbers", async () => { + const calls = installFetch(); + await client().sendVideo({ + ...peer, + video: Buffer.from("vid"), + itemType: 5, + }); + assert.equal(sentItem(calls).type, 5); + }); +}); + +describe("downloadMedia", () => { + const aesKeyHex = "0123456789abcdef0123456789abcdef"; + + it("decrypts CDN bytes and sniffs the mime", async () => { + const cipher = encryptAes128Ecb(PNG, parseAesKey(aesKeyHex)); + installFetch({ cdnDownload: { body: cipher } }); + + const ref: InboundMediaRef = { + kind: "image", + index: 0, + encryptQueryParam: "eqp", + aesKey: aesKeyHex, + encryptType: 1, + }; + const out = await client().downloadMedia(ref); + + assert.equal(out.kind, "image"); + assert.equal(out.mime, "image/png"); + assert.deepEqual(out.data, PNG); + }); + + it("builds the download URL from the CDN base and the encrypted param", async () => { + const cipher = encryptAes128Ecb(PNG, parseAesKey(aesKeyHex)); + const calls = installFetch({ cdnDownload: { body: cipher } }); + await client().downloadMedia({ + kind: "image", + index: 0, + encryptQueryParam: "a b&c", + aesKey: aesKeyHex, + }); + const dl = calls.find((c) => c.path.endsWith("/download"))!; + assert.ok(dl.url.startsWith("https://cdn.test/c2c/download?")); + assert.ok(dl.url.includes(encodeURIComponent("a b&c"))); + }); + + it("prefers an absolute full_url when it points at the CDN", async () => { + const calls = installFetch({ cdnDownload: { body: PNG } }); + await client().downloadMedia({ + kind: "image", + index: 0, + fullUrl: "https://novac2c.cdn.weixin.qq.com/c2c/download?z=1", + }); + const dl = calls.find((c) => c.path.endsWith("/download"))!; + assert.equal(dl.url, "https://novac2c.cdn.weixin.qq.com/c2c/download?z=1"); + }); + + it("ignores an off-CDN full_url and rebuilds from our own base", async () => { + // full_url arrives on an inbound message, so honouring it unchecked would + // be an SSRF primitive against whatever this box can reach. + const calls = installFetch({ cdnDownload: { body: PNG } }); + await client().downloadMedia({ + kind: "image", + index: 0, + fullUrl: "http://169.254.169.254/latest/meta-data/", + encryptQueryParam: "eqp", + }); + const dl = calls.find((c) => c.path.endsWith("/download"))!; + assert.ok( + dl.url.startsWith("https://cdn.test/c2c/download?"), + `should have rebuilt from the CDN base, got ${dl.url}`, + ); + assert.ok(!calls.some((c) => c.url.includes("169.254.169.254"))); + }); + + it("refuses an off-CDN full_url with no fallback rather than fetching it", async () => { + const calls = installFetch({ cdnDownload: { body: PNG } }); + await assert.rejects( + () => + client().downloadMedia({ + kind: "image", + index: 0, + fullUrl: "http://127.0.0.1:6379/", + }), + /full_url host not allowed/, + ); + assert.equal(calls.length, 0, "must not have issued any request"); + }); + + it("does not echo the rejected URL back, only its host", async () => { + installFetch(); + await assert.rejects( + () => + client().downloadMedia({ + kind: "image", + index: 0, + fullUrl: "http://10.0.0.9/secret-path?token=abc", + }), + (err: Error) => { + assert.match(err.message, /10\.0\.0\.9/); + assert.ok(!err.message.includes("secret-path")); + assert.ok(!err.message.includes("token=abc")); + return true; + }, + ); + }); + + it("honours an explicit mediaHostAllowlist", async () => { + const calls = installFetch({ cdnDownload: { body: PNG } }); + await client({ mediaHostAllowlist: ["mirror.test"] }).downloadMedia({ + kind: "image", + index: 0, + fullUrl: "https://mirror.test/download?z=1", + }); + assert.equal( + calls.find((c) => c.path.endsWith("/download"))!.url, + "https://mirror.test/download?z=1", + ); + }); + + it("accepts plaintext bytes when a vestigial aes_key fails to decrypt", async () => { + installFetch({ cdnDownload: { body: PNG } }); + const out = await client().downloadMedia({ + kind: "image", + index: 0, + encryptQueryParam: "eqp", + aesKey: aesKeyHex, + encryptType: 1, + }); + assert.equal(out.mime, "image/png"); + assert.deepEqual(out.data, PNG); + }); + + it("skips decryption when encrypt_type is 0", async () => { + installFetch({ cdnDownload: { body: PNG } }); + const out = await client().downloadMedia({ + kind: "image", + index: 0, + encryptQueryParam: "eqp", + aesKey: aesKeyHex, + encryptType: 0, + }); + assert.deepEqual(out.data, PNG); + }); + + it("rejects on the mid_size hint without fetching", async () => { + const calls = installFetch({ cdnDownload: { body: PNG } }); + await assert.rejects( + () => + client().downloadMedia( + { kind: "image", index: 0, encryptQueryParam: "eqp", cipherSize: 9999 }, + { maxBytes: 2048 }, + ), + /media too large/, + ); + assert.equal(calls.length, 0); + }); + + it("rejects on a declared Content-Length over the cap", async () => { + installFetch({ + cdnDownload: { body: PNG, headers: { "content-length": "9999" } }, + }); + await assert.rejects( + () => + client().downloadMedia( + { kind: "image", index: 0, encryptQueryParam: "eqp" }, + { maxBytes: 1024 }, + ), + /media too large/, + ); + }); + + it("aborts mid-stream when the body exceeds the cap despite no Content-Length", async () => { + installFetch({ cdnDownload: { body: Buffer.alloc(9000, 7) } }); + await assert.rejects( + () => + client().downloadMedia( + { kind: "image", index: 0, encryptQueryParam: "eqp" }, + { maxBytes: 2048 }, + ), + /exceeded 2048 bytes/, + ); + }); + + it("requires somewhere to fetch from", async () => { + installFetch(); + await assert.rejects( + () => client().downloadMedia({ kind: "image", index: 0 }), + /neither full_url nor encrypt_query_param/, + ); + }); +}); diff --git a/packages/ilink/src/types.ts b/packages/ilink/src/types.ts new file mode 100644 index 0000000..c6b8dff --- /dev/null +++ b/packages/ilink/src/types.ts @@ -0,0 +1,255 @@ +/** iLink Bot message item types (community reverse-engineering + plugin behavior). */ +export type WeixinItemType = 1 | 2 | 3 | 4 | 5; + +/** + * Message item type numbers. + * + * 1 (text) and 2 (image) are confirmed — both are constructed on the outbound + * path against the live server. 3 (voice) is confirmed inbound only: getupdates + * delivers voice as `{ type: 3, voice_item: {...} }` (see extractText and + * extract-text.test.ts). 4 and 5 are **inferred** and have never been observed; + * `ITEM_TYPE.video` / `ITEM_TYPE.file` are the send-side guess and can be + * overridden per call (see ILinkClient.sendMedia). + */ +export const ITEM_TYPE = { + text: 1, + image: 2, + voice: 3, + video: 4, + file: 5, +} as const satisfies Record; + +/** getuploadurl media_type */ +export type UploadMediaType = 1 | 2 | 3 | 4; // IMG | VID | FILE | VOICE + +/** getuploadurl media_type by kind (confirmed by the getuploadurl contract). */ +export const UPLOAD_MEDIA_TYPE = { + image: 1, + video: 2, + file: 3, + voice: 4, +} as const satisfies Record; + +export interface TextItem { + text?: string; +} + +export interface CDNMedia { + encrypt_query_param?: string; + aes_key?: string; + encrypt_type?: number; + full_url?: string; +} + +export interface ImageItem { + media?: CDNMedia; + aeskey?: string; + url?: string; + mid_size?: number; + thumb_size?: number; + thumb_height?: number; + thumb_width?: number; + hd_size?: number; +} + +/** + * Voice item. `text` / `voice_text` carry the transcript iLink produced on its + * side — when present we use it and never touch the audio, which matters + * because WeChat voice bytes are SILK/AMR and no OpenAI-compatible ASR endpoint + * accepts those containers. + */ +export interface VoiceItem { + media?: CDNMedia; + aeskey?: string; + url?: string; + text?: string; + voice_text?: string; + mid_size?: number; + voice_size?: number; + voice_length?: number; + duration_ms?: number; +} + +export interface VideoItem { + media?: CDNMedia; + aeskey?: string; + url?: string; + mid_size?: number; + thumb_size?: number; + duration_ms?: number; +} + +export interface FileItem { + media?: CDNMedia; + aeskey?: string; + url?: string; + file_name?: string; + file_size?: number; + mid_size?: number; +} + +export interface MessageItem { + type: WeixinItemType; + text_item?: TextItem; + image_item?: ImageItem; + voice_item?: VoiceItem; + video_item?: VideoItem; + file_item?: FileItem; + // other media fields optional + [key: string]: unknown; +} + +/** Media kinds we can pull off an inbound message. */ +export type MediaKind = "image" | "voice" | "video" | "file"; + +/** + * One downloadable attachment on an inbound message. + * `encryptQueryParam` + `aesKey` are what the WeChat CDN needs; `fullUrl` wins + * when the item already carries an absolute URL. + */ +export interface InboundMediaRef { + kind: MediaKind; + /** Index in the original `item_list` (stable id within one message) */ + index: number; + encryptQueryParam?: string; + aesKey?: string; + encryptType?: number; + fullUrl?: string; + /** Ciphertext size hint from the item (mid_size); used to reject early */ + cipherSize?: number; + fileName?: string; + /** iLink-side transcript (voice only) */ + transcript?: string; +} + +/** Decrypted inbound media plus the mime we sniffed from the bytes. */ +export interface DownloadedMedia { + kind: MediaKind; + data: Buffer; + /** Sniffed from magic bytes; null when unrecognized */ + mime: string | null; + fileName?: string; +} + +export interface WeixinMessage { + from_user_id?: string; + to_user_id?: string; + message_type?: number; + message_state?: number; + context_token?: string; + item_list?: MessageItem[]; + create_time_ms?: number; + group_id?: string; + [key: string]: unknown; +} + +export interface GetUpdatesResponse { + ret?: number; + errcode?: number; + errmsg?: string; + msgs?: WeixinMessage[]; + get_updates_buf?: string; + longpolling_timeout_ms?: number; +} + +export interface SendMessageResponse { + ret?: number; + errcode?: number; + errmsg?: string; + [key: string]: unknown; +} + +export interface GetUploadUrlResponse { + ret?: number; + errcode?: number; + errmsg?: string; + /** Preferred: full pre-signed upload URL */ + upload_full_url?: string; + /** Fallback: encrypted param for CDN base + /upload */ + upload_param?: string; + [key: string]: unknown; +} + +/** + * `1` starts the "对方正在输入中" indicator, `2` clears it. + * The server also expires it on its own, but an explicit stop is what makes the + * indicator disappear the moment the reply lands instead of lingering. + */ +export type TypingStatus = 1 | 2; + +/** + * `/ilink/bot/getconfig` — issues the `typing_ticket` required by sendtyping. + * The ticket is per WeChat user and stays valid ~24h. + */ +export interface GetConfigResponse { + ret?: number; + errcode?: number; + errmsg?: string; + typing_ticket?: string; + [key: string]: unknown; +} + +export interface QrcodeResponse { + qrcode?: string; + qrcode_img_content?: string; + qrcode_url?: string; + ret?: number; + errmsg?: string; + [key: string]: unknown; +} + +export interface QrcodeStatusResponse { + status?: string; + bot_token?: string; + baseurl?: string; + account_id?: string; + ilink_bot_id?: string; + ret?: number; + errmsg?: string; + [key: string]: unknown; +} + +export interface ILinkClientOptions { + /** Default https://ilinkai.weixin.qq.com */ + baseUrl?: string; + /** Bot token after QR login */ + botToken?: string; + /** channel_version sent in base_info */ + channelVersion?: string; + /** fetch timeout for non-long-poll requests (ms) */ + timeoutMs?: number; + /** long-poll timeout (ms); server holds ~35s */ + longPollTimeoutMs?: number; + /** + * WeChat media CDN base (fallback when getuploadurl omits upload_full_url). + * Default https://novac2c.cdn.weixin.qq.com/c2c + */ + cdnBaseUrl?: string; + /** + * How long a cached `typing_ticket` is reused. Server-side validity is ~24h; + * default 20h leaves headroom so we refresh before the server expires it. + */ + typingTicketTtlMs?: number; + /** + * Cap on cached typing tickets (one per WeChat peer). A worker can lease + * hundreds of bots with many peers each, so the map is bounded like + * rate-limit.ts rather than left to grow. + */ + typingTicketMaxEntries?: number; + /** Hard cap for a single inbound media download (default 12MB). */ + mediaMaxBytes?: number; + /** + * Extra hosts an inbound `full_url` may point at, on top of the CDN base and + * WeChat/QQ CDN subdomains. Inbound URLs are attacker-influenced, so anything + * outside this set is ignored in favour of rebuilding from `cdnBaseUrl`. + */ + mediaHostAllowlist?: string[]; +} + +export interface UploadedMedia { + filekey: string; + downloadEncryptedQueryParam: string; + aesKey: Buffer; + rawSize: number; + cipherSize: number; +} diff --git a/packages/ilink/src/typing.test.ts b/packages/ilink/src/typing.test.ts new file mode 100644 index 0000000..05df462 --- /dev/null +++ b/packages/ilink/src/typing.test.ts @@ -0,0 +1,308 @@ +import assert from "node:assert/strict"; +import { afterEach, describe, it } from "node:test"; +import { ILinkClient } from "./client.js"; + +interface Call { + path: string; + body: Record; +} + +type Reply = { status?: number; body: unknown }; +type Handler = (path: string, body: Record) => Reply; + +let restoreFetch: (() => void) | null = null; + +function installFetch(handler: Handler): Call[] { + const calls: Call[] = []; + const original = globalThis.fetch; + restoreFetch = () => { + globalThis.fetch = original; + restoreFetch = null; + }; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + const url = String(input); + const path = new URL(url).pathname; + const body = init?.body + ? (JSON.parse(String(init.body)) as Record) + : {}; + calls.push({ path, body }); + const reply = handler(path, body); + const status = reply.status ?? 200; + return { + ok: status < 400, + status, + headers: new Headers(), + json: async () => reply.body, + text: async () => JSON.stringify(reply.body), + } as unknown as Response; + }) as typeof globalThis.fetch; + return calls; +} + +function client(overrides: Record = {}): ILinkClient { + return new ILinkClient({ + botToken: "tok", + baseUrl: "https://ilink.test", + ...overrides, + }); +} + +const peer = { toUserId: "peer-1", contextToken: "ctx-1" }; + +function deferred(): { promise: Promise; resolve: (v: T) => void } { + let resolve!: (v: T) => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +afterEach(() => { + restoreFetch?.(); +}); + +describe("typing indicator protocol", () => { + it("fetches a ticket then sends status 1 by default", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: "tkt-abc" } } + : { body: { ret: 0 } }, + ); + + await client().sendTyping(peer); + + assert.deepEqual( + calls.map((c) => c.path), + ["/ilink/bot/getconfig", "/ilink/bot/sendtyping"], + ); + assert.equal(calls[0]!.body.ilink_user_id, "peer-1"); + assert.equal(calls[0]!.body.context_token, "ctx-1"); + assert.equal(calls[1]!.body.typing_ticket, "tkt-abc"); + assert.equal(calls[1]!.body.status, 1); + }); + + it("startTyping sends 1 and stopTyping sends 2", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: "tkt" } } + : { body: { ret: 0 } }, + ); + const c = client(); + + await c.startTyping(peer); + await c.stopTyping(peer); + + const statuses = calls + .filter((x) => x.path.endsWith("/sendtyping")) + .map((x) => x.body.status); + assert.deepEqual(statuses, [1, 2]); + }); + + it("caches the ticket across calls (one getconfig per peer)", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: "tkt" } } + : { body: { ret: 0 } }, + ); + const c = client(); + + await c.startTyping(peer); + await c.startTyping(peer); + await c.startTyping(peer); + + assert.equal( + calls.filter((x) => x.path.endsWith("/getconfig")).length, + 1, + "ticket should be reused", + ); + assert.equal(c.getCachedTypingTicket("peer-1"), "tkt"); + }); + + it("collapses concurrent ticket fetches into one getconfig", async () => { + const gate = deferred(); + let getConfigCalls = 0; + installFetch((path) => { + if (path.endsWith("/getconfig")) { + getConfigCalls++; + return { body: { ret: 0, typing_ticket: "tkt" } }; + } + return { body: { ret: 0 } }; + }); + // Hold the first getconfig open so the second caller has to join it. + const original = globalThis.fetch; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + if (String(input).endsWith("/getconfig")) await gate.promise; + return original(input as string, init); + }) as typeof globalThis.fetch; + + const c = client(); + const both = Promise.all([c.startTyping(peer), c.startTyping(peer)]); + gate.resolve(); + await both; + + assert.equal(getConfigCalls, 1); + }); + + it("keeps separate tickets per peer", async () => { + const calls = installFetch((path, body) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: `tkt-${body.ilink_user_id}` } } + : { body: { ret: 0 } }, + ); + const c = client(); + + await c.startTyping({ toUserId: "a", contextToken: "ctx" }); + await c.startTyping({ toUserId: "b", contextToken: "ctx" }); + + assert.equal(c.getCachedTypingTicket("a"), "tkt-a"); + assert.equal(c.getCachedTypingTicket("b"), "tkt-b"); + assert.equal(calls.filter((x) => x.path.endsWith("/getconfig")).length, 2); + }); + + it("does not fetch a ticket just to stop typing", async () => { + const calls = installFetch(() => ({ body: { ret: 0 } })); + + await client().stopTyping(peer); + + assert.deepEqual( + calls.map((c) => c.path), + ["/ilink/bot/sendtyping"], + "stop with no cached ticket must not pay a getconfig round trip", + ); + assert.equal(calls[0]!.body.status, 2); + assert.equal(calls[0]!.body.typing_ticket, undefined); + }); + + it("still sends typing when getconfig fails", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { status: 500, body: { ret: -1, errmsg: "boom" } } + : { body: { ret: 0 } }, + ); + + await client().startTyping(peer); + + const typing = calls.filter((x) => x.path.endsWith("/sendtyping")); + assert.equal(typing.length, 1); + assert.equal(typing[0]!.body.typing_ticket, undefined); + assert.equal(typing[0]!.body.status, 1); + }); + + it("refreshes a stale cached ticket once and retries", async () => { + let issued = 0; + let revoked = false; + const calls = installFetch((path, body) => { + if (path.endsWith("/getconfig")) { + issued++; + return { body: { ret: 0, typing_ticket: `tkt-${issued}` } }; + } + if (revoked && body.typing_ticket === "tkt-1") { + return { body: { ret: -3, errmsg: "invalid typing_ticket" } }; + } + return { body: { ret: 0 } }; + }); + const c = client(); + + await c.startTyping(peer); // primes tkt-1 + assert.equal(c.getCachedTypingTicket("peer-1"), "tkt-1"); + calls.length = 0; + revoked = true; // server drops tkt-1 out from under the cache + + await c.startTyping(peer); + + assert.deepEqual(calls.map((x) => x.path), [ + "/ilink/bot/sendtyping", + "/ilink/bot/getconfig", + "/ilink/bot/sendtyping", + ]); + assert.equal(calls[2]!.body.typing_ticket, "tkt-2"); + assert.equal(c.getCachedTypingTicket("peer-1"), "tkt-2"); + }); + + it("gives up after one retry instead of looping", async () => { + let issued = 0; + const calls = installFetch((path) => { + if (path.endsWith("/getconfig")) { + issued++; + return { body: { ret: 0, typing_ticket: `tkt-${issued}` } }; + } + return { body: { ret: -3, errmsg: "always invalid" } }; + }); + const c = client(); + + await assert.rejects(() => c.startTyping(peer), /always invalid/); + + // first send (tkt-1) → refresh → second send (tkt-2) → stop + assert.equal(calls.filter((x) => x.path.endsWith("/sendtyping")).length, 2); + assert.equal(calls.filter((x) => x.path.endsWith("/getconfig")).length, 2); + }); + + it("honours an explicitly supplied ticket without calling getconfig", async () => { + const calls = installFetch(() => ({ body: { ret: 0 } })); + + await client().startTyping({ ...peer, typingTicket: "manual" }); + + assert.deepEqual(calls.map((c) => c.path), ["/ilink/bot/sendtyping"]); + assert.equal(calls[0]!.body.typing_ticket, "manual"); + }); + + it("expires cached tickets after the configured TTL", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: "tkt" } } + : { body: { ret: 0 } }, + ); + // Floor is 60s, so ask for the minimum and then move the clock past it. + const c = client({ typingTicketTtlMs: 1 }); + await c.startTyping(peer); + assert.equal(calls.filter((x) => x.path.endsWith("/getconfig")).length, 1); + + const realNow = Date.now; + Date.now = () => realNow() + 61_000; + try { + assert.equal(c.getCachedTypingTicket("peer-1"), null); + await c.startTyping(peer); + } finally { + Date.now = realNow; + } + assert.equal(calls.filter((x) => x.path.endsWith("/getconfig")).length, 2); + }); + + it("bounds the ticket cache", async () => { + installFetch((path, body) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: `tkt-${body.ilink_user_id}` } } + : { body: { ret: 0 } }, + ); + // Floor is 64 entries regardless of a smaller request. + const c = client({ typingTicketMaxEntries: 1 }); + for (let i = 0; i < 70; i++) { + await c.startTyping({ toUserId: `p${i}`, contextToken: "ctx" }); + } + + // Oldest evicted, newest retained — never unbounded growth. + assert.equal(c.getCachedTypingTicket("p0"), null); + assert.equal(c.getCachedTypingTicket("p69"), "tkt-p69"); + }); + + it("invalidateTypingTicket forces the next fetch", async () => { + const calls = installFetch((path) => + path.endsWith("/getconfig") + ? { body: { ret: 0, typing_ticket: "tkt" } } + : { body: { ret: 0 } }, + ); + const c = client(); + + await c.startTyping(peer); + c.invalidateTypingTicket("peer-1"); + await c.startTyping(peer); + + assert.equal(calls.filter((x) => x.path.endsWith("/getconfig")).length, 2); + }); + + it("requires a bot token", async () => { + installFetch(() => ({ body: { ret: 0 } })); + const c = new ILinkClient({ baseUrl: "https://ilink.test" }); + await assert.rejects(() => c.startTyping(peer), /bot_token is required/); + }); +}); diff --git a/packages/ilink/tsconfig.json b/packages/ilink/tsconfig.json new file mode 100644 index 0000000..a013e0c --- /dev/null +++ b/packages/ilink/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/packages/llm/package.json b/packages/llm/package.json new file mode 100644 index 0000000..58f110a --- /dev/null +++ b/packages/llm/package.json @@ -0,0 +1,28 @@ +{ + "name": "@wechat-ai/llm", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --import tsx --test src/**/*.test.ts" + }, + "dependencies": { + "openai": "^4.91.1" + }, + "devDependencies": { + "@types/node": "^22.13.10", + "tsx": "^4.19.3", + "typescript": "^5.8.2" + } +} diff --git a/packages/llm/src/client.test.ts b/packages/llm/src/client.test.ts new file mode 100644 index 0000000..342e53b --- /dev/null +++ b/packages/llm/src/client.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { formatCurrentTime, loadLlmConfigFromEnv } from "./client.js"; + +describe("formatCurrentTime", () => { + it("returns JSON with iso and Asia/Shanghai fields", () => { + const raw = formatCurrentTime("Asia/Shanghai"); + const obj = JSON.parse(raw) as { + iso: string; + timeZone: string; + local: string; + weekday: string; + }; + assert.equal(obj.timeZone, "Asia/Shanghai"); + assert.ok(obj.iso.includes("T")); + assert.ok(obj.local.length > 0); + }); +}); + +describe("loadLlmConfigFromEnv", () => { + it("loads platform LLM and optional tools gateway", () => { + const cfg = loadLlmConfigFromEnv({ + LLM_API_KEY: "sk-platform", + LLM_BASE_URL: "https://api.openai.com/v1", + LLM_MODEL: "gpt-4o-mini", + TOOLS_BASE_URL: "http://127.0.0.1:7860", + TOOLS_API_KEY: "tools-secret", + }); + assert.equal(cfg.apiKey, "sk-platform"); + assert.equal(cfg.toolsBaseUrl, "http://127.0.0.1:7860"); + assert.equal(cfg.toolsApiKey, "tools-secret"); + }); + + it("requires LLM_API_KEY for platform", () => { + assert.throws(() => loadLlmConfigFromEnv({}), /LLM_API_KEY/); + }); +}); diff --git a/packages/llm/src/client.ts b/packages/llm/src/client.ts new file mode 100644 index 0000000..400c598 --- /dev/null +++ b/packages/llm/src/client.ts @@ -0,0 +1,735 @@ +import OpenAI, { type ClientOptions } from "openai"; +import type { + ChatCompletionMessageParam, + ChatCompletionTool, +} from "openai/resources/chat/completions"; + +/** + * Platform (admin) LLM: main site connects directly via baseURL/apiKey. + * User custom LLM: route through tools gateway (TOOLS_BASE_URL) with `upstream` + * so only the HF tools process dials the user's API. + */ +export interface LlmUpstream { + baseUrl: string; + apiKey: string; + model: string; +} + +export interface LlmConfig { + /** + * When toolsGateway is set, chat goes to tools /v1/chat/completions. + * Otherwise baseURL is the real OpenAI-compatible platform endpoint. + */ + baseURL: string; + apiKey: string; + model: string; + temperature?: number; + maxTokens?: number; + /** + * HF / tools gateway root (no trailing slash), e.g. http://127.0.0.1:7860 + * Used for user-custom upstream and web search. + */ + toolsBaseUrl?: string; + toolsApiKey?: string; + /** Default upstream injected on every chat (user custom provider). */ + defaultUpstream?: LlmUpstream | null; + /** + * Per-request wall clock for one completion. Without it the OpenAI SDK + * defaults to 600s × 3 retries = 30 minutes, during which the job holds a + * reply-consumer slot and keeps the bot:peer chain locked. + */ + timeoutMs?: number; + /** + * Override the HTTP layer used for the platform path. The OpenAI SDK bundles + * node-fetch, so a global stub cannot reach it — this seam is what makes the + * request body assertable, and it also leaves room for a proxy agent later. + */ + fetchImpl?: typeof fetch; +} + +/** Default completion timeout. Deliberately looser than TOOLS_TIMEOUT_MS (search). */ +export const DEFAULT_LLM_TIMEOUT_MS = Number( + process.env.LLM_TIMEOUT_MS ?? "120000", +); + +export interface ChatTextPart { + type: "text"; + text: string; +} + +export interface ChatImagePart { + type: "image_url"; + image_url: { + /** https URL or a `data:;base64,...` URI */ + url: string; + detail?: "auto" | "low" | "high"; + }; +} + +/** Multimodal content parts (OpenAI-compatible vision shape). */ +export type ChatContentPart = ChatTextPart | ChatImagePart; + +export interface ChatMessage { + role: "system" | "user" | "assistant"; + /** + * Plain text, or content parts for vision. Only `user` messages are sent as + * parts — providers disagree about array content on system/assistant, so + * those are flattened to text. + */ + content: string | ChatContentPart[]; +} + +/** Readable text for history / logs / token bookkeeping. */ +export function flattenChatContent( + content: string | ChatContentPart[], +): string { + if (typeof content === "string") return content; + return content + .map((p) => (p.type === "text" ? p.text : "[图片]")) + .filter(Boolean) + .join("\n"); +} + +export interface ChatResult { + text: string; + promptTokens: number; + completionTokens: number; + totalTokens: number; + model: string; +} + +export type BuiltinToolName = "get_current_time" | "web_search"; + +export interface ChatCallOptions { + /** Whitelist of built-in tools to expose (empty / omit = no tools) */ + tools?: BuiltinToolName[]; + /** IANA timezone for get_current_time (default Asia/Shanghai) */ + timeZone?: string; + /** Max tool round-trips (default 2) */ + maxToolRounds?: number; + /** + * Per-call model override — used to route image messages to a vision model. + * Ignored on the user-custom-upstream path: that provider only knows its own + * model names, so `upstream.model` stays authoritative there. + */ + model?: string; + /** + * Per-call max_tokens override. Used to keep an image caption to a + * description rather than paying for a full reply-sized completion. + */ + maxTokens?: number; + /** Override default upstream for this call (user custom API via tools) */ + upstream?: LlmUpstream | null; + /** + * When web_search tool is enabled, called by the client to execute search + * exclusively via the tools gateway (main site never dials search engines). + */ + webSearch?: (query: string, maxResults?: number) => Promise; +} + +const TIME_TOOL: ChatCompletionTool = { + type: "function", + function: { + name: "get_current_time", + description: + "Get the current date and time. Use when the user asks about now, today, weekday, or relative time.", + parameters: { + type: "object", + properties: { + timeZone: { + type: "string", + description: + "Optional IANA timezone, e.g. Asia/Shanghai. Defaults to server config.", + }, + }, + additionalProperties: false, + }, + }, +}; + +const WEB_SEARCH_TOOL: ChatCompletionTool = { + type: "function", + function: { + name: "web_search", + description: + "Search the public web for up-to-date facts, news, or references. Use when the answer needs current information.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "Search query in natural language", + }, + max_results: { + type: "integer", + description: "Max results 1-10 (default 5)", + }, + }, + required: ["query"], + additionalProperties: false, + }, + }, +}; + +export function formatCurrentTime(timeZone = "Asia/Shanghai"): string { + const tz = timeZone?.trim() || "Asia/Shanghai"; + const now = new Date(); + let local = ""; + let weekday = ""; + try { + local = new Intl.DateTimeFormat("zh-CN", { + timeZone: tz, + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: false, + }).format(now); + weekday = new Intl.DateTimeFormat("zh-CN", { + timeZone: tz, + weekday: "long", + }).format(now); + } catch { + local = now.toISOString(); + weekday = ""; + } + return JSON.stringify({ + iso: now.toISOString(), + timeZone: tz, + local, + weekday, + }); +} + +function normalizeBaseUrl(url: string): string { + return (url || "").trim().replace(/\/+$/, ""); +} + +/** + * Map a ChatMessage onto the SDK param type. + * + * Array content is only emitted for `user`. Support for array content on + * system/assistant varies across the OpenAI-compatible endpoints this talks to, + * so those are flattened to text instead of gambling on the provider. + */ +function toApiMessage(m: ChatMessage): ChatCompletionMessageParam { + if (typeof m.content === "string") { + return { role: m.role, content: m.content } as ChatCompletionMessageParam; + } + if (m.role !== "user") { + return { + role: m.role, + content: flattenChatContent(m.content), + } as ChatCompletionMessageParam; + } + return { + role: "user", + content: m.content.map((p) => + p.type === "text" + ? { type: "text" as const, text: p.text } + : { + type: "image_url" as const, + image_url: { + url: p.image_url.url, + ...(p.image_url.detail ? { detail: p.image_url.detail } : {}), + }, + }, + ), + }; +} + +export class LlmClient { + private client: OpenAI; + private model: string; + private temperature: number; + private maxTokens: number; + private toolsBaseUrl: string | null; + private toolsApiKey: string; + private defaultUpstream: LlmUpstream | null; + private timeoutMs: number; + + constructor(cfg: LlmConfig) { + this.model = cfg.model; + this.temperature = cfg.temperature ?? 0.8; + this.maxTokens = cfg.maxTokens ?? 1024; + this.toolsBaseUrl = cfg.toolsBaseUrl + ? normalizeBaseUrl(cfg.toolsBaseUrl) + : null; + this.toolsApiKey = (cfg.toolsApiKey ?? "").trim(); + this.defaultUpstream = cfg.defaultUpstream ?? null; + this.timeoutMs = Math.max(1000, cfg.timeoutMs ?? DEFAULT_LLM_TIMEOUT_MS); + + // Platform mode: SDK baseURL = real provider (admin LLM). + // User custom path ignores this client and uses createViaToolsGateway + upstream. + this.client = new OpenAI({ + apiKey: cfg.apiKey, + baseURL: normalizeBaseUrl(cfg.baseURL), + // Bounded per attempt; one retry only. A hung upstream must not pin a + // reply-consumer slot for half an hour. + timeout: this.timeoutMs, + maxRetries: 1, + defaultHeaders: { + "User-Agent": "WeChat-AI/1.0", + }, + // The SDK types this against its bundled node-fetch; a native fetch + // satisfies everything it actually uses (ok / headers / json / body). + ...(cfg.fetchImpl + ? { fetch: cfg.fetchImpl as unknown as ClientOptions["fetch"] } + : {}), + }); + } + + /** Platform client (admin LLM), optional tools for search. */ + static forPlatform(cfg: { + baseURL: string; + apiKey: string; + model: string; + temperature?: number; + maxTokens?: number; + toolsBaseUrl?: string; + toolsApiKey?: string; + fetchImpl?: typeof fetch; + }): LlmClient { + return new LlmClient({ + baseURL: cfg.baseURL, + apiKey: cfg.apiKey, + model: cfg.model, + temperature: cfg.temperature, + maxTokens: cfg.maxTokens, + toolsBaseUrl: cfg.toolsBaseUrl, + toolsApiKey: cfg.toolsApiKey, + defaultUpstream: null, + fetchImpl: cfg.fetchImpl, + }); + } + + /** + * User custom LLM: all chat traffic goes through tools gateway with upstream. + * Main site never dials user baseUrl. + */ + static forUserUpstream(cfg: { + toolsBaseUrl: string; + toolsApiKey: string; + upstream: LlmUpstream; + temperature?: number; + maxTokens?: number; + timeoutMs?: number; + }): LlmClient { + const tools = normalizeBaseUrl(cfg.toolsBaseUrl); + if (!tools) { + throw new Error("TOOLS_BASE_URL is required for user custom LLM"); + } + if (!cfg.toolsApiKey?.trim()) { + throw new Error("TOOLS_API_KEY is required for user custom LLM"); + } + return new LlmClient({ + baseURL: tools, + apiKey: cfg.toolsApiKey, + model: cfg.upstream.model, + temperature: cfg.temperature, + maxTokens: cfg.maxTokens, + toolsBaseUrl: tools, + toolsApiKey: cfg.toolsApiKey, + defaultUpstream: cfg.upstream, + timeoutMs: cfg.timeoutMs, + }); + } + + getToolsBaseUrl(): string | null { + return this.toolsBaseUrl; + } + + async chat(messages: ChatMessage[]): Promise { + const r = await this.chatWithUsage(messages); + return r.text; + } + + async chatWithUsage( + messages: ChatMessage[], + opts: ChatCallOptions = {}, + ): Promise { + const toolNames = opts.tools?.length ? opts.tools : []; + const timeZone = opts.timeZone?.trim() || "Asia/Shanghai"; + const maxRounds = Math.max(0, opts.maxToolRounds ?? 2); + const upstream = + opts.upstream === undefined ? this.defaultUpstream : opts.upstream; + + const tools: ChatCompletionTool[] = []; + if (toolNames.includes("get_current_time")) tools.push(TIME_TOOL); + if (toolNames.includes("web_search")) tools.push(WEB_SEARCH_TOOL); + const toolsOpt = tools.length ? tools : undefined; + + const apiMessages: ChatCompletionMessageParam[] = messages.map(toApiMessage); + const modelOverride = opts.model?.trim() || null; + const maxTokensOverride = + typeof opts.maxTokens === "number" && opts.maxTokens > 0 + ? Math.floor(opts.maxTokens) + : null; + + let promptTokens = 0; + let completionTokens = 0; + let model = modelOverride ?? this.model; + + for (let round = 0; round <= maxRounds; round++) { + const res = await this.createCompletion( + apiMessages, + toolsOpt, + upstream, + modelOverride, + maxTokensOverride, + ); + + const usage = res.usage; + promptTokens += usage?.prompt_tokens ?? 0; + completionTokens += usage?.completion_tokens ?? 0; + model = res.model || modelOverride || this.model; + + const choice = res.choices[0]?.message; + if (!choice) { + throw new Error("LLM returned empty choice"); + } + + const toolCalls = choice.tool_calls; + if (toolsOpt && toolCalls?.length && round < maxRounds) { + apiMessages.push({ + role: "assistant", + content: choice.content ?? null, + tool_calls: toolCalls, + }); + for (const tc of toolCalls) { + const fn = tc.function; + const result = await this.runBuiltinTool( + fn?.name ?? "", + fn?.arguments ?? "{}", + timeZone, + opts.webSearch, + ); + apiMessages.push({ + role: "tool", + tool_call_id: tc.id, + content: result, + }); + } + continue; + } + + const text = (choice.content ?? "").trim(); + if (!text) { + throw new Error("LLM returned empty content"); + } + return { + text, + promptTokens, + completionTokens, + totalTokens: promptTokens + completionTokens, + model, + }; + } + + throw new Error("LLM tool loop exceeded max rounds without final text"); + } + + private async createCompletion( + apiMessages: ChatCompletionMessageParam[], + tools: ChatCompletionTool[] | undefined, + upstream: LlmUpstream | null | undefined, + modelOverride?: string | null, + maxTokensOverride?: number | null, + ) { + // User custom path: must go through tools with upstream body field. + if (upstream) { + return this.createViaToolsGateway( + apiMessages, + tools, + upstream, + maxTokensOverride, + ); + } + // Platform path: direct OpenAI SDK (admin-configured LLM). + return this.client.chat.completions.create({ + model: modelOverride ?? this.model, + messages: apiMessages, + temperature: this.temperature, + max_tokens: maxTokensOverride ?? this.maxTokens, + ...(tools ? { tools, tool_choice: "auto" as const } : {}), + }); + } + + /** + * Call tools gateway /v1/chat/completions with upstream credentials. + * Uses fetch so we can inject non-standard `upstream` without SDK stripping it. + */ + private async createViaToolsGateway( + apiMessages: ChatCompletionMessageParam[], + tools: ChatCompletionTool[] | undefined, + upstream: LlmUpstream, + maxTokensOverride?: number | null, + ) { + const toolsRoot = this.toolsBaseUrl; + if (!toolsRoot) { + throw new Error( + "User custom LLM requires TOOLS_BASE_URL (HF tools gateway)", + ); + } + const url = `${toolsRoot}/v1/chat/completions`; + const body: Record = { + model: upstream.model || this.model, + messages: apiMessages, + temperature: this.temperature, + max_tokens: maxTokensOverride ?? this.maxTokens, + upstream: { + base_url: upstream.baseUrl, + api_key: upstream.apiKey, + model: upstream.model || this.model, + }, + }; + if (tools?.length) { + body.tools = tools; + body.tool_choice = "auto"; + } + + const headers: Record = { + "Content-Type": "application/json", + "User-Agent": "WeChat-AI/1.0", + }; + const key = this.toolsApiKey || ""; + if (key) headers.Authorization = `Bearer ${key}`; + + // Sole egress for user-custom providers, and the target is a HF Space that + // can be cold-starting — must be bounded. + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + let resp: Response; + let text: string; + try { + resp = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: ctrl.signal, + }); + text = await resp.text(); + } catch (err: unknown) { + if (ctrl.signal.aborted) { + throw new Error( + `Tools gateway chat timed out after ${this.timeoutMs}ms`, + ); + } + throw err; + } finally { + clearTimeout(timer); + } + if (!resp.ok) { + const snippet = text.slice(0, 400); + throw new Error( + `Tools gateway chat failed HTTP ${resp.status}: ${snippet}`, + ); + } + type GatewayMsg = { + content?: string | null; + tool_calls?: Array<{ + id: string; + type: "function"; + function: { name: string; arguments: string }; + }>; + }; + type GatewayRes = { + choices?: Array<{ message?: GatewayMsg }>; + usage?: { prompt_tokens?: number; completion_tokens?: number }; + model?: string; + }; + let data: GatewayRes; + try { + data = JSON.parse(text) as GatewayRes; + } catch { + throw new Error("Tools gateway returned non-JSON"); + } + // Shape compatible with OpenAI SDK response used in chatWithUsage + return { + choices: (data.choices ?? []).map((c) => ({ + message: { + content: c.message?.content ?? null, + tool_calls: c.message?.tool_calls, + role: "assistant" as const, + }, + })), + usage: { + prompt_tokens: data.usage?.prompt_tokens ?? 0, + completion_tokens: data.usage?.completion_tokens ?? 0, + }, + model: data.model || upstream.model || this.model, + }; + } + + private async runBuiltinTool( + name: string, + argsJson: string, + defaultTimeZone: string, + webSearch?: ChatCallOptions["webSearch"], + ): Promise { + if (name === "get_current_time") { + let tz = defaultTimeZone; + try { + const args = argsJson + ? (JSON.parse(argsJson) as { timeZone?: string }) + : {}; + if (typeof args.timeZone === "string" && args.timeZone.trim()) { + tz = args.timeZone.trim(); + } + } catch { + /* use default */ + } + return formatCurrentTime(tz); + } + if (name === "web_search") { + if (!webSearch) { + return JSON.stringify({ + error: "web_search is not configured (enable WEB_SEARCH + TOOLS)", + }); + } + let query = ""; + let maxResults = 5; + try { + const args = argsJson + ? (JSON.parse(argsJson) as { + query?: string; + max_results?: number; + }) + : {}; + query = typeof args.query === "string" ? args.query.trim() : ""; + if (typeof args.max_results === "number") { + maxResults = args.max_results; + } + } catch { + return JSON.stringify({ error: "invalid web_search arguments" }); + } + if (!query) { + return JSON.stringify({ error: "query is required" }); + } + try { + return await webSearch(query, maxResults); + } catch (err) { + return JSON.stringify({ + error: `web_search failed: ${(err as Error).message}`, + }); + } + } + return JSON.stringify({ error: `unknown tool: ${name}` }); + } +} + +export interface ToolsGatewayConfig { + toolsBaseUrl: string; + toolsApiKey: string; + timeoutMs?: number; +} + +export interface WebSearchHit { + title: string; + url: string; + snippet: string; +} + +/** + * Web search client — **only** calls tools gateway (never DDG from main site). + */ +export class WebSearchClient { + private baseUrl: string; + private apiKey: string; + private timeoutMs: number; + + constructor(cfg: ToolsGatewayConfig) { + this.baseUrl = normalizeBaseUrl(cfg.toolsBaseUrl); + this.apiKey = (cfg.toolsApiKey ?? "").trim(); + this.timeoutMs = cfg.timeoutMs ?? 15_000; + if (!this.baseUrl) { + throw new Error("TOOLS_BASE_URL is required for web search"); + } + } + + async search( + query: string, + maxResults = 5, + ): Promise { + const url = `${this.baseUrl}/v1/web-search`; + const headers: Record = { + "Content-Type": "application/json", + "User-Agent": "WeChat-AI/1.0", + }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), this.timeoutMs); + try { + const resp = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ + query, + max_results: maxResults, + }), + signal: ctrl.signal, + }); + const text = await resp.text(); + if (!resp.ok) { + throw new Error(`HTTP ${resp.status}: ${text.slice(0, 200)}`); + } + const data = JSON.parse(text) as { results?: WebSearchHit[] }; + return Array.isArray(data.results) ? data.results : []; + } finally { + clearTimeout(timer); + } + } + + /** Tool-friendly JSON string for LLM tool results. */ + async searchAsToolResult(query: string, maxResults = 5): Promise { + const results = await this.search(query, maxResults); + return JSON.stringify({ query, results }); + } +} + +export async function probeToolsHealth( + toolsBaseUrl: string, + timeoutMs = 8000, +): Promise<{ ok: boolean; detail: string }> { + const base = normalizeBaseUrl(toolsBaseUrl); + if (!base) return { ok: false, detail: "TOOLS_BASE_URL empty" }; + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const resp = await fetch(`${base}/health`, { signal: ctrl.signal }); + const text = await resp.text(); + if (!resp.ok) { + return { ok: false, detail: `HTTP ${resp.status}: ${text.slice(0, 120)}` }; + } + return { ok: true, detail: text.slice(0, 200) }; + } catch (err) { + return { ok: false, detail: (err as Error).message }; + } finally { + clearTimeout(timer); + } +} + +/** + * Load platform LLM config (admin). Does not use tools for chat. + * Tools URL is separate for search / user custom APIs. + */ +export function loadLlmConfigFromEnv( + env: NodeJS.ProcessEnv = process.env, +): LlmConfig { + const apiKey = env.LLM_API_KEY ?? ""; + if (!apiKey) { + throw new Error("LLM_API_KEY is required (platform / admin LLM)"); + } + const toolsBaseUrl = (env.TOOLS_BASE_URL ?? "").trim() || undefined; + const toolsApiKey = (env.TOOLS_API_KEY ?? "").trim() || undefined; + return { + baseURL: env.LLM_BASE_URL ?? "https://api.openai.com/v1", + apiKey, + model: env.LLM_MODEL ?? "gpt-4o-mini", + toolsBaseUrl, + toolsApiKey, + }; +} diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts new file mode 100644 index 0000000..45a73ab --- /dev/null +++ b/packages/llm/src/index.ts @@ -0,0 +1,19 @@ +export { + LlmClient, + WebSearchClient, + loadLlmConfigFromEnv, + formatCurrentTime, + flattenChatContent, + probeToolsHealth, + type ChatContentPart, + type ChatImagePart, + type ChatMessage, + type ChatResult, + type ChatTextPart, + type LlmConfig, + type LlmUpstream, + type ChatCallOptions, + type BuiltinToolName, + type ToolsGatewayConfig, + type WebSearchHit, +} from "./client.js"; diff --git a/packages/llm/src/multimodal.test.ts b/packages/llm/src/multimodal.test.ts new file mode 100644 index 0000000..55cc40c --- /dev/null +++ b/packages/llm/src/multimodal.test.ts @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import { afterEach, describe, it } from "node:test"; +import { LlmClient, flattenChatContent } from "./client.js"; +import type { ChatMessage } from "./client.js"; + +let restoreFetch: (() => void) | null = null; + +interface Captured { + url: string; + body: Record; +} + +/** + * One recorder used for both paths: the platform path goes through the SDK's + * injectable `fetch`, the tools-gateway path uses raw global fetch. + */ +function installFetch(): Captured[] { + const captured: Captured[] = []; + const original = globalThis.fetch; + restoreFetch = () => { + globalThis.fetch = original; + capturingFetch = null; + restoreFetch = null; + }; + const impl = (async (input: unknown, init?: RequestInit) => { + captured.push({ + url: String( + typeof input === "object" && input && "url" in input + ? (input as { url: string }).url + : input, + ), + body: init?.body + ? (JSON.parse(String(init.body)) as Record) + : {}, + }); + const payload = { + id: "cmpl-1", + object: "chat.completion", + created: 1, + model: "served-model", + choices: [ + { + index: 0, + message: { role: "assistant", content: "看到了" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 11, completion_tokens: 3, total_tokens: 14 }, + }; + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof globalThis.fetch; + globalThis.fetch = impl; + capturingFetch = impl; + return captured; +} + +let capturingFetch: typeof fetch | null = null; + +const IMAGE_MESSAGE: ChatMessage = { + role: "user", + content: [ + { type: "text", text: "这是什么" }, + { + type: "image_url", + image_url: { url: "data:image/png;base64,AAAA", detail: "low" }, + }, + ], +}; + +function platform(): LlmClient { + return LlmClient.forPlatform({ + baseURL: "https://llm.test/v1", + apiKey: "k", + model: "base-model", + // Must be injected: the SDK bundles node-fetch and ignores global stubs. + fetchImpl: capturingFetch ?? undefined, + }); +} + +afterEach(() => { + restoreFetch?.(); +}); + +describe("flattenChatContent", () => { + it("passes strings through", () => { + assert.equal(flattenChatContent("hi"), "hi"); + }); + + it("renders image parts as a placeholder so history stays readable", () => { + assert.equal(flattenChatContent(IMAGE_MESSAGE.content), "这是什么\n[图片]"); + }); + + it("drops empty text parts", () => { + assert.equal( + flattenChatContent([ + { type: "text", text: "" }, + { type: "text", text: "b" }, + ]), + "b", + ); + }); +}); + +describe("multimodal messages (platform path)", () => { + it("forwards user content parts verbatim", async () => { + const captured = installFetch(); + const res = await platform().chatWithUsage([ + { role: "system", content: "你是助手" }, + IMAGE_MESSAGE, + ]); + + assert.equal(res.text, "看到了"); + assert.equal(captured.length, 1); + const msgs = captured[0]!.body.messages as Array>; + assert.equal(msgs[0]!.content, "你是助手"); + const parts = msgs[1]!.content as Array>; + assert.equal(Array.isArray(parts), true); + assert.deepEqual(parts[0], { type: "text", text: "这是什么" }); + assert.deepEqual(parts[1], { + type: "image_url", + image_url: { url: "data:image/png;base64,AAAA", detail: "low" }, + }); + }); + + it("omits detail when unset", async () => { + const captured = installFetch(); + await platform().chatWithUsage([ + { + role: "user", + content: [{ type: "image_url", image_url: { url: "https://x/y.png" } }], + }, + ]); + const parts = ( + (captured[0]!.body.messages as Array>)[0]! + .content as Array> + )[0]!; + assert.deepEqual(parts, { + type: "image_url", + image_url: { url: "https://x/y.png" }, + }); + }); + + it("flattens array content on non-user roles", async () => { + const captured = installFetch(); + await platform().chatWithUsage([ + { + role: "assistant", + content: [ + { type: "text", text: "上一轮" }, + { type: "image_url", image_url: { url: "data:image/png;base64,Z" } }, + ], + }, + { role: "user", content: "继续" }, + ]); + const msgs = captured[0]!.body.messages as Array>; + assert.equal(msgs[0]!.content, "上一轮\n[图片]"); + }); + + it("applies the per-call model override", async () => { + const captured = installFetch(); + const res = await platform().chatWithUsage([IMAGE_MESSAGE], { + model: "vision-model", + }); + assert.equal(captured[0]!.body.model, "vision-model"); + // Served model from the response still wins for accounting. + assert.equal(res.model, "served-model"); + }); + + it("falls back to the constructor model without an override", async () => { + const captured = installFetch(); + await platform().chatWithUsage([{ role: "user", content: "hi" }]); + assert.equal(captured[0]!.body.model, "base-model"); + }); + + it("ignores a blank model override", async () => { + const captured = installFetch(); + await platform().chatWithUsage([{ role: "user", content: "hi" }], { + model: " ", + }); + assert.equal(captured[0]!.body.model, "base-model"); + }); +}); + +describe("multimodal messages (user custom upstream path)", () => { + it("sends parts through the tools gateway and keeps upstream.model", async () => { + const captured = installFetch(); + const client = LlmClient.forUserUpstream({ + toolsBaseUrl: "https://tools.test", + toolsApiKey: "tk", + upstream: { + baseUrl: "https://user-provider.test/v1", + apiKey: "user-key", + model: "user-model", + }, + }); + + await client.chatWithUsage([IMAGE_MESSAGE], { model: "vision-model" }); + + assert.equal(captured.length, 1); + assert.equal(captured[0]!.url, "https://tools.test/v1/chat/completions"); + // The user's provider only knows its own model names, so the override + // must not leak onto this path. + assert.equal(captured[0]!.body.model, "user-model"); + const upstream = captured[0]!.body.upstream as Record; + assert.equal(upstream.model, "user-model"); + const parts = (captured[0]!.body.messages as Array>)[0]! + .content as Array>; + assert.equal(Array.isArray(parts), true); + assert.equal(parts[1]!.type, "image_url"); + }); +}); diff --git a/packages/llm/tsconfig.json b/packages/llm/tsconfig.json new file mode 100644 index 0000000..a013e0c --- /dev/null +++ b/packages/llm/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..e7cbb0b --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,1372 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: {} + + apps/api: + dependencies: + '@fastify/compress': + specifier: ^8.0.1 + version: 8.3.1 + '@wechat-ai/core': + specifier: workspace:* + version: link:../../packages/core + '@wechat-ai/db': + specifier: workspace:* + version: link:../../packages/db + '@wechat-ai/ilink': + specifier: workspace:* + version: link:../../packages/ilink + '@wechat-ai/llm': + specifier: workspace:* + version: link:../../packages/llm + dotenv: + specifier: ^16.4.7 + version: 16.6.1 + fastify: + specifier: ^5.2.1 + version: 5.10.0 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + zod: + specifier: ^3.24.2 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^22.13.10 + version: 22.20.1 + typescript: + specifier: ^5.8.2 + version: 5.9.3 + + packages/core: + dependencies: + '@wechat-ai/db': + specifier: workspace:* + version: link:../db + '@wechat-ai/llm': + specifier: workspace:* + version: link:../llm + devDependencies: + '@types/node': + specifier: ^22.13.10 + version: 22.20.1 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + typescript: + specifier: ^5.8.2 + version: 5.9.3 + + packages/db: + dependencies: + ioredis: + specifier: ^5.6.0 + version: 5.11.1 + devDependencies: + '@types/node': + specifier: ^22.13.10 + version: 22.20.1 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + typescript: + specifier: ^5.8.2 + version: 5.9.3 + + packages/ilink: + devDependencies: + '@types/node': + specifier: ^22.13.10 + version: 22.20.1 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + typescript: + specifier: ^5.8.2 + version: 5.9.3 + + packages/llm: + dependencies: + openai: + specifier: ^4.91.1 + version: 4.104.0(zod@3.25.76) + devDependencies: + '@types/node': + specifier: ^22.13.10 + version: 22.20.1 + tsx: + specifier: ^4.19.3 + version: 4.23.1 + typescript: + specifier: ^5.8.2 + version: 5.9.3 + +packages: + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@fastify/accept-negotiator@2.0.1': + resolution: {integrity: sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ==} + + '@fastify/ajv-compiler@4.0.5': + resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} + + '@fastify/compress@8.3.1': + resolution: {integrity: sha512-BUpItLr6MUX9e9ukg5Y6xekyA/7pBFG8QWtFCrUDm9ctoBc3R2/nA16yOaOWtVoccpXGjdDEYA/MxAb5+8cxag==} + + '@fastify/error@4.2.0': + resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} + + '@fastify/fast-json-stringify-compiler@5.1.0': + resolution: {integrity: sha512-PxcYtKLbQ8Z+yApiqjK8FwxIwvEj38k2OiLc17u8dkJSlmfi2wHHPaSnaoqBPQqtvF8YVsDgDpP2snDCfFrpfw==} + + '@fastify/forwarded@3.0.1': + resolution: {integrity: sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==} + + '@fastify/merge-json-schemas@0.2.1': + resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} + + '@fastify/proxy-addr@5.1.0': + resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + + '@ioredis/commands@1.10.0': + resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + + '@types/node@22.20.1': + resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + avvio@9.3.0: + resolution: {integrity: sha512-g2tQ7LE7oOSqDfwEm3M+ZCMTJc7KiZCdJ4UwyZJb5ckTKyYu50OYmvv0mCFXPuYXoM4zkSt8zM9XQ9KCvxA74A==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + cluster-key-slot@1.1.1: + resolution: {integrity: sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==} + engines: {node: '>=0.10.0'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + duplexify@3.7.1: + resolution: {integrity: sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==} + + duplexify@4.1.3: + resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stringify@7.0.1: + resolution: {integrity: sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA==} + + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-uri@3.1.3: + resolution: {integrity: sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==} + + fast-uri@4.1.0: + resolution: {integrity: sha512-ZodJ2cRiLVWGi9IgPb3mbgSqM4CD3LexCHkuv0FfBXHJI1ADfucTD06m6clO2Cy5RZYsw/SiCVl/dyrFI/SYWA==} + + fastify-plugin@5.1.0: + resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==} + + fastify@5.10.0: + resolution: {integrity: sha512-A9L0ziuWGQHgEEVgF3davQ9vbD93IuX+lo2IsxapQmu5b/Y/ynn9m9K5JHt9dvyJXOFc5iN0Zk5GHEOqnzhWjg==} + + fastq@1.20.1: + resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} + + find-my-way@9.6.0: + resolution: {integrity: sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==} + engines: {node: '>=20'} + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ioredis@5.11.1: + resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==} + engines: {node: '>=12.22.0'} + + ipaddr.js@2.4.0: + resolution: {integrity: sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==} + engines: {node: '>= 10'} + + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + + json-schema-ref-resolver@3.0.0: + resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + light-my-request@6.6.0: + resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + minipass@7.1.3: + resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} + engines: {node: '>=16 || 14 >=14.17'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + peek-stream@1.1.3: + resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==} + + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} + hasBin: true + + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + process-warning@4.0.1: + resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} + + process-warning@5.0.0: + resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + + pumpify@2.0.1: + resolution: {integrity: sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + real-require@1.0.0: + resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} + + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + ret@0.5.0: + resolution: {integrity: sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==} + engines: {node: '>=10'} + + reusify@1.1.0: + resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-regex2@5.1.1: + resolution: {integrity: sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==} + hasBin: true + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + + stream-shift@1.0.3: + resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + thread-stream@4.2.0: + resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} + engines: {node: '>=20'} + + through2@2.0.5: + resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} + + toad-cache@3.7.4: + resolution: {integrity: sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==} + engines: {node: '>=20'} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + tsx@4.23.1: + resolution: {integrity: sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + +snapshots: + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@fastify/accept-negotiator@2.0.1': {} + + '@fastify/ajv-compiler@4.0.5': + dependencies: + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + fast-uri: 3.1.3 + + '@fastify/compress@8.3.1': + dependencies: + '@fastify/accept-negotiator': 2.0.1 + fastify-plugin: 5.1.0 + mime-db: 1.52.0 + minipass: 7.1.3 + peek-stream: 1.1.3 + pump: 3.0.4 + pumpify: 2.0.1 + readable-stream: 4.7.0 + + '@fastify/error@4.2.0': {} + + '@fastify/fast-json-stringify-compiler@5.1.0': + dependencies: + fast-json-stringify: 7.0.1 + + '@fastify/forwarded@3.0.1': {} + + '@fastify/merge-json-schemas@0.2.1': + dependencies: + dequal: 2.0.3 + + '@fastify/proxy-addr@5.1.0': + dependencies: + '@fastify/forwarded': 3.0.1 + ipaddr.js: 2.4.0 + + '@ioredis/commands@1.10.0': {} + + '@pinojs/redact@0.4.0': {} + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 22.20.1 + form-data: 4.0.6 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + + '@types/node@22.20.1': + dependencies: + undici-types: 6.21.0 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + abstract-logging@2.0.1: {} + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + asynckit@0.4.0: {} + + atomic-sleep@1.0.0: {} + + avvio@9.3.0: + dependencies: + '@fastify/error': 4.2.0 + fastq: 1.20.1 + + base64-js@1.5.1: {} + + buffer-from@1.1.2: {} + + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + cluster-key-slot@1.1.1: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + cookie@1.1.1: {} + + core-util-is@1.0.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + delayed-stream@1.0.0: {} + + denque@2.1.0: {} + + dequal@2.0.3: {} + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + duplexify@3.7.1: + dependencies: + end-of-stream: 1.4.5 + inherits: 2.0.4 + readable-stream: 2.3.8 + stream-shift: 1.0.3 + + duplexify@4.1.3: + dependencies: + end-of-stream: 1.4.5 + inherits: 2.0.4 + readable-stream: 3.6.2 + stream-shift: 1.0.3 + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + fast-decode-uri-component@1.0.1: {} + + fast-deep-equal@3.1.3: {} + + fast-json-stringify@7.0.1: + dependencies: + '@fastify/merge-json-schemas': 0.2.1 + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + fast-uri: 4.1.0 + json-schema-ref-resolver: 3.0.0 + rfdc: 1.4.1 + + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-uri@3.1.3: {} + + fast-uri@4.1.0: {} + + fastify-plugin@5.1.0: {} + + fastify@5.10.0: + dependencies: + '@fastify/ajv-compiler': 4.0.5 + '@fastify/error': 4.2.0 + '@fastify/fast-json-stringify-compiler': 5.1.0 + '@fastify/proxy-addr': 5.1.0 + abstract-logging: 2.0.1 + avvio: 9.3.0 + fast-json-stringify: 7.0.1 + find-my-way: 9.6.0 + light-my-request: 6.6.0 + pino: 10.3.1 + process-warning: 5.0.0 + rfdc: 1.4.1 + secure-json-parse: 4.1.0 + semver: 7.8.5 + toad-cache: 3.7.4 + + fastq@1.20.1: + dependencies: + reusify: 1.1.0 + + find-my-way@9.6.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 5.1.1 + + form-data-encoder@1.7.2: {} + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + gopd@1.2.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + ieee754@1.2.1: {} + + inherits@2.0.4: {} + + ioredis@5.11.1: + dependencies: + '@ioredis/commands': 1.10.0 + cluster-key-slot: 1.1.1 + debug: 4.4.3 + denque: 2.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ipaddr.js@2.4.0: {} + + isarray@1.0.0: {} + + json-schema-ref-resolver@3.0.0: + dependencies: + dequal: 2.0.3 + + json-schema-traverse@1.0.0: {} + + light-my-request@6.6.0: + dependencies: + cookie: 1.1.1 + process-warning: 4.0.1 + set-cookie-parser: 2.7.2 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + minipass@7.1.3: {} + + ms@2.1.3: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + on-exit-leak-free@2.1.2: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + openai@4.104.0(zod@3.25.76): + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + zod: 3.25.76 + transitivePeerDependencies: + - encoding + + peek-stream@1.1.3: + dependencies: + buffer-from: 1.1.2 + duplexify: 3.7.1 + through2: 2.0.5 + + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@7.1.0: {} + + pino@10.3.1: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 4.2.0 + + process-nextick-args@2.0.1: {} + + process-warning@4.0.1: {} + + process-warning@5.0.0: {} + + process@0.11.10: {} + + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + pumpify@2.0.1: + dependencies: + duplexify: 4.1.3 + inherits: 2.0.4 + pump: 3.0.4 + + quick-format-unescaped@4.0.4: {} + + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + real-require@0.2.0: {} + + real-require@1.0.0: {} + + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + + require-from-string@2.0.2: {} + + ret@0.5.0: {} + + reusify@1.1.0: {} + + rfdc@1.4.1: {} + + safe-buffer@5.1.2: {} + + safe-buffer@5.2.1: {} + + safe-regex2@5.1.1: + dependencies: + ret: 0.5.0 + + safe-stable-stringify@2.5.0: {} + + secure-json-parse@4.1.0: {} + + semver@7.8.5: {} + + set-cookie-parser@2.7.2: {} + + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + + standard-as-callback@2.1.0: {} + + stream-shift@1.0.3: {} + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + thread-stream@4.2.0: + dependencies: + real-require: 1.0.0 + + through2@2.0.5: + dependencies: + readable-stream: 2.3.8 + xtend: 4.0.2 + + toad-cache@3.7.4: {} + + tr46@0.0.3: {} + + tsx@4.23.1: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@5.26.5: {} + + undici-types@6.21.0: {} + + util-deprecate@1.0.2: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + + wrappy@1.0.2: {} + + xtend@4.0.2: {} + + zod@3.25.76: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..dc6e08b --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,8 @@ +packages: + - "apps/*" + - "packages/*" + +# pnpm 11: dependency lifecycle scripts are blocked unless allowlisted. +# esbuild (via tsx) needs its postinstall to download the native binary. +allowBuilds: + esbuild: true diff --git a/scripts/accept.mjs b/scripts/accept.mjs new file mode 100644 index 0000000..d59e262 --- /dev/null +++ b/scripts/accept.mjs @@ -0,0 +1,78 @@ +/** + * Offline acceptance gate: unit tests + diag + DB seed invariants. + * Does not require live WeChat / real LLM. + */ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const dbPath = path.join(root, "data", "wechat-ai.db"); + +function run(cmd, args, env = {}) { + console.log(`\n> ${cmd} ${args.join(" ")}`); + const r = spawnSync(cmd, args, { + cwd: root, + env: { ...process.env, ...env }, + encoding: "utf8", + shell: true, + }); + if (r.stdout) process.stdout.write(r.stdout); + if (r.stderr) process.stderr.write(r.stderr); + if (r.status !== 0) { + console.error(`FAILED: ${cmd} ${args.join(" ")} (exit ${r.status})`); + process.exit(r.status ?? 1); + } +} + +console.log("=== WeChat-AI offline acceptance ==="); +console.log("root:", root); + +run("pnpm", ["test"]); + +// ensure schema + seed +run("pnpm", ["db:migrate"], { WECHAT_AI_DB_PATH: dbPath }); +run("pnpm", ["db:seed"], { WECHAT_AI_DB_PATH: dbPath }); +run("pnpm", ["diag"], { + WECHAT_AI_DB_PATH: dbPath, + LLM_API_KEY: process.env.LLM_API_KEY || "sk-accept-placeholder", + WECHAT_AI_TOKEN: process.env.WECHAT_AI_TOKEN || "accept-token", +}); + +// structural checks +const required = [ + "apps/api/public/admin.html", + "apps/api/public/chatflow.html", + "docs/runbook.md", + "docs/e2e-checklist.md", + "docs/admin-api.md", + "docs/ai-gateway.md", + "docs/chatflow.md", + "docs/adr/0001-ilink-direct.md", + "packages/ilink/src/client.ts", + "packages/core/src/chat-service.ts", + "packages/core/src/chatflow/engine.ts", + "packages/db/src/schema.ts", + "packages/db/src/llm-provider-repos.ts", + "huggingface/wechat-ai-tools/app.py", + "huggingface/wechat-ai-tools/Dockerfile", +]; +for (const rel of required) { + const p = path.join(root, rel); + if (!fs.existsSync(p)) { + console.error("MISSING file:", rel); + process.exit(1); + } + console.log(" ✓ file", rel); +} + +if (!fs.existsSync(dbPath)) { + console.error("DB missing after migrate/seed"); + process.exit(1); +} +console.log(" ✓ db", dbPath); + +console.log("\n=== OFFLINE ACCEPTANCE: PASS ==="); +console.log("True WeChat E2E still needs: pnpm ilink:login + real LLM key"); +console.log("See docs/e2e-checklist.md"); diff --git a/scripts/docker-build.mjs b/scripts/docker-build.mjs new file mode 100644 index 0000000..53c668b --- /dev/null +++ b/scripts/docker-build.mjs @@ -0,0 +1,198 @@ +/** + * Bump root package.json, pack OTA channel artifact, then Docker build. + * + * Default: pack only. Super-admin publishes via + * /admin → 部署节点 →「上传通道包」(files.json), then「更新」nodes. + * + * pnpm docker:build + * pnpm docker:up + * node scripts/docker-build.mjs -- docker build -t e51l6pwpe/wxai:latest . + */ +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { + applyRootVersion, + parseVersionArgs, + repoRoot, +} from "./lib/version.mjs"; + +function printHelp() { + console.log(`Usage: node scripts/docker-build.mjs [opts] [mode] [-- docker args...] + +Version: + (default) bump patch on root package.json + --bump patch|minor|major|none + --no-bump + --version X + --no-write + +OTA channel pack (default: pack → dist/release//files.json): + --no-channel skip pack + +Publish channel in browser (no CLI cookie): + /admin → 部署节点 → 上传通道包 → 选择 files.json → 更新节点 + +Mode: + (default) docker compose build + --up docker compose up -d --build + --raw docker build -t wechat-ai . + -- custom command after bump/pack + +Examples: + pnpm docker:build -- --bump minor -- docker build -t e51l6pwpe/wxai:latest . + pnpm docker:build -- -- docker build -t e51l6pwpe/wxai:latest . + pnpm docker:build -- --no-channel -- docker build -t wechat-ai . + +Note: everything after the FIRST \`--\` goes to pnpm; script flags such as +--bump/--no-channel come next, then a second \`--\` before a custom command. +`); +} + +/** + * Split `[opts] -- `. + * + * pnpm forwards its own `--` separator into argv, so + * `pnpm docker:build -- --bump minor -- docker build .` arrives here as + * ["--", "--bump", "minor", "--", "docker", ...]. A real passthrough command + * always starts with an executable name, never with `-`, so a leading `--` + * followed by a flag is pnpm's artifact and is dropped. That leaves + * `node scripts/docker-build.mjs -- docker build .` parsed as before. + */ +function splitPassthrough(argv) { + const args = argv.slice(2); + while (args[0] === "--" && args[1] && args[1].startsWith("-")) args.shift(); + const idx = args.indexOf("--"); + if (idx === -1) return { head: args, tail: [] }; + const head = args.slice(0, idx); + const tail = args.slice(idx + 1); + // `-- -- -- docker build .` leaves an extra separator in front of the + // command; a command never starts with `--`, so drop them. + while (tail[0] === "--") tail.shift(); + return { head, tail }; +} + +function runNodeScript(scriptRel, args) { + const script = path.join(repoRoot, "scripts", scriptRel); + console.log(`$ node ${scriptRel} ${args.join(" ")}`); + const r = spawnSync(process.execPath, [script, ...args], { + stdio: "inherit", + cwd: repoRoot, + env: process.env, + }); + if (r.error) { + console.error(r.error.message); + process.exit(1); + } + if ((r.status ?? 1) !== 0) process.exit(r.status ?? 1); +} + +function peelDockerBuildFlags(head) { + const out = { + mode: "compose-build", + pack: true, + rest: [], + }; + for (let i = 0; i < head.length; i++) { + const a = head[i]; + if (a === "--up") out.mode = "compose-up"; + else if (a === "--raw") out.mode = "raw"; + else if (a === "--build-only") out.mode = "compose-build"; + else if (a === "--no-channel" || a === "--skip-channel") out.pack = false; + else if (a === "--pack-only") out.pack = true; + // Reject removed CLI push flags with a clear message + else if ( + a === "--push" || + a === "--cookie" || + a === "-c" || + a === "--base" || + a === "-b" || + a === "--no-current" + ) { + console.error( + `Removed flag: ${a}\n` + + "Channel publish is web-only: /admin → 部署节点 → 上传通道包 (files.json).\n" + + "Do not use WA_SESSION_COOKIE / --cookie / --push.", + ); + process.exit(1); + } else out.rest.push(a); + } + return out; +} + +function main() { + const { head, tail } = splitPassthrough(process.argv); + const peeled = peelDockerBuildFlags(head); + + let parsed; + try { + parsed = parseVersionArgs(["node", "docker-build", ...peeled.rest], 2); + } catch (e) { + console.error(String(e?.message || e)); + process.exit(1); + } + + if (parsed.help) { + printHelp(); + process.exit(0); + } + + const forward = parsed.rest; + + let version; + try { + ({ version } = applyRootVersion({ + version: parsed.version, + bump: parsed.bump, + write: parsed.write, + })); + } catch (e) { + console.error(String(e?.message || e)); + process.exit(1); + } + + if (peeled.pack) { + runNodeScript("release-pack.mjs", ["--no-bump"]); + const filesJson = path.join( + repoRoot, + "dist", + "release", + version, + "files.json", + ); + console.log(`[channel] packed → ${filesJson}`); + console.log( + `[channel] next: /admin → 部署节点 → 上传通道包 → 选择 files.json → 更新节点`, + ); + } else { + console.log("[channel] skipped (--no-channel)"); + } + + /** @type {string[]} */ + let cmd; + let mode = peeled.mode; + if (tail.length > 0) { + cmd = tail; + mode = "custom"; + } else if (mode === "compose-up") { + cmd = ["docker", "compose", "up", "-d", "--build", ...forward]; + } else if (mode === "raw") { + cmd = ["docker", "build", "-t", "wechat-ai", ".", ...forward]; + } else { + cmd = ["docker", "compose", "build", ...forward]; + } + + console.log(`$ ${cmd.join(" ")}`); + const r = spawnSync(cmd[0], cmd.slice(1), { + stdio: "inherit", + shell: process.platform === "win32", + env: process.env, + cwd: repoRoot, + }); + if (r.error) { + console.error(r.error.message); + process.exit(1); + } + process.exit(r.status ?? 1); +} + +main(); diff --git a/scripts/lib/version.mjs b/scripts/lib/version.mjs new file mode 100644 index 0000000..c3e9854 --- /dev/null +++ b/scripts/lib/version.mjs @@ -0,0 +1,147 @@ +/** + * Shared root package.json version helpers for release-pack / docker-build. + */ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../..", +); +export const rootPkgPath = path.join(repoRoot, "package.json"); + +/** + * Bump semver core (major.minor.patch). Prerelease/build metadata is dropped. + * @param {string} version + * @param {"patch"|"minor"|"major"|"none"} level + */ +export function bumpSemver(version, level) { + const m = String(version || "0.0.0") + .trim() + .match(/^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/); + if (!m) { + throw new Error( + `cannot bump non-semver version "${version}" (use --version X)`, + ); + } + let major = Number(m[1]); + let minor = Number(m[2]); + let patch = Number(m[3]); + if (level === "major") { + major += 1; + minor = 0; + patch = 0; + } else if (level === "minor") { + minor += 1; + patch = 0; + } else if (level === "patch") { + patch += 1; + } else if (level !== "none") { + throw new Error(`unknown bump level: ${level}`); + } + return `${major}.${minor}.${patch}`; +} + +export function readRootPackage() { + return JSON.parse(fs.readFileSync(rootPkgPath, "utf8")); +} + +export function writeRootPackageVersion(pkg, version) { + const next = { ...pkg, version }; + const raw = fs.readFileSync(rootPkgPath, "utf8"); + const replaced = raw.replace( + /("version"\s*:\s*")([^"]*)(")/, + `$1${version}$3`, + ); + if (replaced !== raw && /"version"\s*:/.test(raw)) { + fs.writeFileSync(rootPkgPath, replaced); + } else { + fs.writeFileSync(rootPkgPath, `${JSON.stringify(next, null, 2)}\n`); + } +} + +/** + * Resolve next version and optionally write root package.json. + * @param {{ version?: string|null, bump?: string, write?: boolean, log?: (s: string) => void }} opts + * @returns {{ prevVersion: string, version: string, wrote: boolean }} + */ +export function applyRootVersion(opts = {}) { + const bump = opts.bump ?? "patch"; + const write = opts.write !== false; + const log = opts.log ?? console.log; + const pkg = readRootPackage(); + const prevVersion = String(pkg.version || "0.0.0"); + + let version = opts.version ? String(opts.version).trim() : null; + if (!version) { + if (bump === "none") version = prevVersion; + else version = bumpSemver(prevVersion, bump); + } + + if (!/^[0-9A-Za-z][0-9A-Za-z._+-]*$/.test(version)) { + throw new Error(`invalid version: ${version}`); + } + + let wrote = false; + if (write && version !== prevVersion) { + writeRootPackageVersion(pkg, version); + wrote = true; + log(`version ${prevVersion} → ${version} (wrote package.json)`); + } else if (version === prevVersion) { + log(`version ${version} (unchanged)`); + } else { + log( + `version ${prevVersion} → ${version} (not written to package.json)`, + ); + } + + return { prevVersion, version, wrote }; +} + +/** + * Parse common version flags from argv slice (mutates by consuming). + * Shared by release-pack / docker-build. + * @param {string[]} argv full process.argv + * @param {number} start index to start (default 2) + * @returns {{ version: string|null, bump: string, write: boolean, rest: string[], help: boolean }} + */ +export function parseVersionArgs(argv, start = 2) { + const out = { + version: null, + bump: "patch", + write: true, + rest: [], + help: false, + }; + for (let i = start; i < argv.length; i++) { + const a = argv[i]; + if (a === "--version" || a === "-v") { + out.version = argv[++i]; + out.bump = "none"; + } else if (a === "--bump") { + const level = String(argv[++i] || "") + .trim() + .toLowerCase(); + if (!["patch", "minor", "major", "none"].includes(level)) { + throw new Error( + `invalid --bump (use patch|minor|major|none): ${level}`, + ); + } + out.bump = level; + } else if (a === "--no-bump") { + out.bump = "none"; + } else if (a === "--no-write") { + out.write = false; + } else if (a === "--help" || a === "-h") { + out.help = true; + } else if (a === "--") { + // End-of-options marker. pnpm forwards its own `--` into argv, so + // `pnpm release:pack -- --bump minor` would otherwise fail with + // "unknown argument: --". + } else { + out.rest.push(a); + } + } + return out; +} diff --git a/scripts/release-pack.mjs b/scripts/release-pack.mjs new file mode 100644 index 0000000..a331768 --- /dev/null +++ b/scripts/release-pack.mjs @@ -0,0 +1,246 @@ +/** + * Build an OTA release pack from the monorepo (file list + sha256 + bodies). + * + * pnpm release:pack + * node scripts/release-pack.mjs + * node scripts/release-pack.mjs --bump minor + * node scripts/release-pack.mjs --version 0.3.0 --out dist/release + * node scripts/release-pack.mjs --no-bump + * + * Default: bump root package.json patch (0.2.0 → 0.2.1), write it back, then pack + * so OTA files + runtime version stay in sync without manual APP_VERSION. + * + * Writes: + * //manifest.json + * //files.json (paths + sha + base64 for push) + * package.json version (unless --no-write) + */ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { applyRootVersion, parseVersionArgs, repoRoot } from "./lib/version.mjs"; + +const root = repoRoot; + +const ROOT_FILES = [ + "package.json", + "pnpm-workspace.yaml", + "pnpm-lock.yaml", + "tsconfig.base.json", +]; + +const DIR_PREFIXES = [ + "apps/api", + "packages/core", + "packages/db", + "packages/ilink", + "packages/llm", + "scripts", +]; + +const DENY_SEG = new Set([ + "node_modules", + "data", + ".git", + ".wa-update-staging", + ".wa-backup", + "dist", + "coverage", +]); + +const INSTALL_TRIGGERS = new Set([ + "pnpm-lock.yaml", + "package.json", + "pnpm-workspace.yaml", + "apps/api/package.json", + "packages/core/package.json", + "packages/db/package.json", + "packages/ilink/package.json", + "packages/llm/package.json", +]); + +function parseArgs(argv) { + let parsed; + try { + parsed = parseVersionArgs(argv, 2); + } catch (e) { + console.error(String(e?.message || e)); + process.exit(1); + } + + const out = { + version: parsed.version, + bump: parsed.bump, + write: parsed.write, + outDir: path.join(root, "dist", "release"), + help: parsed.help, + }; + + for (let i = 0; i < parsed.rest.length; i++) { + const a = parsed.rest[i]; + if (a === "--out" || a === "-o") { + out.outDir = path.resolve(parsed.rest[++i]); + } else { + console.error("unknown argument:", a); + process.exit(1); + } + } + + if (out.help) { + console.log(`Usage: node scripts/release-pack.mjs [options] + + (default) bump patch on root package.json, write, then pack + --bump patch|minor|major|none + --no-bump keep current package.json version + --version X set exact version (implies no auto-bump) + --no-write do not write package.json (pack only) + --out dir output root (default dist/release) +`); + process.exit(0); + } + + return out; +} + +function sha256(buf) { + return createHash("sha256").update(buf).digest("hex"); +} + +function isDenied(relPosix) { + const lower = relPosix.toLowerCase(); + const base = lower.split("/").pop() || ""; + if (base.startsWith(".env")) return true; + if (base === ".ds_store") return true; + if (/\.(db|db-wal|db-shm|log|bak)$/i.test(base)) return true; + for (const s of lower.split("/")) { + if (DENY_SEG.has(s)) return true; + } + return false; +} + +function isAllowed(relPosix) { + if (!relPosix || isDenied(relPosix)) return false; + if (ROOT_FILES.includes(relPosix)) return true; + for (const dir of DIR_PREFIXES) { + if (relPosix !== dir && !relPosix.startsWith(dir + "/")) continue; + if (dir.startsWith("packages/")) { + const rest = relPosix.slice(dir.length + 1); + if (rest === "package.json" || rest === "tsconfig.json") return true; + if (rest.startsWith("src/")) return true; + return false; + } + return true; + } + return false; +} + +function walk(absDir, relPrefix, list) { + let entries; + try { + entries = fs.readdirSync(absDir, { withFileTypes: true }); + } catch { + return; + } + for (const ent of entries) { + const name = ent.name; + if (DENY_SEG.has(name.toLowerCase())) continue; + const rel = (relPrefix ? `${relPrefix}/${name}` : name).replace(/\\/g, "/"); + const abs = path.join(absDir, name); + if (ent.isDirectory()) walk(abs, rel, list); + else if (ent.isFile() && isAllowed(rel)) list.push(rel); + } +} + +function main() { + const args = parseArgs(process.argv); + + let version; + try { + ({ version } = applyRootVersion({ + version: args.version, + bump: args.bump, + write: args.write, + })); + } catch (e) { + console.error(String(e?.message || e)); + process.exit(1); + } + + const list = []; + for (const f of ROOT_FILES) { + const abs = path.join(root, f); + if (fs.existsSync(abs) && isAllowed(f)) list.push(f); + } + for (const dir of DIR_PREFIXES) { + const abs = path.join(root, ...dir.split("/")); + if (fs.existsSync(abs)) walk(abs, dir, list); + } + list.sort(); + + const files = []; + let totalBytes = 0; + let requiresInstall = false; + + for (const rel of list) { + const abs = path.join(root, ...rel.split("/")); + const buf = fs.readFileSync(abs); + const hash = sha256(buf); + files.push({ + path: rel, + sha256: hash, + size: buf.length, + dataBase64: buf.toString("base64"), + }); + totalBytes += buf.length; + if (INSTALL_TRIGGERS.has(rel)) requiresInstall = true; + } + + const packLines = files + .map((f) => `${f.path}:${f.sha256}`) + .sort() + .join("\n"); + const packSha256 = sha256(Buffer.from(packLines, "utf8")); + + const manifest = { + version, + createdAt: new Date().toISOString(), + files: files.map(({ path: p, sha256: h, size }) => ({ + path: p, + sha256: h, + size, + })), + requiresInstall, + totalBytes, + packSha256, + fileCount: files.length, + }; + + const outDir = path.join(args.outDir, version); + fs.mkdirSync(outDir, { recursive: true }); + fs.writeFileSync( + path.join(outDir, "manifest.json"), + JSON.stringify(manifest, null, 2), + ); + fs.writeFileSync( + path.join(outDir, "files.json"), + JSON.stringify({ + version, + files: files.map((f) => ({ + path: f.path, + sha256: f.sha256, + size: f.size, + dataBase64: f.dataBase64, + })), + }), + ); + + console.log(`Packed ${files.length} files (${totalBytes} bytes) → ${outDir}`); + console.log(`version=${version} packSha256=${packSha256.slice(0, 12)}…`); + console.log(`requiresInstall=${requiresInstall}`); + console.log( + `Next: /admin → 部署节点 → 上传通道包 → 选择 ${path.join(outDir, "files.json")}`, + ); +} + +main(); diff --git a/scripts/ui-preview.mjs b/scripts/ui-preview.mjs new file mode 100644 index 0000000..229900e --- /dev/null +++ b/scripts/ui-preview.mjs @@ -0,0 +1,460 @@ +/** + * UI preview harness: serves apps/api/public with mocked /api/v1/* data and + * captures desktop + mobile, light + dark screenshots of /app and /admin. + * + * Usage: node scripts/ui-preview.mjs [--out scripts/ui-shots] + * Needs: playwright (root devDep preferred; falls back to the npx cache) + * and its chromium browser (`npx playwright install chromium`). + */ +import http from "node:http"; +import { readFile, mkdir, readdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, ".."); +const PUBLIC_DIR = path.join(ROOT, "apps", "api", "public"); +const OUT_DIR = path.resolve( + ROOT, + process.argv.includes("--out") + ? process.argv[process.argv.indexOf("--out") + 1] + : "scripts/ui-shots", +); +const PORT = 8891; + +// ── Playwright loader (node_modules first, then npx cache) ────────── +async function loadPlaywright() { + try { + return await import("playwright"); + } catch (_) {} + try { + return await import("playwright-core"); + } catch (_) {} + const cacheRoot = path.join( + process.env.LOCALAPPDATA || path.join(process.env.USERPROFILE || "", "AppData", "Local"), + "npm-cache", + "_npx", + ); + if (existsSync(cacheRoot)) { + for (const entry of await readdir(cacheRoot)) { + for (const pkg of ["playwright", "playwright-core"]) { + const candidate = path.join(cacheRoot, entry, "node_modules", pkg, "index.mjs"); + if (existsSync(candidate)) { + return await import(`file://${candidate.replace(/\\/g, "/")}`); + } + } + } + } + throw new Error( + "playwright not found — run `pnpm add -w -D playwright && npx playwright install chromium`", + ); +} + +// ── Fixture data ──────────────────────────────────────────────────── +const todayStr = new Date().toISOString().slice(0, 10); +const dayN = (n) => new Date(Date.now() - n * 86400000).toISOString().slice(0, 10); +const iso = (n) => new Date(Date.now() - n * 3600000).toISOString(); + +const ME_ADMIN = { + user: { + id: "u_1001", + username: "marina", + name: "晚风 Marina", + avatarUrl: null, + isAdmin: true, + trustLevel: 3, + }, +}; + +const MY_BOTS = { + bots: [ + { id: "bot_xt01", displayName: "小桃", accountRef: "wxid_peach01", status: "active", ownerUserId: "u_1001", hasToken: true }, + { id: "bot_xg02", displayName: "雪糕", accountRef: "wxid_icecream", status: "active", ownerUserId: "u_1001", hasToken: false }, + ], +}; + +const MY_PEERS = { + peers: [ + { bot_account_id: "bot_xt01", peer_id: "wxid_friend_a01", approved: 1, created_at: iso(30), personaId: "p_fuhei", personaSlug: "fuhei-xuejie", personaName: "腹黑学姐" }, + { bot_account_id: "bot_xt01", peer_id: "wxid_friend_b02", approved: 0, created_at: iso(4), personaId: null, personaSlug: null, personaName: null }, + { bot_account_id: "bot_xg02", peer_id: "wxid_friend_c03", approved: 1, created_at: iso(52), personaId: "p_catgirl", personaSlug: "official-catgirl", personaName: "温柔猫娘" }, + ], +}; + +const PERSONAS = [ + { id: "p_catgirl", slug: "official-catgirl", displayName: "温柔猫娘", description: "软萌粘人的猫娘，说话带喵，会撒娇会关心人。", tags: ["官方", "治愈", "猫娘"], visibility: "public", ownerUserId: "system", useCount: 128, enabled: true, isDefault: true, updatedAt: iso(200) }, + { id: "p_fuhei", slug: "fuhei-xuejie", displayName: "腹黑学姐", description: "表面温柔实则腹黑的学姐，喜欢逗人，偶尔真情流露。", tags: ["恋爱", "腹黑"], visibility: "public", ownerUserId: "u_1001", useCount: 86, enabled: true, isDefault: false, updatedAt: iso(80) }, + { id: "p_ceo", slug: "gaoleng-zongcai", displayName: "高冷总裁", description: "毒舌高冷但心软的总裁，口是心非，行动派宠人。", tags: ["恋爱", "高冷"], visibility: "public", ownerUserId: "u_1002", useCount: 54, enabled: true, isDefault: false, updatedAt: iso(120) }, + { id: "p_genki", slug: "yuanqi-shaonv", displayName: "元气少女", description: "永远活力满满的元气少女，自带阳光和感叹号！", tags: ["日常", "元气"], visibility: "public", ownerUserId: "u_1003", useCount: 40, enabled: true, isDefault: false, updatedAt: iso(60) }, + { id: "p_radio", slug: "shenye-diantai", displayName: "深夜电台主播", description: "凌晨两点的电台主播，声音温柔，善于倾听与安慰。", tags: ["治愈", "夜聊"], visibility: "public", ownerUserId: "u_1002", useCount: 33, enabled: true, isDefault: false, updatedAt: iso(30) }, + { id: "p_dushé", slug: "dushe-guimi", displayName: "毒舌闺蜜", description: "嘴上不饶人心里最护你的闺蜜，吐槽一流。", tags: ["日常", "毒舌"], visibility: "private", ownerUserId: "u_1001", useCount: 12, enabled: true, isDefault: false, updatedAt: iso(10) }, +]; + +const SQUARE = { + personas: PERSONAS.filter((p) => p.visibility === "public").map((p) => ({ + ...p, + inLibrary: ["p_catgirl", "p_fuhei"].includes(p.id), + systemPromptPreview: "你是「{{bot_name}}」……", + })), + total: 5, page: 1, limit: 40, +}; + +const MY_PERSONAS = { + library: PERSONAS.slice(0, 3).map((p) => ({ ...p })), + created: PERSONAS.filter((p) => p.ownerUserId === "u_1001").map((p) => ({ ...p })), +}; + +const usageDay = (day, seed) => { + const t = 240000 + seed * 91000; + return { + day, + prompt_tokens: Math.round(t * 0.72), + completion_tokens: Math.round(t * 0.28), + total_tokens: t, + requests: 120 + seed * 37, + by_user: { + u_1001: { total_tokens: Math.round(t * 0.4), requests: 60 + seed * 9, username: "marina" }, + u_1002: { total_tokens: Math.round(t * 0.35), requests: 40 + seed * 12, username: "azhe" }, + u_1003: { total_tokens: Math.round(t * 0.25), requests: 20 + seed * 16, username: "tianmei" }, + }, + by_bot: { + bot_xt01: { total_tokens: Math.round(t * 0.5), requests: 70 + seed * 15, display_name: "小桃" }, + bot_mm04: { total_tokens: Math.round(t * 0.3), requests: 30 + seed * 12, display_name: "momo" }, + bot_xg02: { total_tokens: Math.round(t * 0.2), requests: 20 + seed * 10, display_name: "雪糕" }, + }, + }; +}; + +const SNAPSHOT = { + bots: 4, activeBots: 3, personas: 6, defaultPersona: "official-catgirl", + peers: 9, approvedPeers: 7, unapprovedPeers: 2, assignments: 5, + // These three are real counts now (they used to be hardcoded 0 in repos.ts). + // deepStats=false would mean "not measured" rather than "none". + messages: 412, memories: 37, users: 5, deepStats: true, +}; + +const SAFE_CONFIG = { + publicBaseUrl: "https://wa.example.com", + workerEnabled: true, + llmModel: "deepseek-v3", + llmBaseUrl: "https://api.example.com/v1", + multiBubbleJson: true, + splitReply: true, + allowUnapproved: false, + maxReplyChunks: 4, + maxChunkChars: 120, +}; + +const ADMIN_DASHBOARD = { + snapshot: SNAPSHOT, + usage: { today: usageDay(todayStr, 6), yesterday: usageDay(dayN(1), 4) }, + workers: ["bot_xt01", "bot_mm04"], + workerBots: [ + { id: "bot_xt01", displayName: "小桃", status: "active" }, + { id: "bot_mm04", displayName: "momo", status: "active" }, + ], + redisOk: true, + safeConfig: SAFE_CONFIG, +}; + +const ADMIN_BOTS = { + bots: [ + { id: "bot_xt01", displayName: "小桃", ownerUserId: "u_1001", ownerUsername: "marina", ownerName: "晚风 Marina", status: "active", accountRef: "wxid_peach01", workerActive: true, hasToken: true, peerCount: 4, unapprovedPeerCount: 0, updatedAt: iso(2) }, + { id: "bot_mm04", displayName: "momo", ownerUserId: "u_1002", ownerUsername: "azhe", ownerName: "阿哲", status: "active", accountRef: "wxid_momo04", workerActive: true, hasToken: true, peerCount: 3, unapprovedPeerCount: 2, updatedAt: iso(6) }, + { id: "bot_xg02", displayName: "雪糕", ownerUserId: "u_1001", ownerUsername: "marina", ownerName: "晚风 Marina", status: "active", accountRef: "wxid_icecream", workerActive: false, hasToken: false, peerCount: 2, unapprovedPeerCount: 0, updatedAt: iso(30) }, + { id: "bot_ay03", displayName: "阿云", ownerUserId: "u_1003", ownerUsername: "tianmei", ownerName: "甜妹研究所", status: "inactive", accountRef: "wxid_cloud03", workerActive: false, hasToken: true, peerCount: 0, unapprovedPeerCount: 0, updatedAt: iso(96) }, + ], +}; + +const ADMIN_USERS = { + users: [ + { id: "u_1001", username: "marina", name: "晚风 Marina", isAdmin: true, trustLevel: 3, avatarUrl: null, botCount: 2, createdAt: iso(24 * 90) }, + { id: "u_1002", username: "azhe", name: "阿哲", isAdmin: false, trustLevel: 2, avatarUrl: null, botCount: 1, createdAt: iso(24 * 60) }, + { id: "u_1003", username: "tianmei", name: "甜妹研究所", isAdmin: false, trustLevel: 2, avatarUrl: null, botCount: 1, createdAt: iso(24 * 30) }, + { id: "u_1004", username: "nightowl", name: "夜猫子", isAdmin: false, trustLevel: 1, avatarUrl: null, botCount: 0, createdAt: iso(24 * 12) }, + { id: "u_1005", username: "lucas", name: "Lucas", isAdmin: false, trustLevel: 4, avatarUrl: null, botCount: 0, createdAt: iso(24 * 5) }, + ], +}; + +const ADMIN_PERSONAS = { + total: PERSONAS.length, + personas: PERSONAS.map((p) => ({ ...p })), +}; + +const ADMIN_AUDIT = { + logs: [ + { id: "a10", action: "admin_workers_restart_all", actor_user_id: "u_1001", meta: { started: 2, skipped: 2 }, created_at: iso(1) }, + { id: "a09", action: "peer_approved", actor_user_id: "u_1001", meta: { botAccountId: "bot_xt01", peerId: "wxid_friend_b02" }, created_at: iso(3) }, + { id: "a08", action: "bot_renamed", actor_user_id: "u_1002", meta: { botId: "bot_mm04", displayName: "momo" }, created_at: iso(8) }, + { id: "a07", action: "persona_published_square", actor_user_id: "u_1001", meta: { id: "p_fuhei", visibility: "public" }, created_at: iso(20) }, + { id: "a06", action: "admin_persona_set_default", actor_user_id: "u_1001", meta: { id: "p_catgirl" }, created_at: iso(26) }, + { id: "a05", action: "admin_peer_approve_all", actor_user_id: "u_1001", meta: { approved: 3 }, created_at: iso(40) }, + { id: "a04", action: "bot_deleted", actor_user_id: "u_1003", meta: { botId: "bot_old99" }, created_at: iso(55) }, + { id: "a03", action: "persona_added_lib", actor_user_id: "u_1002", meta: { personaId: "p_radio" }, created_at: iso(70) }, + { id: "a02", action: "admin_seed_personas", actor_user_id: "u_1001", meta: { before: 4, after: 6, added: 2 }, created_at: iso(88) }, + { id: "a01", action: "bot_created", actor_user_id: "u_1001", meta: { botId: "bot_xt01" }, created_at: iso(110) }, + ], +}; + +const ADMIN_PEERS = { + total: 2, + peers: [ + { botAccountId: "bot_mm04", peerId: "wxid_new_friend_07", approved: false, botName: "momo", ownerUserId: "u_1002", ownerUsername: "azhe", personaId: null, createdAt: iso(2) }, + { botAccountId: "bot_mm04", peerId: "wxid_new_friend_08", approved: false, botName: "momo", ownerUserId: "u_1002", ownerUsername: "azhe", personaId: null, createdAt: iso(5) }, + ], +}; + +const ADMIN_SYSTEM = { + snapshot: SNAPSHOT, + workers: ["bot_xt01", "bot_mm04"], + redisOk: true, + uptimeSec: 2 * 86400 + 3 * 3600 + 1200, + node: process.version, + safeConfig: SAFE_CONFIG, +}; + +/** Route an intercepted /api/v1/* request to fixture JSON. + * mode: "ok" (authed, healthy) | "anon" (401 on auth/me) | "botsError" (bots/peers 500) */ +function mockApi(pathname, query, mode) { + if (pathname === "/api/v1/auth/me") { + return mode === "anon" + ? { status: 401, body: { error: "unauthorized" } } + : { status: 200, body: ME_ADMIN }; + } + if (mode === "botsError" && (pathname === "/api/v1/me/bots" || pathname === "/api/v1/me/peers")) { + return { status: 500, body: { error: "mock backend failure" } }; + } + if (pathname === "/api/v1/me/bots/login/start" || pathname === "/api/v1/me/bots/login/login_mock") { + return { + status: 200, + body: { + session: { + sessionId: "login_mock", + displayName: "小桃", + ownerUserId: "u_1001", + status: "wait_scan", + mode: "create", + qrcode: "mock", + openUrl: "https://work.weixin.qq.com/ca/mock-qr-target", + message: "请用微信扫描二维码（或打开下方链接）", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + }, + }; + } + const table = { + "/api/v1/me/bots": MY_BOTS, + "/api/v1/me/peers": MY_PEERS, + "/api/v1/me/personas": MY_PERSONAS, + "/api/v1/square/personas": SQUARE, + "/api/v1/admin/dashboard": ADMIN_DASHBOARD, + "/api/v1/admin/bots": ADMIN_BOTS, + "/api/v1/admin/users": ADMIN_USERS, + "/api/v1/admin/personas": ADMIN_PERSONAS, + "/api/v1/admin/audit": ADMIN_AUDIT, + "/api/v1/admin/peers": ADMIN_PEERS, + "/api/v1/admin/system": ADMIN_SYSTEM, + }; + if (pathname === "/api/v1/admin/usage") { + if (query.get("days")) { + const n = Math.min(Number(query.get("days")) || 7, 30); + return { status: 200, body: { days: Array.from({ length: n }, (_, i) => usageDay(dayN(i), Math.max(1, 6 - i)) ) } }; + } + return { status: 200, body: { usage: usageDay(query.get("day") || todayStr, 6) } }; + } + if (pathname.startsWith("/api/v1/square/personas/")) { + const id = pathname.split("/").pop(); + const p = PERSONAS.find((x) => x.id === id) || PERSONAS[0]; + return { status: 200, body: { persona: { ...p, systemPrompt: "你是「{{bot_name}}」，一位" + p.description, inLibrary: false } } }; + } + if (table[pathname]) return { status: 200, body: table[pathname] }; + return { status: 200, body: { ok: true } }; +} + +/** Grey placeholder standing in for the external QR image service. */ +const QR_PLACEHOLDER_SVG = ` + + + + + QR MOCK +`; + +// ── Static server ─────────────────────────────────────────────────── +const MIME = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript", + ".css": "text/css", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + ".json": "application/json", +}; + +function startServer() { + const server = http.createServer(async (req, res) => { + try { + let p = new URL(req.url, `http://127.0.0.1:${PORT}`).pathname; + if (p === "/" || p === "/app") p = "/app.html"; + if (p === "/admin") p = "/admin.html"; + const file = path.join(PUBLIC_DIR, p.replace(/^\/+/, "")); + if (!file.startsWith(PUBLIC_DIR)) throw new Error("traversal"); + const data = await readFile(file); + res.writeHead(200, { "Content-Type": MIME[path.extname(file)] || "application/octet-stream" }); + res.end(data); + } catch { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end('{"error":"not found"}'); + } + }); + return new Promise((resolve) => server.listen(PORT, "127.0.0.1", () => resolve(server))); +} + +// ── Capture ───────────────────────────────────────────────────────── +const VIEWPORTS = { + mobile: { width: 393, height: 852, deviceScaleFactor: 2, isMobile: true, hasTouch: true }, + desktop: { width: 1440, height: 900, deviceScaleFactor: 1 }, +}; + +async function newPage(browser, { viewport, theme, mode }) { + const context = await browser.newContext({ + viewport: { width: viewport.width, height: viewport.height }, + deviceScaleFactor: viewport.deviceScaleFactor, + isMobile: !!viewport.isMobile, + hasTouch: !!viewport.hasTouch, + locale: "zh-CN", + }); + await context.addInitScript((t) => { + try { localStorage.setItem("wa_theme", t); } catch (_) {} + }, theme); + await context.route("**/api/v1/**", (route) => { + const u = new URL(route.request().url()); + const { status, body } = mockApi(u.pathname, u.searchParams, mode); + route.fulfill({ status, contentType: "application/json", body: JSON.stringify(body) }); + }); + // External QR image service → local placeholder; other external hosts → abort + await context.route(/^https?:\/\/api\.qrserver\.com\//, (route) => + route.fulfill({ contentType: "image/svg+xml", body: QR_PLACEHOLDER_SVG }), + ); + await context.route(/^https?:\/\/(?!127\.0\.0\.1|api\.qrserver\.com)/, (route) => route.abort()); + const page = await context.newPage(); + return { context, page }; +} + +async function shoot(page, name, { fullPage = true } = {}) { + await page.waitForTimeout(450); + await page.screenshot({ path: path.join(OUT_DIR, `${name}.png`), fullPage }); + console.log(" ✓", name); +} + +async function main() { + await mkdir(OUT_DIR, { recursive: true }); + const server = await startServer(); + const { chromium } = await loadPlaywright(); + const browser = await chromium.launch(); + const base = `http://127.0.0.1:${PORT}`; + + console.log("app.html:"); + { + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "light", mode: "anon" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await shoot(page, "app-gate-mobile-light"); + await context.close(); + } + for (const [vpName, vp] of Object.entries(VIEWPORTS)) { + for (const theme of ["light", "dark"]) { + const { context, page } = await newPage(browser, { viewport: vp, theme, mode: "ok" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await shoot(page, `app-bots-${vpName}-${theme}`); + if (theme === "light") { + await page.click('#mainTabs button[data-pane="square"]'); + await page.waitForTimeout(350); + await shoot(page, `app-square-${vpName}-${theme}`); + await page.click('#mainTabs button[data-pane="mine"]'); + await page.waitForTimeout(350); + await shoot(page, `app-mine-${vpName}-${theme}`); + } + await context.close(); + } + } + + console.log("app.html interactions:"); + { + // QR login panel (wait_scan with mocked QR image) + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "light", mode: "ok" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await page.click("#addBot"); + await page.waitForTimeout(900); + await shoot(page, "app-qr-waitscan-mobile-light"); + await context.close(); + } + { + // Persona detail modal (bottom sheet on mobile) + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "light", mode: "ok" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await page.click('#mainTabs button[data-pane="square"]'); + await page.waitForTimeout(500); + await page.click("[data-detail]"); + await page.waitForTimeout(500); + await shoot(page, "app-modal-detail-mobile-light", { fullPage: false }); + await context.close(); + } + { + // Destructive confirm card + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "light", mode: "ok" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await page.click("[data-del]"); + await page.waitForTimeout(400); + await shoot(page, "app-confirm-mobile-light", { fullPage: false }); + await context.close(); + } + { + // Data-load failure → error rows + retry buttons (auth still ok) + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "light", mode: "botsError" }); + await page.goto(`${base}/app`, { waitUntil: "networkidle" }); + await page.waitForTimeout(400); + await shoot(page, "app-error-mobile-light"); + await context.close(); + } + + console.log("admin.html:"); + const adminTabs = ["dash", "usage", "users", "bots", "personas", "audit", "system"]; + for (const [vpName, vp] of Object.entries(VIEWPORTS)) { + const { context, page } = await newPage(browser, { viewport: vp, theme: "light", mode: "ok" }); + await page.goto(`${base}/admin`, { waitUntil: "networkidle" }); + await page.waitForTimeout(600); + for (const tab of adminTabs) { + await page.click(`.sidebar button[data-tab="${tab}"]`); + await page.waitForTimeout(500); + await shoot(page, `admin-${tab}-${vpName}-light`); + } + await context.close(); + } + { + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.desktop, theme: "dark", mode: "ok" }); + await page.goto(`${base}/admin`, { waitUntil: "networkidle" }); + await page.waitForTimeout(600); + await shoot(page, "admin-dash-desktop-dark"); + await context.close(); + } + { + const { context, page } = await newPage(browser, { viewport: VIEWPORTS.mobile, theme: "dark", mode: "ok" }); + await page.goto(`${base}/admin`, { waitUntil: "networkidle" }); + await page.waitForTimeout(600); + await shoot(page, "admin-dash-mobile-dark"); + await context.close(); + } + + await browser.close(); + server.close(); + console.log("done →", OUT_DIR); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..b134129 --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "resolveJsonModule": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + } +}