Merge pull request #403 from lintonxue00/main

添加 Discord/HuggingFace 渠道支持,优化 TG 分块
This commit is contained in:
叁月柒
2025-12-31 08:43:00 +08:00
committed by GitHub
170 changed files with 2040 additions and 19 deletions
+4 -1
View File
@@ -2,4 +2,7 @@
data/* data/*
node_modules node_modules
.DS_Store .DS_Store
docs/Sanyue-ImgHub docs/Sanyue-ImgHub
docs/Sanyue-ImgHub源码
docs/CloudFlare-ImgBed源码
docs/discord-image
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
+145
View File
@@ -0,0 +1,145 @@
/**
* HuggingFace 大文件提交 API
*
* 在前端直接上传文件到 S3 调用此 API 提交 LFS 文件引用
*/
import { HuggingFaceAPI } from '../../utils/huggingfaceAPI.js';
import { fetchUploadConfig } from '../../utils/sysConfig.js';
import { getDatabase } from '../../utils/databaseAdapter.js';
import { moderateContent, endUpload } from '../../upload/uploadTools.js';
export async function onRequestPost(context) {
const { request, env, waitUntil } = context;
try {
// 验证认证码
const authCode = request.headers.get('authcode');
if (env.AUTH_CODE && authCode !== env.AUTH_CODE) {
return new Response(JSON.stringify({ error: 'Unauthorized' }), {
status: 401,
headers: { 'Content-Type': 'application/json' }
});
}
const body = await request.json();
const { fullId, filePath, sha256, fileSize, fileName, channelName, multipartParts } = body;
if (!fullId || !filePath || !sha256 || !fileSize) {
return new Response(JSON.stringify({
error: 'Missing required fields: fullId, filePath, sha256, fileSize'
}), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
// 获取 HuggingFace 配置
const uploadConfig = await fetchUploadConfig(env);
const hfSettings = uploadConfig.huggingface;
if (!hfSettings || !hfSettings.channels || hfSettings.channels.length === 0) {
return new Response(JSON.stringify({ error: 'No HuggingFace channel configured' }), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
// 选择渠道
let hfChannel;
if (channelName) {
hfChannel = hfSettings.channels.find(c => c.name === channelName);
}
if (!hfChannel) {
hfChannel = hfSettings.channels[0];
}
if (!hfChannel || !hfChannel.token || !hfChannel.repo) {
return new Response(JSON.stringify({ error: 'HuggingFace channel not properly configured' }), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
const huggingfaceAPI = new HuggingFaceAPI(hfChannel.token, hfChannel.repo, hfChannel.isPrivate || false);
// 如果有 multipart parts,需要先完成 multipart 上传
if (multipartParts && multipartParts.length > 0) {
console.log('Completing multipart upload...');
// multipartParts 格式: [{ partNumber, etag, completionUrl }]
// 这里需要调用 HuggingFace 的 multipart complete API
// 但由于前端已经完成了所有分片上传,这里只需要提交
}
// 提交 LFS 文件引用
console.log('Committing LFS file...');
const commitResult = await huggingfaceAPI.commitLfsFile(
filePath,
sha256,
fileSize,
`Upload ${fileName || fullId}`
);
console.log('Commit result:', JSON.stringify(commitResult));
// 构建文件 URL
const fileUrl = `https://huggingface.co/datasets/${hfChannel.repo}/resolve/main/${filePath}`;
// 构建 metadata
const metadata = {
FileName: fileName || fullId,
Channel: "HuggingFace",
ChannelName: hfChannel.name || "HuggingFace_env",
FileSize: (fileSize / 1024 / 1024).toFixed(2),
HfRepo: hfChannel.repo,
HfFilePath: filePath,
HfToken: hfChannel.token,
HfIsPrivate: hfChannel.isPrivate || false,
HfFileUrl: fileUrl,
TimeStamp: Date.now(),
Label: "None"
};
// 图像审查(公开仓库)
if (!hfChannel.isPrivate) {
try {
metadata.Label = await moderateContent(env, fileUrl);
} catch (e) {
console.warn('Content moderation failed:', e.message);
}
}
// 写入数据库
const db = getDatabase(env);
await db.put(fullId, "", { metadata });
// 结束上传(更新索引等)
// 构造 url 对象用于 endUpload
const url = new URL(request.url);
const uploadContext = {
env,
waitUntil,
uploadConfig,
url
};
waitUntil(endUpload(uploadContext, fullId, metadata));
// 返回成功响应
const returnLink = `/file/${fullId}`;
return new Response(JSON.stringify({
success: true,
src: returnLink,
fileUrl,
fullId
}), {
status: 200,
headers: { 'Content-Type': 'application/json' }
});
} catch (error) {
console.error('commitUpload error:', error.message);
return new Response(JSON.stringify({ error: error.message }), {
status: 500,
headers: { 'Content-Type': 'application/json' }
});
}
}
+105
View File
@@ -0,0 +1,105 @@
/**
* HuggingFace 大文件直传 API
*
* 流程
* 1. 前端计算 SHA256 和文件样本
* 2. 前端调用此 API 获取 LFS 上传 URL
* 3. 前端直接上传到 HuggingFace S3
* 4. 前端调用 commitUpload API 提交文件引用
*
* 这样可以绕过 CF Workers 100MB 请求体限制和 CPU 时间限制
*/
import { HuggingFaceAPI } from '../../utils/huggingfaceAPI.js';
import { fetchUploadConfig } from '../../utils/sysConfig.js';
export async function onRequestPost(context) {
const { request, env } = context;
try {
// 验证认证码
const authCode = request.headers.get('authcode');
if (env.AUTH_CODE && authCode !== env.AUTH_CODE) {
return new Response(JSON.stringify({ error: 'Unauthorized' }), {
status: 401,
headers: { 'Content-Type': 'application/json' }
});
}
const body = await request.json();
const { fileSize, fileName, sha256, fileSample, channelName } = body;
if (!fileSize || !fileName || !sha256 || !fileSample) {
return new Response(JSON.stringify({
error: 'Missing required fields: fileSize, fileName, sha256, fileSample'
}), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
// 获取 HuggingFace 配置
const uploadConfig = await fetchUploadConfig(env);
const hfSettings = uploadConfig.huggingface;
if (!hfSettings || !hfSettings.channels || hfSettings.channels.length === 0) {
return new Response(JSON.stringify({ error: 'No HuggingFace channel configured' }), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
// 选择渠道
let hfChannel;
if (channelName) {
hfChannel = hfSettings.channels.find(c => c.name === channelName);
}
if (!hfChannel) {
hfChannel = hfSettings.loadBalance?.enabled
? hfSettings.channels[Math.floor(Math.random() * hfSettings.channels.length)]
: hfSettings.channels[0];
}
if (!hfChannel || !hfChannel.token || !hfChannel.repo) {
return new Response(JSON.stringify({ error: 'HuggingFace channel not properly configured' }), {
status: 400,
headers: { 'Content-Type': 'application/json' }
});
}
// 构建文件路径
const now = new Date();
const yearMonth = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}`;
// 生成唯一文件名
const ext = fileName.includes('.') ? fileName.substring(fileName.lastIndexOf('.')) : '';
const uniqueId = crypto.randomUUID().replace(/-/g, '');
const fullId = uniqueId + ext;
const filePath = `images/${yearMonth}/${fullId}`;
// 获取 LFS 上传信息
const huggingfaceAPI = new HuggingFaceAPI(hfChannel.token, hfChannel.repo, hfChannel.isPrivate || false);
const uploadInfo = await huggingfaceAPI.getLfsUploadInfo(fileSize, filePath, sha256, fileSample);
// 返回上传信息
return new Response(JSON.stringify({
success: true,
fullId,
filePath,
channelName: hfChannel.name,
repo: hfChannel.repo,
isPrivate: hfChannel.isPrivate || false,
...uploadInfo
}), {
status: 200,
headers: { 'Content-Type': 'application/json' }
});
} catch (error) {
console.error('getUploadUrl error:', error.message);
return new Response(JSON.stringify({ error: error.message }), {
status: 500,
headers: { 'Content-Type': 'application/json' }
});
}
}
+63 -1
View File
@@ -2,6 +2,8 @@ import { S3Client, DeleteObjectCommand } from "@aws-sdk/client-s3";
import { purgeCFCache } from "../../../utils/purgeCache"; import { purgeCFCache } from "../../../utils/purgeCache";
import { removeFileFromIndex, batchRemoveFilesFromIndex } from "../../../utils/indexManager.js"; import { removeFileFromIndex, batchRemoveFilesFromIndex } from "../../../utils/indexManager.js";
import { getDatabase } from '../../../utils/databaseAdapter.js'; import { getDatabase } from '../../../utils/databaseAdapter.js';
import { DiscordAPI } from '../../../utils/discordAPI.js';
import { HuggingFaceAPI } from '../../../utils/huggingfaceAPI.js';
// CORS 跨域响应头 // CORS 跨域响应头
const corsHeaders = { const corsHeaders = {
@@ -142,6 +144,16 @@ async function deleteFile(env, fileId, cdnUrl, url) {
await deleteS3File(img); await deleteS3File(img);
} }
// Discord 渠道的图片,需要删除 Discord 中对应的消息
if (img.metadata?.Channel === 'Discord') {
await deleteDiscordFile(img);
}
// HuggingFace 渠道的图片,需要删除 HuggingFace 中对应的文件
if (img.metadata?.Channel === 'HuggingFace') {
await deleteHuggingFaceFile(img);
}
// 删除数据库中的记录 // 删除数据库中的记录
// 注意:容量统计现在由索引自动维护,删除文件后索引更新时会自动重新计算 // 注意:容量统计现在由索引自动维护,删除文件后索引更新时会自动重新计算
await db.delete(fileId); await db.delete(fileId);
@@ -194,4 +206,54 @@ async function deleteS3File(img) {
console.error("S3 Delete Failed:", error); console.error("S3 Delete Failed:", error);
return false; return false;
} }
} }
// 删除 Discord 渠道的图片(删除 Discord 消息)
async function deleteDiscordFile(img) {
const botToken = img.metadata?.DiscordBotToken;
const channelId = img.metadata?.DiscordChannelId;
const messageId = img.metadata?.DiscordMessageId;
if (!botToken || !channelId || !messageId) {
console.warn('Discord file missing required metadata for deletion');
return false;
}
try {
const discordAPI = new DiscordAPI(botToken);
const success = await discordAPI.deleteMessage(channelId, messageId);
if (!success) {
console.error('Discord Delete Failed: API returned false');
}
return success;
} catch (error) {
console.error("Discord Delete Failed:", error);
return false;
}
}
// 删除 HuggingFace 渠道的图片
async function deleteHuggingFaceFile(img) {
const token = img.metadata?.HfToken;
const repo = img.metadata?.HfRepo;
const filePath = img.metadata?.HfFilePath;
const isPrivate = img.metadata?.HfIsPrivate || false;
if (!token || !repo || !filePath) {
console.warn('HuggingFace file missing required metadata for deletion');
return false;
}
try {
const huggingfaceAPI = new HuggingFaceAPI(token, repo, isPrivate);
const success = await huggingfaceAPI.deleteFile(filePath, `Delete ${filePath}`);
if (!success) {
console.error('HuggingFace Delete Failed: API returned false');
}
return success;
} catch (error) {
console.error("HuggingFace Delete Failed:", error);
return false;
}
}
+89
View File
@@ -172,10 +172,99 @@ export async function getUploadConfig(db, env) {
s3.loadBalance = s3LoadBalance s3.loadBalance = s3LoadBalance
// =====================读取 Discord 渠道配置=====================
const discord = {}
const discordChannels = []
discord.channels = discordChannels
// 从环境变量读取 Discord 配置
if (env.DISCORD_BOT_TOKEN) {
discordChannels.push({
id: 1,
name: 'Discord_env',
type: 'discord',
savePath: 'environment variable',
botToken: env.DISCORD_BOT_TOKEN,
channelId: env.DISCORD_CHANNEL_ID,
proxyUrl: env.DISCORD_PROXY_URL || '', // 可选的代理 URL
isNitro: env.DISCORD_IS_NITRO === 'true', // Nitro 会员,支持 25MB
enabled: true,
fixed: true,
})
}
for (const dc of settingsKV.discord?.channels || []) {
// 如果 savePath 是 environment variable,修改可变参数
if (dc.savePath === 'environment variable') {
// 如果环境变量未删除,进行覆盖操作
if (discordChannels[0]) {
discordChannels[0].enabled = dc.enabled
discordChannels[0].proxyUrl = dc.proxyUrl
discordChannels[0].isNitro = dc.isNitro
}
continue
}
// id 自增
dc.id = discordChannels.length + 1
discordChannels.push(dc)
}
// 负载均衡
const discordLoadBalance = settingsKV.discord?.loadBalance || {
enabled: false,
channels: [],
}
discord.loadBalance = discordLoadBalance
// =====================读取 HuggingFace 渠道配置=====================
const huggingface = {}
const huggingfaceChannels = []
huggingface.channels = huggingfaceChannels
// 从环境变量读取 HuggingFace 配置
if (env.HF_TOKEN) {
huggingfaceChannels.push({
id: 1,
name: 'HuggingFace_env',
type: 'huggingface',
savePath: 'environment variable',
token: env.HF_TOKEN,
repo: env.HF_REPO,
isPrivate: env.HF_PRIVATE === 'true',
enabled: true,
fixed: true,
})
}
for (const hf of settingsKV.huggingface?.channels || []) {
// 如果 savePath 是 environment variable,修改可变参数
if (hf.savePath === 'environment variable') {
// 如果环境变量未删除,进行覆盖操作
if (huggingfaceChannels[0]) {
huggingfaceChannels[0].enabled = hf.enabled
huggingfaceChannels[0].isPrivate = hf.isPrivate
}
continue
}
// id 自增
hf.id = huggingfaceChannels.length + 1
huggingfaceChannels.push(hf)
}
// 负载均衡
const huggingfaceLoadBalance = settingsKV.huggingface?.loadBalance || {
enabled: false,
channels: [],
}
huggingface.loadBalance = huggingfaceLoadBalance
settings.telegram = telegram settings.telegram = telegram
settings.cfr2 = cfr2 settings.cfr2 = cfr2
settings.s3 = s3 settings.s3 = s3
settings.discord = discord
settings.huggingface = huggingface
return settings; return settings;
} }
+351
View File
@@ -1,6 +1,8 @@
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"; import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
import { fetchSecurityConfig } from "../utils/sysConfig"; import { fetchSecurityConfig } from "../utils/sysConfig";
import { TelegramAPI } from "../utils/telegramAPI"; import { TelegramAPI } from "../utils/telegramAPI";
import { DiscordAPI } from "../utils/discordAPI";
import { HuggingFaceAPI } from "../utils/huggingfaceAPI";
import { setCommonHeaders, setRangeHeaders, handleHeadRequest, getFileContent, isTgChannel, import { setCommonHeaders, setRangeHeaders, handleHeadRequest, getFileContent, isTgChannel,
returnWithCheck, return404, isDomainAllowed } from './fileTools'; returnWithCheck, return404, isDomainAllowed } from './fileTools';
import { getDatabase } from '../utils/databaseAdapter.js'; import { getDatabase } from '../utils/databaseAdapter.js';
@@ -72,6 +74,20 @@ export async function onRequest(context) { // Contents of context object
return await handleS3File(context, imgRecord.metadata, encodedFileName, fileType); return await handleS3File(context, imgRecord.metadata, encodedFileName, fileType);
} }
/* Discord 渠道 */
if (imgRecord.metadata?.Channel === 'Discord') {
// 检查是否为分片文件
if (imgRecord.metadata?.IsChunked === true) {
return await handleDiscordChunkedFile(context, imgRecord, encodedFileName, fileType);
}
return await handleDiscordFile(context, imgRecord.metadata, encodedFileName, fileType);
}
/* HuggingFace 渠道 */
if (imgRecord.metadata?.Channel === 'HuggingFace') {
return await handleHuggingFaceFile(context, imgRecord.metadata, encodedFileName, fileType);
}
/* 外链渠道 */ /* 外链渠道 */
if (imgRecord.metadata?.Channel === 'External') { if (imgRecord.metadata?.Channel === 'External') {
// 直接重定向到外链 // 直接重定向到外链
@@ -328,6 +344,202 @@ async function fetchTelegramChunkWithRetry(botToken, chunk, maxRetries = 3) {
return null; return null;
} }
// 处理 Discord 渠道分片文件读取
async function handleDiscordChunkedFile(context, imgRecord, encodedFileName, fileType) {
const { request, url, Referer } = context;
const metadata = imgRecord.metadata;
const botToken = metadata.DiscordBotToken;
const proxyUrl = metadata.DiscordProxyUrl;
// 从KV的value中读取分片信息
let chunks = [];
try {
if (imgRecord.value) {
chunks = JSON.parse(imgRecord.value);
// 确保分片按索引排序
chunks.sort((a, b) => a.index - b.index);
}
} catch (parseError) {
console.error('Failed to parse Discord chunks data:', parseError);
return new Response('Error: Invalid chunks data', { status: 500 });
}
if (chunks.length === 0) {
return new Response('Error: No chunks found for this file', { status: 500 });
}
// 验证分片完整性
const expectedChunks = metadata.TotalChunks || chunks.length;
if (chunks.length !== expectedChunks) {
return new Response(`Error: Missing chunks, expected ${expectedChunks}, got ${chunks.length}`, { status: 500 });
}
// 计算文件总大小
const totalSize = chunks.reduce((total, chunk) => total + (chunk.size || 0), 0);
// 构建响应头
const headers = new Headers();
setCommonHeaders(headers, encodedFileName, fileType, Referer, url);
headers.set('Content-Length', totalSize.toString());
// 添加ETag支持
const etag = `"${metadata.TimeStamp || Date.now()}-${totalSize}"`;
headers.set('ETag', etag);
// 检查If-None-Match头(304缓存)
const ifNoneMatch = request.headers.get('If-None-Match');
if (ifNoneMatch && ifNoneMatch === etag) {
return new Response(null, {
status: 304,
headers: {
'ETag': etag,
'Cache-Control': headers.get('Cache-Control'),
'Accept-Ranges': 'bytes'
}
});
}
// 检查Range请求头
const range = request.headers.get('Range');
let rangeStart = 0;
let rangeEnd = totalSize - 1;
let isRangeRequest = false;
if (range) {
const matches = range.match(/bytes=(\d+)-(\d*)/);
if (matches) {
rangeStart = parseInt(matches[1]);
rangeEnd = matches[2] ? parseInt(matches[2]) : totalSize - 1;
isRangeRequest = true;
// 验证范围有效性
if (rangeStart >= totalSize || rangeEnd >= totalSize || rangeStart > rangeEnd) {
return new Response('Range Not Satisfiable', { status: 416 });
}
}
}
// 处理HEAD请求
if (request.method === 'HEAD') {
return handleHeadRequest(headers, etag);
}
try {
// 创建支持Range请求的流
const stream = new ReadableStream({
async start(controller) {
try {
let currentPosition = 0;
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
const chunkSize = chunk.size || 0;
// 如果当前分片完全在请求范围之前,跳过
if (currentPosition + chunkSize <= rangeStart) {
currentPosition += chunkSize;
continue;
}
// 如果当前分片完全在请求范围之后,结束
if (currentPosition > rangeEnd) {
break;
}
// 获取分片数据
const chunkData = await fetchDiscordChunkWithRetry(chunk, proxyUrl, 3);
if (!chunkData) {
throw new Error(`Failed to fetch Discord chunk ${chunk.index} after retries`);
}
// 计算在当前分片中的起始和结束位置
const chunkStart = Math.max(0, rangeStart - currentPosition);
const chunkEnd = Math.min(chunkSize, rangeEnd - currentPosition + 1);
// 如果需要部分分片数据
if (chunkStart > 0 || chunkEnd < chunkSize) {
const partialData = chunkData.slice(chunkStart, chunkEnd);
controller.enqueue(partialData);
} else {
controller.enqueue(chunkData);
}
currentPosition += chunkSize;
}
controller.close();
} catch (error) {
controller.error(error);
}
}
});
// 设置Range相关头部
if (isRangeRequest) {
setRangeHeaders(headers, rangeStart, rangeEnd, totalSize);
return new Response(stream, {
status: 206, // Partial Content
headers,
});
} else {
headers.set('Cache-Control', 'private, max-age=86400');
return new Response(stream, {
status: 200,
headers,
});
}
} catch (error) {
return new Response(`Error: Failed to reconstruct Discord chunked file - ${error.message}`, { status: 500 });
}
}
// 带重试机制的Discord分片获取函数
async function fetchDiscordChunkWithRetry(chunk, proxyUrl, maxRetries = 3) {
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
let fileUrl = chunk.url;
// 如果配置了代理 URL,替换 Discord CDN 域名
if (proxyUrl) {
fileUrl = fileUrl.replace('https://cdn.discordapp.com', `https://${proxyUrl}`);
}
const response = await fetch(fileUrl);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
// 验证分片大小是否匹配
const chunkData = await response.arrayBuffer();
const actualSize = chunkData.byteLength;
// 如果有期望大小且不匹配,记录警告
if (chunk.size && actualSize !== chunk.size) {
console.warn(`Discord chunk ${chunk.index} size mismatch: expected ${chunk.size}, got ${actualSize}`);
}
return new Uint8Array(chunkData);
} catch (error) {
console.warn(`Discord chunk ${chunk.index} fetch attempt ${attempt + 1} failed:`, error.message);
if (attempt === maxRetries - 1) {
return null; // 最后一次尝试也失败了
}
// 重试前等待一段时间
await new Promise(resolve => setTimeout(resolve, 500 * (attempt + 1)));
}
}
return null;
}
// 处理R2文件读取 // 处理R2文件读取
async function handleR2File(context, fileId, encodedFileName, fileType) { async function handleR2File(context, fileId, encodedFileName, fileType) {
const { env, request, url, Referer } = context; const { env, request, url, Referer } = context;
@@ -464,3 +676,142 @@ async function handleS3File(context, metadata, encodedFileName, fileType) {
return new Response(`Error: Failed to fetch from S3 - ${error.message}`, { status: 500 }); return new Response(`Error: Failed to fetch from S3 - ${error.message}`, { status: 500 });
} }
} }
// 处理 Discord 文件读取
async function handleDiscordFile(context, metadata, encodedFileName, fileType) {
const { env, request, url, Referer } = context;
try {
// 优先使用存储的附件 URL
let fileUrl = metadata.DiscordAttachmentUrl;
// 如果没有存储 URL,尝试通过 API 获取
if (!fileUrl && metadata.DiscordMessageId && metadata.DiscordChannelId && metadata.DiscordBotToken) {
const discordAPI = new DiscordAPI(metadata.DiscordBotToken);
fileUrl = await discordAPI.getFileURL(metadata.DiscordChannelId, metadata.DiscordMessageId);
}
if (!fileUrl) {
return new Response('Error: Discord file URL not found', { status: 500 });
}
// 如果配置了代理 URL,替换 Discord CDN 域名
if (metadata.DiscordProxyUrl) {
fileUrl = fileUrl.replace('https://cdn.discordapp.com', `https://${metadata.DiscordProxyUrl}`);
}
// 处理 HEAD 请求
if (request.method === 'HEAD') {
const headers = new Headers();
setCommonHeaders(headers, encodedFileName, fileType, Referer, url);
return handleHeadRequest(headers);
}
// 获取文件内容(支持 Range 请求)
const fetchHeaders = {};
const range = request.headers.get('Range');
if (range) {
fetchHeaders['Range'] = range;
}
const response = await fetch(fileUrl, {
method: 'GET',
headers: fetchHeaders
});
if (!response.ok && response.status !== 206) {
return new Response(`Error: Failed to fetch from Discord - ${response.status}`, { status: response.status });
}
// 构建响应头
const headers = new Headers();
setCommonHeaders(headers, encodedFileName, fileType, Referer, url);
// 复制相关头部
if (response.headers.get('Content-Length')) {
headers.set('Content-Length', response.headers.get('Content-Length'));
}
if (response.headers.get('Content-Range')) {
headers.set('Content-Range', response.headers.get('Content-Range'));
}
return new Response(response.body, {
status: response.status,
headers
});
} catch (error) {
return new Response(`Error: Failed to fetch from Discord - ${error.message}`, { status: 500 });
}
}
// 处理 HuggingFace 文件读取
async function handleHuggingFaceFile(context, metadata, encodedFileName, fileType) {
const { request, url, Referer } = context;
try {
const hfRepo = metadata.HfRepo;
const hfFilePath = metadata.HfFilePath;
const hfToken = metadata.HfToken;
const hfIsPrivate = metadata.HfIsPrivate || false;
if (!hfRepo || !hfFilePath) {
return new Response('Error: HuggingFace file info not found', { status: 500 });
}
// 构建文件 URL
const fileUrl = metadata.HfFileUrl || `https://huggingface.co/datasets/${hfRepo}/resolve/main/${hfFilePath}`;
// 处理 HEAD 请求
if (request.method === 'HEAD') {
const headers = new Headers();
setCommonHeaders(headers, encodedFileName, fileType, Referer, url);
return handleHeadRequest(headers);
}
// 构建请求头
const fetchHeaders = {};
// 私有仓库需要 Authorization
if (hfIsPrivate && hfToken) {
fetchHeaders['Authorization'] = `Bearer ${hfToken}`;
}
// 支持 Range 请求
const range = request.headers.get('Range');
if (range) {
fetchHeaders['Range'] = range;
}
const response = await fetch(fileUrl, {
method: 'GET',
headers: fetchHeaders
});
if (!response.ok && response.status !== 206) {
return new Response(`Error: Failed to fetch from HuggingFace - ${response.status}`, { status: response.status });
}
// 构建响应头
const headers = new Headers();
setCommonHeaders(headers, encodedFileName, fileType, Referer, url);
// 复制相关头部
if (response.headers.get('Content-Length')) {
headers.set('Content-Length', response.headers.get('Content-Length'));
}
if (response.headers.get('Content-Range')) {
headers.set('Content-Range', response.headers.get('Content-Range'));
}
return new Response(response.body, {
status: response.status,
headers
});
} catch (error) {
return new Response(`Error: Failed to fetch from HuggingFace - ${error.message}`, { status: 500 });
}
}
+74
View File
@@ -204,6 +204,8 @@ async function handleChannelBasedMerge(context, uploadId, totalChunks, originalF
result = await mergeS3ChunksInfo(context, uploadId, completedChunks, metadata); result = await mergeS3ChunksInfo(context, uploadId, completedChunks, metadata);
} else if (uploadChannel === 'telegram') { } else if (uploadChannel === 'telegram') {
result = await mergeTelegramChunksInfo(context, uploadId, completedChunks, metadata); result = await mergeTelegramChunksInfo(context, uploadId, completedChunks, metadata);
} else if (uploadChannel === 'discord') {
result = await mergeDiscordChunksInfo(context, uploadId, completedChunks, metadata);
} else { } else {
throw new Error(`Unsupported upload channel: ${uploadChannel}`); throw new Error(`Unsupported upload channel: ${uploadChannel}`);
} }
@@ -461,3 +463,75 @@ async function mergeTelegramChunksInfo(context, uploadId, completedChunks, metad
throw new Error(`Telegram merge failed: ${error.message}`); throw new Error(`Telegram merge failed: ${error.message}`);
} }
} }
// 合并Discord分块信息
async function mergeDiscordChunksInfo(context, uploadId, completedChunks, metadata) {
const { env, waitUntil, uploadConfig, url } = context;
const db = getDatabase(env);
try {
const discordSettings = uploadConfig.discord;
const discordChannels = discordSettings.channels;
const discordChannel = selectConsistentChannel(discordChannels, uploadId, discordSettings.loadBalance?.enabled);
console.log(`Merging Discord chunks for uploadId: ${uploadId}, selected channel: ${discordChannel.name || 'default'}`);
const botToken = discordChannel.botToken;
const channelId = discordChannel.channelId;
// 按顺序排列分块
const sortedChunks = completedChunks.sort((a, b) => a.index - b.index);
// 计算总大小
const totalSize = sortedChunks.reduce((sum, chunk) => sum + chunk.uploadResult.size, 0);
// 构建分块信息数组
const chunks = sortedChunks.map(chunk => ({
index: chunk.index,
messageId: chunk.uploadResult.messageId,
attachmentId: chunk.uploadResult.attachmentId,
url: chunk.uploadResult.url,
size: chunk.uploadResult.size,
fileName: chunk.uploadResult.fileName
}));
// 生成 finalFileId
const finalFileId = await buildUniqueFileId(context, metadata.FileName, metadata.FileType);
// 更新metadata
metadata.Channel = "Discord";
metadata.ChannelName = discordChannel.name;
metadata.DiscordChannelId = channelId;
metadata.DiscordBotToken = botToken;
metadata.DiscordProxyUrl = discordChannel.proxyUrl || '';
metadata.IsChunked = true;
metadata.TotalChunks = completedChunks.length;
metadata.FileSize = (totalSize / 1024 / 1024).toFixed(2);
// 将分片信息存储到value中
const chunksData = JSON.stringify(chunks);
// 写入数据库
await db.put(finalFileId, chunksData, { metadata });
// 异步结束上传
waitUntil(endUpload(context, finalFileId, metadata));
// 生成返回链接
const returnFormat = url.searchParams.get('returnFormat') || 'default';
let updatedReturnLink = '';
if (returnFormat === 'full') {
updatedReturnLink = `${url.origin}/file/${finalFileId}`;
} else {
updatedReturnLink = `/file/${finalFileId}`;
}
return {
success: true,
result: [{ 'src': updatedReturnLink }]
};
} catch (error) {
throw new Error(`Discord merge failed: ${error.message}`);
}
}
+105 -1
View File
@@ -1,6 +1,7 @@
/* ======= 客户端分块上传处理 ======= */ /* ======= 客户端分块上传处理 ======= */
import { createResponse, selectConsistentChannel, getUploadIp, getIPAddress, buildUniqueFileId, endUpload } from './uploadTools'; import { createResponse, selectConsistentChannel, getUploadIp, getIPAddress, buildUniqueFileId, endUpload } from './uploadTools';
import { TelegramAPI } from '../utils/telegramAPI'; import { TelegramAPI } from '../utils/telegramAPI';
import { DiscordAPI } from '../utils/discordAPI';
import { S3Client, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand } from "@aws-sdk/client-s3"; import { S3Client, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand } from "@aws-sdk/client-s3";
import { getDatabase } from '../utils/databaseAdapter.js'; import { getDatabase } from '../utils/databaseAdapter.js';
@@ -268,6 +269,8 @@ async function uploadChunkToStorage(context, chunkIndex, totalChunks, uploadId,
uploadResult = await uploadSingleChunkToS3Multipart(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType); uploadResult = await uploadSingleChunkToS3Multipart(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType);
} else if (uploadChannel === 'telegram') { } else if (uploadChannel === 'telegram') {
uploadResult = await uploadSingleChunkToTelegram(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType); uploadResult = await uploadSingleChunkToTelegram(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType);
} else if (uploadChannel === 'discord') {
uploadResult = await uploadSingleChunkToDiscord(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType);
} }
if (uploadResult && uploadResult.success) { if (uploadResult && uploadResult.success) {
@@ -587,6 +590,105 @@ async function uploadSingleChunkToTelegram(context, chunkData, chunkIndex, total
} }
} }
// 上传单个分块到Discord
async function uploadSingleChunkToDiscord(context, chunkData, chunkIndex, totalChunks, uploadId, originalFileName, originalFileType) {
const { uploadConfig } = context;
try {
const discordSettings = uploadConfig.discord;
const discordChannels = discordSettings.channels;
const discordChannel = selectConsistentChannel(discordChannels, uploadId, discordSettings.loadBalance?.enabled);
console.log(`Uploading Discord chunk ${chunkIndex} for uploadId: ${uploadId}, selected channel: ${discordChannel.name || 'default'}`);
if (!discordChannel) {
return { success: false, error: 'No Discord channel provided' };
}
const botToken = discordChannel.botToken;
const channelId = discordChannel.channelId;
// 创建分块文件名
const chunkFileName = `${originalFileName}.part${chunkIndex.toString().padStart(3, '0')}`;
const chunkBlob = new Blob([chunkData], { type: 'application/octet-stream' });
// 上传分块到Discord(带重试)
const chunkInfo = await uploadChunkToDiscordWithRetry(
botToken,
channelId,
chunkBlob,
chunkFileName,
chunkIndex,
totalChunks,
2 // maxRetries
);
if (!chunkInfo) {
return { success: false, error: 'Failed to upload chunk to Discord' };
}
return {
success: true,
messageId: chunkInfo.message_id,
attachmentId: chunkInfo.attachment_id,
url: chunkInfo.url,
size: chunkInfo.file_size,
fileName: chunkFileName,
uploadTime: Date.now(),
discordChannel: discordChannel.name
};
} catch (error) {
return {
success: false,
error: error.message
};
}
}
// 将每个分块上传至Discord,支持失败重试和 rate limit 处理
async function uploadChunkToDiscordWithRetry(botToken, channelId, chunkBlob, chunkFileName, chunkIndex, totalChunks, maxRetries = 2) {
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
const discordAPI = new DiscordAPI(botToken);
const response = await discordAPI.sendFile(chunkBlob, channelId, chunkFileName);
if (!response || !response.id) {
throw new Error('Invalid Discord response');
}
const fileInfo = discordAPI.getFileInfo(response);
if (!fileInfo) {
throw new Error('Failed to extract file info from response');
}
return fileInfo;
} catch (error) {
console.warn(`Discord chunk ${chunkIndex} upload attempt ${attempt + 1} failed:`, error.message);
// 检查是否是 rate limit (429)
if (error.message && error.message.includes('429')) {
// 从错误消息中提取 retry_after,或使用默认值
const retryAfter = 5000; // 默认等待 5 秒
console.log(`Discord rate limited, waiting ${retryAfter}ms...`);
await new Promise(resolve => setTimeout(resolve, retryAfter));
continue; // 不计入重试次数
}
if (attempt === maxRetries - 1) {
return null; // 最后一次尝试也失败了
}
// 指数退避延迟
await new Promise(resolve => setTimeout(resolve, 1000 * (attempt + 1)));
}
}
return null;
}
/* ======== 分块合并时与上传相关的工具函数 ======= */ /* ======== 分块合并时与上传相关的工具函数 ======= */
// 重传失败的分块 // 重传失败的分块
@@ -726,6 +828,8 @@ async function retrySingleChunk(context, chunk, uploadChannel, maxRetries = 5, r
return await uploadSingleChunkToS3Multipart(context, chunkData, chunk.index, totalChunks, uploadId, originalFileName, originalFileType); return await uploadSingleChunkToS3Multipart(context, chunkData, chunk.index, totalChunks, uploadId, originalFileName, originalFileType);
} else if (uploadChannel === 'telegram') { } else if (uploadChannel === 'telegram') {
return await uploadSingleChunkToTelegram(context, chunkData, chunk.index, totalChunks, uploadId, originalFileName, originalFileType); return await uploadSingleChunkToTelegram(context, chunkData, chunk.index, totalChunks, uploadId, originalFileName, originalFileType);
} else if (uploadChannel === 'discord') {
return await uploadSingleChunkToDiscord(context, chunkData, chunk.index, totalChunks, uploadId, originalFileName, originalFileType);
} }
return null; return null;
})(); })();
@@ -1019,7 +1123,7 @@ export async function uploadLargeFileToTelegram(context, file, fullId, metadata,
const { env, waitUntil } = context; const { env, waitUntil } = context;
const db = getDatabase(env); const db = getDatabase(env);
const CHUNK_SIZE = 20 * 1024 * 1024; // 20MB const CHUNK_SIZE = 16 * 1024 * 1024; // 16MB (TG Bot getFile download limit: 20MB, leave 4MB safety margin)
const fileSize = file.size; const fileSize = file.size;
const totalChunks = Math.ceil(fileSize / CHUNK_SIZE); const totalChunks = Math.ceil(fileSize / CHUNK_SIZE);
+230 -6
View File
@@ -7,6 +7,8 @@ import {
import { initializeChunkedUpload, handleChunkUpload, uploadLargeFileToTelegram, handleCleanupRequest } from "./chunkUpload"; import { initializeChunkedUpload, handleChunkUpload, uploadLargeFileToTelegram, handleCleanupRequest } from "./chunkUpload";
import { handleChunkMerge } from "./chunkMerge"; import { handleChunkMerge } from "./chunkMerge";
import { TelegramAPI } from "../utils/telegramAPI"; import { TelegramAPI } from "../utils/telegramAPI";
import { DiscordAPI } from "../utils/discordAPI";
import { HuggingFaceAPI } from "../utils/huggingfaceAPI";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
import { getDatabase } from '../utils/databaseAdapter.js'; import { getDatabase } from '../utils/databaseAdapter.js';
@@ -101,6 +103,12 @@ async function processFileUpload(context, formdata = null) {
case 's3': case 's3':
uploadChannel = 'S3'; uploadChannel = 'S3';
break; break;
case 'discord':
uploadChannel = 'Discord';
break;
case 'huggingface':
uploadChannel = 'HuggingFace';
break;
case 'external': case 'external':
uploadChannel = 'External'; uploadChannel = 'External';
break; break;
@@ -188,6 +196,22 @@ async function processFileUpload(context, formdata = null) {
} else { } else {
err = await res.text(); err = await res.text();
} }
} else if (uploadChannel === 'Discord') {
// ---------------------Discord 渠道------------------
const res = await uploadFileToDiscord(context, fullId, metadata, returnLink);
if (res.status === 200 || !autoRetry) {
return res;
} else {
err = await res.text();
}
} else if (uploadChannel === 'HuggingFace') {
// ---------------------HuggingFace 渠道------------------
const res = await uploadFileToHuggingFace(context, fullId, metadata, returnLink);
if (res.status === 200 || !autoRetry) {
return res;
} else {
err = await res.text();
}
} else if (uploadChannel === 'External') { } else if (uploadChannel === 'External') {
// --------------------外链渠道---------------------- // --------------------外链渠道----------------------
const res = await uploadFileToExternal(context, fullId, metadata, returnLink); const res = await uploadFileToExternal(context, fullId, metadata, returnLink);
@@ -393,8 +417,8 @@ async function uploadFileToTelegram(context, fullId, metadata, fileExt, fileName
const telegramAPI = new TelegramAPI(tgBotToken); const telegramAPI = new TelegramAPI(tgBotToken);
// 20MB 分片阈值 // 16MB 分片阈值 (TG Bot getFile download limit: 20MB, leave 4MB safety margin)
const CHUNK_SIZE = 20 * 1024 * 1024; // 20MB const CHUNK_SIZE = 16 * 1024 * 1024; // 16MB
if (fileSize > CHUNK_SIZE) { if (fileSize > CHUNK_SIZE) {
// 大文件分片上传 // 大文件分片上传
@@ -529,12 +553,208 @@ async function uploadFileToExternal(context, fullId, metadata, returnLink) {
); );
} }
// 上传到 Discord
async function uploadFileToDiscord(context, fullId, metadata, returnLink) {
const { env, waitUntil, uploadConfig, formdata } = context;
const db = getDatabase(env);
// 获取 Discord 渠道配置
const discordSettings = uploadConfig.discord;
if (!discordSettings || !discordSettings.channels || discordSettings.channels.length === 0) {
return createResponse('Error: No Discord channel configured', { status: 400 });
}
// 选择渠道(支持负载均衡)
const discordChannels = discordSettings.channels;
const discordChannel = discordSettings.loadBalance?.enabled
? discordChannels[Math.floor(Math.random() * discordChannels.length)]
: discordChannels[0];
if (!discordChannel || !discordChannel.botToken || !discordChannel.channelId) {
return createResponse('Error: Discord channel not properly configured', { status: 400 });
}
const file = formdata.get('file');
const fileSize = file.size;
const fileName = metadata.FileName;
// Discord 文件大小限制:Nitro 会员 25MB,免费用户 10MB
const isNitro = discordChannel.isNitro || false;
const DISCORD_MAX_SIZE = isNitro ? 25 * 1024 * 1024 : 10 * 1024 * 1024;
if (fileSize > DISCORD_MAX_SIZE) {
const limitMB = isNitro ? 25 : 10;
return createResponse(`Error: File size exceeds Discord limit (${limitMB}MB), please use another channel`, { status: 413 });
}
const discordAPI = new DiscordAPI(discordChannel.botToken);
try {
// 上传文件到 Discord
const response = await discordAPI.sendFile(file, discordChannel.channelId, fileName);
const fileInfo = discordAPI.getFileInfo(response);
if (!fileInfo) {
throw new Error('Failed to get file info from Discord response');
}
// 更新 metadata
metadata.Channel = "Discord";
metadata.ChannelName = discordChannel.name || "Discord_env";
metadata.FileSize = (fileInfo.file_size / 1024 / 1024).toFixed(2);
metadata.DiscordMessageId = fileInfo.message_id;
metadata.DiscordChannelId = discordChannel.channelId;
metadata.DiscordBotToken = discordChannel.botToken;
metadata.DiscordAttachmentUrl = fileInfo.url;
// 如果配置了代理 URL,保存代理信息
if (discordChannel.proxyUrl) {
metadata.DiscordProxyUrl = discordChannel.proxyUrl;
}
// 图像审查(使用 Discord CDN URL 或代理 URL
let moderateUrl = fileInfo.url;
if (discordChannel.proxyUrl) {
moderateUrl = fileInfo.url.replace('https://cdn.discordapp.com', `https://${discordChannel.proxyUrl}`);
}
metadata.Label = await moderateContent(env, moderateUrl);
// 写入 KV 数据库
try {
await db.put(fullId, "", { metadata });
} catch (error) {
return createResponse('Error: Failed to write to KV database', { status: 500 });
}
// 结束上传
waitUntil(endUpload(context, fullId, metadata));
// 返回成功响应
return createResponse(
JSON.stringify([{ 'src': returnLink }]),
{
status: 200,
headers: { 'Content-Type': 'application/json' }
}
);
} catch (error) {
console.error('Discord upload error:', error.message);
return createResponse(`Error: Discord upload failed - ${error.message}`, { status: 500 });
}
}
// 上传到 HuggingFace
async function uploadFileToHuggingFace(context, fullId, metadata, returnLink) {
const { env, waitUntil, uploadConfig, formdata } = context;
const db = getDatabase(env);
console.log('=== HuggingFace Upload Start ===');
// 获取 HuggingFace 渠道配置
const hfSettings = uploadConfig.huggingface;
console.log('HuggingFace settings:', hfSettings ? 'found' : 'not found');
if (!hfSettings || !hfSettings.channels || hfSettings.channels.length === 0) {
console.log('Error: No HuggingFace channel configured');
return createResponse('Error: No HuggingFace channel configured', { status: 400 });
}
// 选择渠道(支持负载均衡)
const hfChannels = hfSettings.channels;
console.log('HuggingFace channels count:', hfChannels.length);
const hfChannel = hfSettings.loadBalance?.enabled
? hfChannels[Math.floor(Math.random() * hfChannels.length)]
: hfChannels[0];
console.log('Selected channel:', hfChannel?.name, 'repo:', hfChannel?.repo);
if (!hfChannel || !hfChannel.token || !hfChannel.repo) {
console.log('Error: HuggingFace channel not properly configured', {
hasChannel: !!hfChannel,
hasToken: !!hfChannel?.token,
hasRepo: !!hfChannel?.repo
});
return createResponse('Error: HuggingFace channel not properly configured', { status: 400 });
}
const file = formdata.get('file');
const fileName = metadata.FileName;
// 获取前端预计算的 SHA256(如果有)
const precomputedSha256 = formdata.get('sha256') || null;
console.log('File to upload:', fileName, 'size:', file?.size, 'precomputed SHA256:', precomputedSha256 ? 'yes' : 'no');
// 构建文件路径:images/年月/文件名
const now = new Date();
const yearMonth = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}`;
const hfFilePath = `images/${yearMonth}/${fullId}`;
console.log('HuggingFace file path:', hfFilePath);
const huggingfaceAPI = new HuggingFaceAPI(hfChannel.token, hfChannel.repo, hfChannel.isPrivate || false);
try {
// 上传文件到 HuggingFace(传入预计算的 SHA256
console.log('Starting HuggingFace upload...');
const result = await huggingfaceAPI.uploadFile(file, hfFilePath, `Upload ${fileName}`, precomputedSha256);
console.log('HuggingFace upload result:', result);
if (!result.success) {
throw new Error('Failed to upload file to HuggingFace');
}
// 更新 metadata
metadata.Channel = "HuggingFace";
metadata.ChannelName = hfChannel.name || "HuggingFace_env";
metadata.FileSize = (file.size / 1024 / 1024).toFixed(2);
metadata.HfRepo = hfChannel.repo;
metadata.HfFilePath = hfFilePath;
metadata.HfToken = hfChannel.token;
metadata.HfIsPrivate = hfChannel.isPrivate || false;
metadata.HfFileUrl = result.fileUrl;
// 图像审查(公开仓库直接访问,私有仓库需要代理)
let moderateUrl = result.fileUrl;
if (!hfChannel.isPrivate) {
metadata.Label = await moderateContent(env, moderateUrl);
} else {
// 私有仓库暂不支持图像审查,标记为 None
metadata.Label = "None";
}
// 写入 KV 数据库
try {
await db.put(fullId, "", { metadata });
} catch (error) {
return createResponse('Error: Failed to write to KV database', { status: 500 });
}
// 结束上传
waitUntil(endUpload(context, fullId, metadata));
// 返回成功响应
return createResponse(
JSON.stringify([{ 'src': returnLink }]),
{
status: 200,
headers: { 'Content-Type': 'application/json' }
}
);
} catch (error) {
console.error('HuggingFace upload error:', error.message);
return createResponse(`Error: HuggingFace upload failed - ${error.message}`, { status: 500 });
}
}
// 自动切换渠道重试 // 自动切换渠道重试
async function tryRetry(err, context, uploadChannel, fullId, metadata, fileExt, fileName, fileType, returnLink) { async function tryRetry(err, context, uploadChannel, fullId, metadata, fileExt, fileName, fileType, returnLink) {
const { env, url, formdata } = context; const { env, url, formdata } = context;
// 渠道列表 // 渠道列表Discord 因为有 10MB 限制,放在最后尝试)
const channelList = ['CloudflareR2', 'TelegramNew', 'S3']; const channelList = ['CloudflareR2', 'TelegramNew', 'S3', 'HuggingFace', 'Discord'];
const errMessages = {}; const errMessages = {};
errMessages[uploadChannel] = 'Error: ' + uploadChannel + err; errMessages[uploadChannel] = 'Error: ' + uploadChannel + err;
@@ -547,11 +767,15 @@ async function tryRetry(err, context, uploadChannel, fullId, metadata, fileExt,
res = await uploadFileToTelegram(context, fullId, metadata, fileExt, fileName, fileType, returnLink); res = await uploadFileToTelegram(context, fullId, metadata, fileExt, fileName, fileType, returnLink);
} else if (channelList[i] === 'S3') { } else if (channelList[i] === 'S3') {
res = await uploadFileToS3(context, fullId, metadata, returnLink); res = await uploadFileToS3(context, fullId, metadata, returnLink);
} else if (channelList[i] === 'HuggingFace') {
res = await uploadFileToHuggingFace(context, fullId, metadata, returnLink);
} else if (channelList[i] === 'Discord') {
res = await uploadFileToDiscord(context, fullId, metadata, returnLink);
} }
if (res.status === 200) { if (res && res.status === 200) {
return res; return res;
} else { } else if (res) {
errMessages[channelList[i]] = 'Error: ' + channelList[i] + await res.text(); errMessages[channelList[i]] = 'Error: ' + channelList[i] + await res.text();
} }
} }
+166
View File
@@ -0,0 +1,166 @@
/**
* Discord API 封装类
* 用于上传文件到 Discord 频道并获取文件
*/
export class DiscordAPI {
constructor(botToken) {
this.botToken = botToken;
this.baseURL = 'https://discord.com/api/v10';
this.defaultHeaders = {
'Authorization': `Bot ${this.botToken}`,
'User-Agent': 'DiscordBot (CloudFlare-ImgBed, 1.0)'
};
}
/**
* 发送文件到 Discord 频道
* @param {File|Blob} file - 要发送的文件
* @param {string} channelId - 频道 ID
* @param {string} fileName - 文件名
* @returns {Promise<Object>} API 响应结果
*/
async sendFile(file, channelId, fileName = '') {
const formData = new FormData();
// Discord 使用 files[0] 作为文件字段名
if (fileName) {
formData.append('files[0]', file, fileName);
} else {
formData.append('files[0]', file);
}
const response = await fetch(`${this.baseURL}/channels/${channelId}/messages`, {
method: 'POST',
headers: this.defaultHeaders,
body: formData
});
console.log('Discord API response:', response.status, response.statusText);
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(`Discord API error: ${response.status} - ${errorData.message || response.statusText}`);
}
const responseData = await response.json();
return responseData;
}
/**
* 从响应中提取文件信息
* @param {Object} responseData - Discord API 响应数据
* @returns {Object|null} 文件信息对象或 null
*/
getFileInfo(responseData) {
try {
if (!responseData || !responseData.id) {
console.error('Invalid Discord response:', responseData);
return null;
}
// Discord 消息中的附件在 attachments 数组中
if (responseData.attachments && responseData.attachments.length > 0) {
const attachment = responseData.attachments[0];
return {
message_id: responseData.id,
attachment_id: attachment.id,
file_name: attachment.filename,
file_size: attachment.size,
content_type: attachment.content_type,
url: attachment.url,
proxy_url: attachment.proxy_url
};
}
return null;
} catch (error) {
console.error('Error parsing Discord response:', error.message);
return null;
}
}
/**
* 获取消息信息用于获取文件 URL
* @param {string} channelId - 频道 ID
* @param {string} messageId - 消息 ID
* @returns {Promise<Object|null>} 消息数据或 null
*/
async getMessage(channelId, messageId) {
try {
const response = await fetch(`${this.baseURL}/channels/${channelId}/messages/${messageId}`, {
method: 'GET',
headers: this.defaultHeaders
});
if (!response.ok) {
console.error('Discord getMessage error:', response.status, response.statusText);
return null;
}
const messageData = await response.json();
return messageData;
} catch (error) {
console.error('Error getting Discord message:', error.message);
return null;
}
}
/**
* 获取文件 URL
* @param {string} channelId - 频道 ID
* @param {string} messageId - 消息 ID
* @returns {Promise<string|null>} 文件 URL null
*/
async getFileURL(channelId, messageId) {
const message = await this.getMessage(channelId, messageId);
if (message && message.attachments && message.attachments.length > 0) {
return message.attachments[0].url;
}
return null;
}
/**
* 获取文件内容
* @param {string} channelId - 频道 ID
* @param {string} messageId - 消息 ID
* @returns {Promise<Response>} 文件响应
*/
async getFileContent(channelId, messageId) {
const fileURL = await this.getFileURL(channelId, messageId);
if (!fileURL) {
throw new Error(`File URL not found for messageId: ${messageId}`);
}
const response = await fetch(fileURL);
return response;
}
/**
* 删除消息用于删除文件
* @param {string} channelId - 频道 ID
* @param {string} messageId - 消息 ID
* @returns {Promise<boolean>} 是否删除成功
*/
async deleteMessage(channelId, messageId) {
try {
const response = await fetch(`${this.baseURL}/channels/${channelId}/messages/${messageId}`, {
method: 'DELETE',
headers: this.defaultHeaders
});
// Discord 删除成功返回 204 No Content
if (response.status === 204 || response.ok) {
return true;
}
console.error('Discord deleteMessage error:', response.status, response.statusText);
return false;
} catch (error) {
console.error('Error deleting Discord message:', error.message);
return false;
}
}
}
+508
View File
@@ -0,0 +1,508 @@
/**
* Hugging Face Hub API 封装类
* 手动实现 LFS 上传协议Cloudflare Workers 兼容
*
* HuggingFace 要求二进制文件通过 LFS 协议上传
* 流程preupload -> LFS batch -> upload to LFS storage -> commit
*
* 优化方案
* 1. 小文件<20MB前端 CF Workers HuggingFace S3
* 2. 大文件>=20MB前端直接上传到 HuggingFace S3CF Workers 只负责获取签名 URL 和提交
*
* SHA256 由前端预计算传入避免后端 CPU 超时
*/
export class HuggingFaceAPI {
constructor(token, repo, isPrivate = false) {
this.token = token;
this.repo = repo; // 格式: username/repo-name
this.isPrivate = isPrivate;
this.baseURL = 'https://huggingface.co';
}
/**
* 计算文件的 SHA256 哈希仅在未提供预计算哈希时使用
* @param {Blob} blob
* @returns {Promise<string>} hex string
*/
async sha256(blob) {
const buffer = await blob.arrayBuffer();
const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
const hashArray = Array.from(new Uint8Array(hashBuffer));
return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
}
/**
* 检查仓库是否存在
*/
async repoExists() {
try {
const response = await fetch(`${this.baseURL}/api/datasets/${this.repo}`, {
headers: { 'Authorization': `Bearer ${this.token}` }
});
return response.ok;
} catch (error) {
console.error('Error checking repo:', error.message);
return false;
}
}
/**
* 创建仓库如果不存在
*/
async createRepoIfNotExists() {
try {
if (await this.repoExists()) {
console.log('Repository exists:', this.repo);
return true;
}
console.log('Creating repository:', this.repo);
const response = await fetch(`${this.baseURL}/api/repos/create`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({
name: this.repo.split('/')[1],
type: 'dataset',
private: this.isPrivate
})
});
if (response.ok || response.status === 409) {
console.log('Repository ready');
return true;
}
const errorText = await response.text();
throw new Error(`Failed to create repo: ${response.status} - ${errorText}`);
} catch (error) {
console.error('Error creating repo:', error.message);
return false;
}
}
/**
* 步骤1: Preupload - 检查文件是否需要 LFS
*/
async preupload(filePath, fileSize, fileSample) {
const url = `${this.baseURL}/api/datasets/${this.repo}/preupload/main`;
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({
files: [{
path: filePath,
size: fileSize,
sample: fileSample
}]
})
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Preupload failed: ${response.status} - ${error}`);
}
return await response.json();
}
/**
* 步骤2: LFS Batch - 获取上传 URL
*/
async lfsBatch(oid, fileSize) {
const url = `${this.baseURL}/datasets/${this.repo}.git/info/lfs/objects/batch`;
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json'
},
body: JSON.stringify({
operation: 'upload',
transfers: ['basic', 'multipart'],
hash_algo: 'sha_256',
ref: { name: 'main' },
objects: [{ oid, size: fileSize }]
})
});
if (!response.ok) {
const error = await response.text();
throw new Error(`LFS batch failed: ${response.status} - ${error}`);
}
return await response.json();
}
/**
* 步骤3: 上传文件到 LFS 存储
* @param {object} uploadAction - 上传动作信息
* @param {File|Blob} file - 文件
* @param {string} oid - 文件的 SHA256 哈希
*/
async uploadToLFS(uploadAction, file, oid) {
const { href, header } = uploadAction;
// 检查是否是分片上传
if (header?.chunk_size) {
return await this.uploadMultipart(uploadAction, file, oid);
}
// 基本上传
console.log('Uploading to LFS (basic):', href);
const response = await fetch(href, {
method: 'PUT',
headers: header || {},
body: file
});
if (!response.ok) {
const error = await response.text();
throw new Error(`LFS upload failed: ${response.status} - ${error}`);
}
return true;
}
/**
* 分片上传大文件
* @param {object} uploadAction - 上传动作信息
* @param {File|Blob} file - 文件
* @param {string} oid - 文件的 SHA256 哈希
*/
async uploadMultipart(uploadAction, file, oid) {
const { href: completionUrl, header } = uploadAction;
const chunkSize = parseInt(header.chunk_size);
// 获取所有分片的上传 URL
const parts = Object.keys(header).filter(key => /^[0-9]+$/.test(key));
console.log(`Multipart upload: ${parts.length} parts, chunk size: ${chunkSize}`);
const completeParts = [];
for (const part of parts) {
const index = parseInt(part) - 1;
const start = index * chunkSize;
const end = Math.min(start + chunkSize, file.size);
const chunk = file.slice(start, end);
console.log(`Uploading part ${part}/${parts.length}`);
const response = await fetch(header[part], {
method: 'PUT',
body: chunk
});
if (!response.ok) {
throw new Error(`Failed to upload part ${part}: ${response.status}`);
}
const etag = response.headers.get('ETag');
if (!etag) {
throw new Error(`No ETag for part ${part}`);
}
completeParts.push({ partNumber: parseInt(part), etag });
}
// 完成分片上传
console.log('Completing multipart upload...');
const completeResponse = await fetch(completionUrl, {
method: 'POST',
headers: {
'Accept': 'application/vnd.git-lfs+json',
'Content-Type': 'application/vnd.git-lfs+json'
},
body: JSON.stringify({
oid: oid,
parts: completeParts
})
});
if (!completeResponse.ok) {
const error = await completeResponse.text();
throw new Error(`Multipart complete failed: ${completeResponse.status} - ${error}`);
}
return true;
}
/**
* 步骤4: 提交 LFS 文件引用
*/
async commitLfsFile(filePath, oid, fileSize, commitMessage) {
const url = `${this.baseURL}/api/datasets/${this.repo}/commit/main`;
// NDJSON 格式
const body = [
JSON.stringify({
key: 'header',
value: { summary: commitMessage }
}),
JSON.stringify({
key: 'lfsFile',
value: {
path: filePath,
algo: 'sha256',
size: fileSize,
oid: oid
}
})
].join('\n');
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/x-ndjson'
},
body
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Commit failed: ${response.status} - ${error}`);
}
return await response.json();
}
/**
* 获取 LFS 上传信息用于前端直传大文件
* 返回上传 URL 和必要的信息让前端直接上传到 S3
* @param {number} fileSize - 文件大小
* @param {string} filePath - 存储路径
* @param {string} sha256 - 文件的 SHA256 哈希
* @param {string} fileSample - 文件前512字节的 base64
*/
async getLfsUploadInfo(fileSize, filePath, sha256, fileSample) {
// 确保仓库存在
if (!await this.createRepoIfNotExists()) {
throw new Error('Failed to create or access repository');
}
// 1. Preupload 检查
console.log('Preupload check for direct upload...');
const preuploadResult = await this.preupload(filePath, fileSize, fileSample);
console.log('Preupload result:', JSON.stringify(preuploadResult));
const fileInfo = preuploadResult.files?.[0];
const needsLfs = fileInfo?.uploadMode === 'lfs';
if (!needsLfs) {
// 小文件不需要 LFS,返回 null 让后端处理
return { needsLfs: false };
}
// 2. LFS Batch - 获取上传 URL
console.log('LFS batch request for direct upload...');
const batchResult = await this.lfsBatch(sha256, fileSize);
console.log('LFS batch result:', JSON.stringify(batchResult));
const obj = batchResult.objects?.[0];
if (obj?.error) {
throw new Error(`LFS error: ${obj.error.message}`);
}
// 检查文件是否已存在
if (!obj?.actions?.upload) {
return {
needsLfs: true,
alreadyExists: true,
oid: sha256,
filePath
};
}
// 返回上传信息
return {
needsLfs: true,
alreadyExists: false,
oid: sha256,
filePath,
uploadAction: obj.actions.upload
};
}
/**
* 上传文件完整流程- 用于小文件或后端代理上传
* @param {File|Blob} file - 要上传的文件
* @param {string} filePath - 存储路径
* @param {string} commitMessage - 提交信息
* @param {string} precomputedSha256 - 前端预计算的 SHA256可选传入可避免后端计算
*/
async uploadFile(file, filePath, commitMessage = 'Upload file', precomputedSha256 = null) {
try {
// 确保仓库存在
if (!await this.createRepoIfNotExists()) {
throw new Error('Failed to create or access repository');
}
console.log('=== HuggingFace LFS Upload ===');
console.log('Repo:', this.repo);
console.log('Path:', filePath);
console.log('Size:', file.size);
// 1. 使用预计算的 SHA256 或在后端计算
let oid;
if (precomputedSha256) {
console.log('Using precomputed SHA256:', precomputedSha256);
oid = precomputedSha256;
} else {
console.log('Computing SHA256 on server (may timeout for large files)...');
oid = await this.sha256(file);
console.log('SHA256:', oid);
}
// 2. 获取文件样本(前512字节的base64)
const sampleBytes = new Uint8Array(await file.slice(0, 512).arrayBuffer());
const sample = btoa(String.fromCharCode(...sampleBytes));
// 3. Preupload 检查
console.log('Preupload check...');
const preuploadResult = await this.preupload(filePath, file.size, sample);
console.log('Preupload result:', JSON.stringify(preuploadResult));
const fileInfo = preuploadResult.files?.[0];
const needsLfs = fileInfo?.uploadMode === 'lfs';
console.log('Needs LFS:', needsLfs);
if (needsLfs) {
// 4. LFS Batch - 获取上传 URL
console.log('LFS batch request...');
const batchResult = await this.lfsBatch(oid, file.size);
console.log('LFS batch result:', JSON.stringify(batchResult));
const obj = batchResult.objects?.[0];
if (obj?.error) {
throw new Error(`LFS error: ${obj.error.message}`);
}
// 5. 上传到 LFS 存储(如果需要)
if (obj?.actions?.upload) {
console.log('Uploading to LFS storage...');
await this.uploadToLFS(obj.actions.upload, file, oid);
console.log('LFS upload complete');
} else {
console.log('File already exists in LFS');
}
// 6. 提交 LFS 文件引用
console.log('Committing LFS file...');
const commitResult = await this.commitLfsFile(filePath, oid, file.size, commitMessage);
console.log('Commit result:', JSON.stringify(commitResult));
} else {
// 非 LFS 文件:直接 base64 提交(小文本文件)
console.log('Direct commit (non-LFS)...');
await this.commitDirectFile(filePath, file, commitMessage);
}
const fileUrl = `${this.baseURL}/datasets/${this.repo}/resolve/main/${filePath}`;
return {
success: true,
filePath,
fileUrl,
fileSize: file.size,
oid
};
} catch (error) {
console.error('HuggingFace upload error:', error.message);
throw error;
}
}
/**
* 直接提交文件 LFS用于小文本文件
*/
async commitDirectFile(filePath, file, commitMessage) {
const url = `${this.baseURL}/api/datasets/${this.repo}/commit/main`;
const content = btoa(String.fromCharCode(...new Uint8Array(await file.arrayBuffer())));
const body = [
JSON.stringify({
key: 'header',
value: { summary: commitMessage }
}),
JSON.stringify({
key: 'file',
value: {
path: filePath,
content: content,
encoding: 'base64'
}
})
].join('\n');
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/x-ndjson'
},
body
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Direct commit failed: ${response.status} - ${error}`);
}
return await response.json();
}
/**
* 删除文件
*/
async deleteFile(filePath, commitMessage = 'Delete file') {
const url = `${this.baseURL}/api/datasets/${this.repo}/commit/main`;
const body = [
JSON.stringify({
key: 'header',
value: { summary: commitMessage }
}),
JSON.stringify({
key: 'deletedFile',
value: { path: filePath }
})
].join('\n');
const response = await fetch(url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/x-ndjson'
},
body
});
return response.ok;
}
/**
* 获取文件内容用于私有仓库代理
*/
async getFileContent(filePath) {
const fileUrl = `${this.baseURL}/datasets/${this.repo}/resolve/main/${filePath}`;
return await fetch(fileUrl, {
headers: this.isPrivate ? { 'Authorization': `Bearer ${this.token}` } : {}
});
}
/**
* 获取文件 URL
*/
getFileURL(filePath) {
return `${this.baseURL}/datasets/${this.repo}/resolve/main/${filePath}`;
}
}
+5 -1
View File
@@ -61,6 +61,8 @@ export async function fetchUploadConfig(env, context = null) {
settings.telegram.channels = settings.telegram.channels.filter((channel) => channel.enabled); settings.telegram.channels = settings.telegram.channels.filter((channel) => channel.enabled);
settings.cfr2.channels = settings.cfr2.channels.filter((channel) => channel.enabled); settings.cfr2.channels = settings.cfr2.channels.filter((channel) => channel.enabled);
settings.s3.channels = settings.s3.channels.filter((channel) => channel.enabled); settings.s3.channels = settings.s3.channels.filter((channel) => channel.enabled);
settings.discord.channels = settings.discord.channels.filter((channel) => channel.enabled);
settings.huggingface.channels = settings.huggingface.channels.filter((channel) => channel.enabled);
// 根据容量限制过滤渠道(仅 R2 和 S3) // 根据容量限制过滤渠道(仅 R2 和 S3)
// 需要 context 来调用 getIndexMeta // 需要 context 来调用 getIndexMeta
@@ -76,7 +78,9 @@ export async function fetchUploadConfig(env, context = null) {
return { return {
telegram: { channels: [] }, telegram: { channels: [] },
cfr2: { channels: [] }, cfr2: { channels: [] },
s3: { channels: [] } s3: { channels: [] },
discord: { channels: [] },
huggingface: { channels: [] }
}; };
} }
} }
+1 -1
View File
@@ -1 +1 @@
<!doctype html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/logo.png"><link rel="apple-touch-icon" href="/logo.png"><link rel="mask-icon" href="/logo.png" color="#f4b400"><meta name="description" content="Sanyue ImgHub - A modern file hosting platform"><meta name="keywords" content="Sanyue, ImgHub, file hosting, image hosting, cloud storage"><meta name="author" content="SanyueQi"><title>Sanyue ImgHub</title><script defer="defer" src="/js/chunk-vendors.780b6559.js"></script><script defer="defer" src="/js/app.5bf21eb7.js"></script><link href="/css/chunk-vendors.4363ed49.css" rel="stylesheet"><link href="/css/app.14879ca1.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sanyue_imghub doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html> <!doctype html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/logo.png"><link rel="apple-touch-icon" href="/logo.png"><link rel="mask-icon" href="/logo.png" color="#f4b400"><meta name="description" content="Sanyue ImgHub - A modern file hosting platform"><meta name="keywords" content="Sanyue, ImgHub, file hosting, image hosting, cloud storage"><meta name="author" content="SanyueQi"><title>Sanyue ImgHub</title><script defer="defer" src="/js/chunk-vendors.780b6559.js"></script><script defer="defer" src="/js/app.e18ad382.js"></script><link href="/css/chunk-vendors.4363ed49.css" rel="stylesheet"><link href="/css/app.14879ca1.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but sanyue_imghub doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
+23
View File
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
+2 -2
View File
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.
File diff suppressed because one or more lines are too long
Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More