Merge pull request #576 from outlook84/pr

feat: add HuggingFace multipart upload completion proxy

deploy/worker/index.js

@@ -44,6 +44,7 @@ import * as apiManageList from '../../functions/api/manage/list.js';
 import * as apiManageQuota from '../../functions/api/manage/quota.js';
 import * as apiPublicList from '../../functions/api/public/list.js';
 import * as uploadHuggingfaceCommitUpload from '../../functions/upload/huggingface/commitUpload.js';
+import * as uploadHuggingfaceCompleteMultipart from '../../functions/upload/huggingface/completeMultipart.js';
 import * as uploadHuggingfaceGetUploadUrl from '../../functions/upload/huggingface/getUploadUrl.js';
 import * as apiChannels from '../../functions/api/channels.js';
 import * as apiDirectoryTree from '../../functions/api/directoryTree.js';
@@ -93,6 +94,7 @@ const routes = [
     { path: '/api/manage/quota', module: apiManageQuota, middlewares: [mw_api, mw_api_manage] },
     { path: '/api/public/list', module: apiPublicList, middlewares: [mw_api] },
     { path: '/upload/huggingface/commitUpload', module: uploadHuggingfaceCommitUpload, middlewares: [mw_upload] },
+    { path: '/upload/huggingface/completeMultipart', module: uploadHuggingfaceCompleteMultipart, middlewares: [mw_upload] },
     { path: '/upload/huggingface/getUploadUrl', module: uploadHuggingfaceGetUploadUrl, middlewares: [mw_upload] },
     { path: '/api/channels', module: apiChannels, middlewares: [mw_api] },
     { path: '/api/directoryTree', module: apiDirectoryTree, middlewares: [mw_api] },

functions/upload/huggingface/commitUpload.js

@@ -22,7 +22,7 @@ export async function onRequestPost(context) {
         }
 
         const body = await request.json();
-        const { fullId, filePath, sha256, fileSize, fileName, fileType, channelName, multipartParts } = body;
+        const { fullId, filePath, sha256, fileSize, fileName, fileType, channelName } = body;
 
         if (!fullId || !filePath || !sha256 || !fileSize) {
             return createResponse(JSON.stringify({
@@ -73,11 +73,6 @@ export async function onRequestPost(context) {
 
         const huggingfaceAPI = new HuggingFaceAPI(hfChannel.token, hfChannel.repo, hfChannel.isPrivate || false);
 
-        // 如果有 multipart parts，需要先完成 multipart 上传
-        if (multipartParts && multipartParts.length > 0) {
-            console.log('Completing multipart upload...');
-        }
-
         // 提交 LFS 文件引用
         console.log('Committing LFS file...');
         const commitResult = await huggingfaceAPI.commitLfsFile(

functions/upload/huggingface/completeMultipart.js

@@ -0,0 +1,111 @@
+import { userAuthCheck, UnauthorizedResponse } from '../../utils/auth/userAuth.js';
+import { createResponse } from '../uploadTools.js';
+
+export async function onRequestPost(context) {
+    const { request, env } = context;
+    const url = new URL(request.url);
+
+    try {
+        const requiredPermission = 'upload';
+        if (!await userAuthCheck(env, url, request, requiredPermission)) {
+            return UnauthorizedResponse('Unauthorized');
+        }
+
+        const target = url.searchParams.get('target');
+        if (!target) {
+            return createResponse(JSON.stringify({ error: 'Missing target URL' }), {
+                status: 400,
+                headers: { 'Content-Type': 'application/json' }
+            });
+        }
+
+        let targetUrl;
+        try {
+            targetUrl = new URL(target);
+        } catch {
+            return jsonError('Invalid multipart completion target', 400);
+        }
+
+        if (!isValidCompletionTarget(targetUrl)) {
+            return jsonError('Invalid multipart completion target', 400);
+        }
+
+        const body = await request.text();
+        try {
+            validateMultipartBody(body);
+        } catch (error) {
+            return jsonError(error.message, 400);
+        }
+
+        const completeResponse = await fetch(targetUrl.toString(), {
+            method: 'POST',
+            headers: {
+                'Accept': 'application/vnd.git-lfs+json',
+                'Content-Type': 'application/vnd.git-lfs+json'
+            },
+            body
+        });
+
+        const responseText = await completeResponse.text();
+        if (!completeResponse.ok) {
+            const contentType = completeResponse.headers.get('Content-Type') || 'application/json';
+            const errorBody = responseText || JSON.stringify({
+                error: `Multipart complete failed: ${completeResponse.status}`
+            });
+            return createResponse(errorBody, {
+                status: completeResponse.status,
+                headers: { 'Content-Type': contentType }
+            });
+        }
+
+        return createResponse(responseText || JSON.stringify({ success: true }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' }
+        });
+
+    } catch (error) {
+        console.error('completeMultipart error:', error.message);
+        return createResponse(JSON.stringify({ error: error.message }), {
+            status: 500,
+            headers: { 'Content-Type': 'application/json' }
+        });
+    }
+}
+
+function jsonError(message, status) {
+    return createResponse(JSON.stringify({ error: message }), {
+        status,
+        headers: { 'Content-Type': 'application/json' }
+    });
+}
+
+function isValidCompletionTarget(targetUrl) {
+    return targetUrl.protocol === 'https:' &&
+        targetUrl.hostname === 'huggingface.co' &&
+        targetUrl.pathname === '/api/complete_multipart';
+}
+
+function validateMultipartBody(body) {
+    let parsed;
+    try {
+        parsed = JSON.parse(body);
+    } catch {
+        throw new Error('Invalid multipart completion body');
+    }
+
+    if (!parsed || typeof parsed.oid !== 'string' || !Array.isArray(parsed.parts)) {
+        throw new Error('Invalid multipart completion payload');
+    }
+
+    const hasInvalidPart = parsed.parts.some(part =>
+        !part ||
+        !Number.isInteger(Number(part.partNumber)) ||
+        Number(part.partNumber) <= 0 ||
+        typeof part.etag !== 'string' ||
+        part.etag.length === 0
+    );
+
+    if (hasInvalidPart) {
+        throw new Error('Invalid multipart parts');
+    }
+}

functions/upload/huggingface/getUploadUrl.js

@@ -38,10 +38,11 @@ export async function onRequestPost(context) {
 
         const body = await request.json();
         const { fileSize, fileName, fileType, sha256, fileSample, channelName, uploadNameType, uploadFolder } = body;
+        const normalizedFileType = fileType || 'application/octet-stream';
 
-        if (!fileSize || !fileName || !fileType || !sha256 || !fileSample) {
+        if (!fileSize || !fileName || !sha256 || !fileSample) {
             return createResponse(JSON.stringify({
-                error: 'Missing required fields: fileSize, fileName, fileType, sha256, fileSample'
+                error: 'Missing required fields: fileSize, fileName, sha256, fileSample'
             }), {
                 status: 400,
                 headers: { 'Content-Type': 'application/json' }
@@ -86,7 +87,7 @@ export async function onRequestPost(context) {
         }
 
         // 使用统一的文件命名函数生成文件ID
-        const fullId = await buildUniqueFileId(context, fileName, fileType || 'application/octet-stream');
+        const fullId = await buildUniqueFileId(context, fileName, normalizedFileType);
 
         // 生成唯一标识符前缀（UUID格式），加在文件名前面
         const uniquePrefix = crypto.randomUUID();
@@ -98,6 +99,7 @@ export async function onRequestPost(context) {
         // 获取 LFS 上传信息
         const huggingfaceAPI = new HuggingFaceAPI(hfChannel.token, hfChannel.repo, hfChannel.isPrivate || false);
         const uploadInfo = await huggingfaceAPI.getLfsUploadInfo(fileSize, filePath, sha256, fileSample);
+        rewriteMultipartCompletionUrl(url, uploadInfo);
 
         // 返回上传信息
         return createResponse(JSON.stringify({
@@ -121,3 +123,13 @@ export async function onRequestPost(context) {
         });
     }
 }
+
+function rewriteMultipartCompletionUrl(requestUrl, uploadInfo) {
+    const uploadAction = uploadInfo?.uploadAction;
+    if (!uploadAction?.header?.chunk_size || !uploadAction.href) {
+        return;
+    }
+
+    const originalCompletionUrl = uploadAction.href;
+    uploadAction.href = `${requestUrl.origin}/upload/huggingface/completeMultipart?target=${encodeURIComponent(originalCompletionUrl)}`;
+}