diff --git a/functions/file/[[path]].js b/functions/file/[[path]].js index f99cf4ad..f16ef8b9 100644 --- a/functions/file/[[path]].js +++ b/functions/file/[[path]].js @@ -879,55 +879,31 @@ async function handleHuggingFaceFile(context, metadata, encodedFileName, fileTyp } // 构建文件 URL - const huggingfaceAPI = new HuggingFaceAPI(hfToken, hfRepo, hfIsPrivate); - let fileUrl = metadata.HfFileUrl || huggingfaceAPI.getFileURL(hfFilePath); - fileUrl = await huggingfaceAPI.resolveFileURL(hfFilePath, { fileUrl }); - const fileSize = await huggingfaceAPI.getRemoteFileSize(hfFilePath, { fileUrl, metadata }); - - // 构建响应头 - const headers = new Headers(); - setCommonHeaders(headers, encodedFileName, fileType, getFileCacheControl(context)); - if (fileSize) { - headers.set('Content-Length', fileSize.toString()); - } + const fileUrl = metadata.HfFileUrl || `https://huggingface.co/datasets/${hfRepo}/resolve/main/${hfFilePath}`; // 处理 HEAD 请求 if (request.method === 'HEAD') { + const headers = new Headers(); + setCommonHeaders(headers, encodedFileName, fileType, getFileCacheControl(context)); return handleHeadRequest(headers); } // 构建请求头 const fetchHeaders = {}; + // 私有仓库需要 Authorization + if (hfIsPrivate && hfToken) { + fetchHeaders['Authorization'] = `Bearer ${hfToken}`; + } + // 支持 Range 请求 const range = request.headers.get('Range'); if (range) { fetchHeaders['Range'] = range; - - if (fileSize) { - const proxyRange = huggingfaceAPI.getProxyRange(range, fileSize); - if (!proxyRange) { - return new Response('Range Not Satisfiable', { - status: 416, - headers: { - 'Content-Range': `bytes */${fileSize}`, - 'Accept-Ranges': 'bytes' - } - }); - } - - const rangeResponse = await huggingfaceAPI.fetchRange(hfFilePath, proxyRange.start, proxyRange.end, { fileUrl }); - setRangeHeaders(headers, rangeResponse.start, rangeResponse.end, fileSize); - - return new Response(rangeResponse.body, { - status: 206, - headers - }); - } } - const response = await huggingfaceAPI.fetchFile(hfFilePath, { - fileUrl, + const response = await fetch(fileUrl, { + method: 'GET', headers: fetchHeaders }); @@ -935,6 +911,10 @@ async function handleHuggingFaceFile(context, metadata, encodedFileName, fileTyp return new Response(`Error: Failed to fetch from HuggingFace - ${response.status}`, { status: response.status }); } + // 构建响应头 + const headers = new Headers(); + setCommonHeaders(headers, encodedFileName, fileType, getFileCacheControl(context)); + // 复制相关头部 if (response.headers.get('Content-Length')) { headers.set('Content-Length', response.headers.get('Content-Length')); diff --git a/functions/utils/storage/huggingfaceAPI.js b/functions/utils/storage/huggingfaceAPI.js index 200a0539..375f84a7 100644 --- a/functions/utils/storage/huggingfaceAPI.js +++ b/functions/utils/storage/huggingfaceAPI.js @@ -12,9 +12,6 @@ * SHA256 由前端预计算传入，避免后端 CPU 超时 */ -const HF_PROXY_RANGE_CHUNK_SIZE = 4 * 1024 * 1024; -const HF_UPSTREAM_RANGE_CHUNK_SIZE = 4 * 1024 * 1024; - export class HuggingFaceAPI { constructor(token, repo, isPrivate = false) { this.token = token; @@ -508,7 +505,10 @@ export class HuggingFaceAPI { * 获取文件内容（用于私有仓库代理） */ async getFileContent(filePath) { - return await this.fetchFile(filePath); + const fileUrl = `${this.baseURL}/datasets/${this.repo}/resolve/main/${filePath}`; + return await fetch(fileUrl, { + headers: this.isPrivate ? { 'Authorization': `Bearer ${this.token}` } : {} + }); } /** @@ -517,343 +517,4 @@ export class HuggingFaceAPI { getFileURL(filePath) { return `${this.baseURL}/datasets/${this.repo}/resolve/main/${filePath}`; } - - getRequestHeaders(extraHeaders = {}) { - const headers = new Headers(extraHeaders); - - if (this.isPrivate && this.token && shouldSendAuthorization(headers.get('X-HF-Target-URL')) && !headers.has('Authorization')) { - headers.set('Authorization', `Bearer ${this.token}`); - } - headers.delete('X-HF-Target-URL'); - - return headers; - } - - async fetchFile(filePath, options = {}) { - const fileUrl = options.fileUrl || this.getFileURL(filePath); - return await fetch(fileUrl, { - method: options.method || 'GET', - headers: this.getRequestHeaders({ - ...(options.headers || {}), - 'X-HF-Target-URL': fileUrl, - }), - }); - } - - async resolveFileURL(filePath, options = {}) { - const fileUrl = options.fileUrl || this.getFileURL(filePath); - - try { - const response = await fetch(fileUrl, { - method: 'HEAD', - headers: this.getRequestHeaders({ 'X-HF-Target-URL': fileUrl }), - redirect: 'manual', - }); - - const location = response.headers.get('Location'); - if (response.status >= 300 && response.status < 400 && location) { - return new URL(location, fileUrl).toString(); - } - } catch (error) { - console.warn('HuggingFace download URL resolve failed:', error.message); - } - - if (shouldSendAuthorization(fileUrl)) { - try { - const response = await fetch(fileUrl, { - method: 'GET', - headers: this.getRequestHeaders({ - Range: 'bytes=0-0', - 'X-HF-Target-URL': fileUrl, - }), - redirect: 'manual', - }); - - try { - const location = response.headers.get('Location'); - if (response.status >= 300 && response.status < 400 && location) { - return new URL(location, fileUrl).toString(); - } - } finally { - if (response.body) { - await response.body.cancel().catch(() => {}); - } - } - } catch (error) { - console.warn('HuggingFace range URL resolve failed:', error.message); - } - } - - return fileUrl; - } - - async getRemoteFileSize(filePath, options = {}) { - const metadataSize = getExactMetadataFileSize(options.metadata); - if (metadataSize) { - return metadataSize; - } - - try { - const headResponse = await this.fetchFile(filePath, { - fileUrl: options.fileUrl, - method: 'HEAD', - }); - - if (headResponse.ok) { - const contentLength = Number(headResponse.headers.get('Content-Length')); - if (Number.isFinite(contentLength) && contentLength > 0) { - return Math.floor(contentLength); - } - } - } catch (error) { - console.warn('HuggingFace HEAD size probe failed:', error.message); - } - - try { - const rangeResponse = await this.fetchFile(filePath, { - fileUrl: options.fileUrl, - headers: { Range: 'bytes=0-0' }, - }); - - try { - const contentRange = parseContentRange(rangeResponse.headers.get('Content-Range')); - if (contentRange?.total) { - return contentRange.total; - } - - if (rangeResponse.status === 200) { - const contentLength = Number(rangeResponse.headers.get('Content-Length')); - if (Number.isFinite(contentLength) && contentLength > 0) { - return Math.floor(contentLength); - } - } - } finally { - if (rangeResponse.body) { - await rangeResponse.body.cancel().catch(() => {}); - } - } - } catch (error) { - console.warn('HuggingFace range size probe failed:', error.message); - } - - return getApproximateMetadataFileSize(options.metadata); - } - - getProxyRange(rangeHeader, totalSize) { - const range = parseRangeHeader(rangeHeader, totalSize); - if (!range) { - return null; - } - - const rangeEnd = range.openEnded - ? Math.min(range.start + HF_PROXY_RANGE_CHUNK_SIZE - 1, range.end) - : range.end; - - return { - start: range.start, - end: rangeEnd, - openEnded: range.openEnded, - }; - } - - async fetchRange(filePath, rangeStart, rangeEnd, options = {}) { - const fileUrl = options.fileUrl || this.getFileURL(filePath); - const chunks = []; - let nextStart = rangeStart; - let totalBytes = 0; - - while (nextStart <= rangeEnd) { - const upstreamEnd = Math.min(nextStart + HF_UPSTREAM_RANGE_CHUNK_SIZE - 1, rangeEnd); - const response = await this.fetchFile(filePath, { - fileUrl, - headers: { Range: `bytes=${nextStart}-${upstreamEnd}` }, - }); - - if (!response.ok && response.status !== 206) { - if (totalBytes > 0) { - break; - } - throw new Error(`HuggingFace range fetch failed: ${response.status}`); - } - - const contentRange = parseContentRange(response.headers.get('Content-Range')); - let readableBytes = upstreamEnd - nextStart + 1; - - if (contentRange) { - if (contentRange.start !== nextStart || contentRange.end < nextStart) { - if (totalBytes > 0) { - break; - } - throw new Error(`Unexpected HuggingFace Content-Range: ${response.headers.get('Content-Range')}`); - } - readableBytes = Math.min(contentRange.end - contentRange.start + 1, rangeEnd - nextStart + 1); - } else if (response.status === 200 && nextStart !== 0) { - if (totalBytes > 0) { - break; - } - throw new Error('HuggingFace did not honor range request'); - } else { - const contentLength = Number(response.headers.get('Content-Length')); - if (Number.isFinite(contentLength) && contentLength > 0) { - readableBytes = Math.min(contentLength, rangeEnd - nextStart + 1); - } - } - - const bytes = await readResponseBytes(response.body, readableBytes); - if (bytes.byteLength === 0) { - break; - } - - chunks.push(bytes); - totalBytes += bytes.byteLength; - nextStart += bytes.byteLength; - } - - if (totalBytes <= 0) { - throw new Error('Empty HuggingFace range response'); - } - - return { - body: concatUint8Arrays(chunks, totalBytes), - start: rangeStart, - end: rangeStart + totalBytes - 1, - contentLength: totalBytes, - }; - } -} - -function shouldSendAuthorization(fileUrl) { - if (!fileUrl) { - return true; - } - - try { - return new URL(fileUrl).hostname === 'huggingface.co'; - } catch { - return true; - } -} - -function parseContentRange(contentRange) { - if (!contentRange) { - return null; - } - - const match = contentRange.match(/^bytes\s+(\d+)-(\d+)\/(\d+|\*)$/i); - if (!match) { - return null; - } - - return { - start: parseInt(match[1], 10), - end: parseInt(match[2], 10), - total: match[3] === '*' ? null : parseInt(match[3], 10), - }; -} - -function parseRangeHeader(rangeHeader, totalSize) { - if (!rangeHeader || !totalSize) { - return null; - } - - const match = String(rangeHeader).trim().match(/^bytes=(\d*)-(\d*)$/i); - if (!match || (match[1] === '' && match[2] === '')) { - return null; - } - - let start; - let end; - let openEnded = false; - - if (match[1] === '') { - const suffixLength = parseInt(match[2], 10); - if (!Number.isFinite(suffixLength) || suffixLength <= 0) { - return null; - } - start = Math.max(totalSize - suffixLength, 0); - end = totalSize - 1; - } else { - start = parseInt(match[1], 10); - openEnded = match[2] === ''; - end = openEnded ? totalSize - 1 : parseInt(match[2], 10); - } - - if (!Number.isFinite(start) || !Number.isFinite(end) || start >= totalSize || start > end) { - return null; - } - - return { - start, - end: Math.min(end, totalSize - 1), - openEnded, - }; -} - -function getExactMetadataFileSize(metadata = {}) { - const fileSizeBytes = Number(metadata?.FileSizeBytes); - if (Number.isFinite(fileSizeBytes) && fileSizeBytes > 0) { - return Math.floor(fileSizeBytes); - } - - return null; -} - -function getApproximateMetadataFileSize(metadata = {}) { - const fileSizeMB = Number(metadata?.FileSize); - if (Number.isFinite(fileSizeMB) && fileSizeMB > 0) { - return Math.floor(fileSizeMB * 1024 * 1024); - } - - return null; -} - -async function readResponseBytes(body, maxBytes) { - if (!body || maxBytes <= 0) { - return new Uint8Array(0); - } - - const reader = body.getReader(); - let remaining = maxBytes; - let totalBytes = 0; - const chunks = []; - - try { - while (remaining > 0) { - const { done, value } = await reader.read(); - if (done) { - break; - } - - const chunk = value.byteLength > remaining ? value.slice(0, remaining) : value; - chunks.push(chunk); - - totalBytes += chunk.byteLength; - remaining -= chunk.byteLength; - - if (chunk.byteLength < value.byteLength) { - await reader.cancel(); - break; - } - } - } finally { - reader.releaseLock(); - } - - return concatUint8Arrays(chunks, totalBytes); -} - -function concatUint8Arrays(chunks, totalBytes) { - if (chunks.length === 1 && chunks[0].byteLength === totalBytes) { - return chunks[0]; - } - - const result = new Uint8Array(totalBytes); - let offset = 0; - - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.byteLength; - } - - return result; }