refactor: rely on config for channel metadata

This commit is contained in:
MarSeventh
2026-06-03 10:57:51 +08:00
parent fed53e807d
commit d061353ddd
13 changed files with 125 additions and 175 deletions
+4 -2
View File
@@ -5,6 +5,7 @@
*/
import { getDatabase } from '../../../../utils/databaseAdapter.js';
import { cleanPersistedMetadata } from '../../../../utils/metadata/metadataSecurity.js';
// CORS 跨域响应头
const corsHeaders = {
@@ -73,15 +74,16 @@ export async function onRequestPost(context) {
// 恢复文件数据 - 并行写入
const filePromises = Object.entries(data).map(async ([key, fileData]) => {
try {
const metadata = cleanPersistedMetadata(fileData.metadata);
if (fileData.value) {
// 有 value 的文件(如 Telegram/Discord 分块文件)
await db.put(key, fileData.value, {
metadata: fileData.metadata
metadata
});
} else if (fileData.metadata) {
// 只有元数据的文件
await db.put(key, '', {
metadata: fileData.metadata
metadata
});
}
return { success: true };
+6 -6
View File
@@ -1,6 +1,7 @@
import { purgeCFCache, purgeRandomFileListCache, purgePublicFileListCache } from "../../../utils/purgeCache";
import { addFileToIndex } from "../../../utils/indexManager.js";
import { getDatabase } from "../../../utils/databaseAdapter.js";
import { cleanPersistedMetadata } from "../../../utils/metadata/metadataSecurity.js";
export async function onRequest(context) {
// Contents of context object
@@ -9,8 +10,6 @@ export async function onRequest(context) {
env, // same as existing Worker API
params, // if filename includes [id] or [[path]]
waitUntil, // same as ctx.waitUntil in existing Worker API
next, // used for middleware or to fetch assets
data, // arbitrary space for passing data between middlewares
} = context;
// 组装 CDN URL
@@ -30,8 +29,9 @@ export async function onRequest(context) {
//change the metadata
value.metadata.ListType = "Block"
await db.put(params.path, value.value, {metadata: value.metadata});
const info = JSON.stringify({ success: true, listType: value.metadata.ListType });
const metadata = cleanPersistedMetadata(value.metadata);
await db.put(params.path, value.value, {metadata});
const info = JSON.stringify({ success: true, listType: metadata.ListType });
// 清除CDN缓存
await purgeCFCache(env, cdnUrl);
@@ -42,7 +42,7 @@ export async function onRequest(context) {
await purgePublicFileListCache(url.origin, normalizedFolder);
// 更新索引
waitUntil(addFileToIndex(context, params.path, value.metadata));
waitUntil(addFileToIndex(context, params.path, metadata));
return new Response(info);
}
}
+5 -3
View File
@@ -1,6 +1,7 @@
import { addFileToIndex } from '../../../utils/indexManager.js';
import { getDatabase } from '../../../utils/databaseAdapter.js';
import { buildFileMetadataForManagement } from '../../../utils/metadata/metadataView.js';
import { cleanPersistedMetadata } from '../../../utils/metadata/metadataSecurity.js';
// CORS 跨域响应头
const corsHeaders = {
@@ -94,16 +95,17 @@ export async function onRequest(context) {
if (typeof body.FileType === 'string') {
updatedMetadata.FileType = body.FileType;
}
const metadataToSave = cleanPersistedMetadata(updatedMetadata);
// 保存更新后的 metadata
await db.put(fileId, fileData.value, { metadata: updatedMetadata });
await db.put(fileId, fileData.value, { metadata: metadataToSave });
// 更新索引
waitUntil(addFileToIndex(context, fileId, updatedMetadata));
waitUntil(addFileToIndex(context, fileId, metadataToSave));
return new Response(JSON.stringify({
success: true,
metadata: await buildFileMetadataForManagement(db, env, updatedMetadata),
metadata: await buildFileMetadataForManagement(db, env, metadataToSave),
}), {
status: 200,
headers: { 'Content-Type': 'application/json', ...corsHeaders },
+5 -31
View File
@@ -5,13 +5,10 @@ import { getDatabase } from '../../../utils/databaseAdapter.js';
import { sanitizeUploadFolder } from "../../../upload/uploadTools.js";
import { WebDAVAPI } from "../../../utils/storage/webdavAPI.js";
import {
resolveDiscordCredentials,
resolveHuggingFaceCredentials,
resolveS3Credentials,
resolveTelegramCredentials,
resolveWebDAVCredentials,
} from "../../../utils/metadata/channelCredentials.js";
import { stripConfigDerivedMetadata, stripSensitiveMetadata } from "../../../utils/metadata/metadataSecurity.js";
import { cleanPersistedMetadata } from "../../../utils/metadata/metadataSecurity.js";
export async function onRequest(context) {
const { request, env, params, waitUntil } = context;
@@ -159,12 +156,10 @@ async function moveFile(env, fileId, newFileId, cdnUrl, url) {
// S3 渠道的图片,需要移动S3中对应的图片
if (img.metadata?.Channel === 'S3') {
const { success, newKey, error } = await moveS3File(env, img, newFileId);
if (success) {
// 更新 metadata
img.metadata.S3FileKey = newKey;
} else {
// do nothing
if (!success) {
throw new Error(error || 'S3 Move Failed');
}
img.metadata.S3FileKey = newKey;
}
// WebDAV 渠道的图片,需要移动 WebDAV 中对应的文件
@@ -184,7 +179,7 @@ async function moveFile(env, fileId, newFileId, cdnUrl, url) {
// 更新文件夹信息,根目录为空,否则为 aaa/123/ 的格式
const DirectoryPath = newFileId.split('/').slice(0, -1).join('/') === '' ? '' : newFileId.split('/').slice(0, -1).join('/') + '/';
img.metadata.Directory = DirectoryPath;
img.metadata = await stripMetadataAfterConfigResolution(db, env, img.metadata);
img.metadata = cleanPersistedMetadata(img.metadata);
// 更新KV存储
await db.put(newFileId, img.value, { metadata: img.metadata });
@@ -206,27 +201,6 @@ async function moveFile(env, fileId, newFileId, cdnUrl, url) {
}
}
async function stripMetadataAfterConfigResolution(db, env, metadata) {
let credentials = null;
if (metadata?.Channel === 'S3') {
credentials = await resolveS3Credentials(db, env, metadata);
} else if (metadata?.Channel === 'TelegramNew') {
credentials = await resolveTelegramCredentials(db, env, metadata);
} else if (metadata?.Channel === 'Discord') {
credentials = await resolveDiscordCredentials(db, env, metadata);
} else if (metadata?.Channel === 'HuggingFace') {
credentials = await resolveHuggingFaceCredentials(db, env, metadata);
} else if (metadata?.Channel === 'WebDAV') {
credentials = await resolveWebDAVCredentials(db, env, metadata);
}
if (credentials?.source !== 'config') {
return metadata;
}
return stripConfigDerivedMetadata(stripSensitiveMetadata(metadata));
}
// 移动 S3 渠道的图片
async function moveS3File(env, img, newFileId) {
const db = getDatabase(env);
+5 -35
View File
@@ -5,16 +5,10 @@ import { getDatabase } from '../../../utils/databaseAdapter.js';
import { sanitizeUploadFolder } from "../../../upload/uploadTools.js";
import { WebDAVAPI } from "../../../utils/storage/webdavAPI.js";
import {
resolveDiscordCredentials,
resolveHuggingFaceCredentials,
resolveS3Credentials,
resolveTelegramCredentials,
resolveWebDAVCredentials,
} from "../../../utils/metadata/channelCredentials.js";
import {
stripConfigDerivedMetadataInPlace,
stripSensitiveMetadataInPlace,
} from "../../../utils/metadata/metadataSecurity.js";
import { cleanPersistedMetadataInPlace } from "../../../utils/metadata/metadataSecurity.js";
import { buildFileMetadataForManagement } from "../../../utils/metadata/metadataView.js";
// CORS 跨域响应头
@@ -140,12 +134,10 @@ export async function onRequest(context) {
// S3 渠道的图片,需要移动 S3 中对应的图片
if (metadata?.Channel === 'S3') {
const { success, newKey, error } = await moveS3File(env, fileData, newFileId);
if (success) {
// 更新 metadata
metadata.S3FileKey = newKey;
} else {
// do nothing
if (!success) {
throw new Error(error || 'S3 Move Failed');
}
metadata.S3FileKey = newKey;
}
// WebDAV 渠道的图片,需要移动 WebDAV 中对应的文件
@@ -171,7 +163,7 @@ export async function onRequest(context) {
// 更新文件夹信息,根目录为空,否则为 aaa/123/ 的格式
const DirectoryPath = newFileId.split('/').slice(0, -1).join('/') === '' ? '' : newFileId.split('/').slice(0, -1).join('/') + '/';
metadata.Directory = DirectoryPath;
await stripMetadataInPlaceAfterConfigResolution(db, env, metadata);
cleanPersistedMetadataInPlace(metadata);
// 更新 KV 存储
await db.put(newFileId, fileData.value, { metadata });
@@ -211,28 +203,6 @@ export async function onRequest(context) {
}
}
async function stripMetadataInPlaceAfterConfigResolution(db, env, metadata) {
let credentials = null;
if (metadata?.Channel === 'S3') {
credentials = await resolveS3Credentials(db, env, metadata);
} else if (metadata?.Channel === 'TelegramNew') {
credentials = await resolveTelegramCredentials(db, env, metadata);
} else if (metadata?.Channel === 'Discord') {
credentials = await resolveDiscordCredentials(db, env, metadata);
} else if (metadata?.Channel === 'HuggingFace') {
credentials = await resolveHuggingFaceCredentials(db, env, metadata);
} else if (metadata?.Channel === 'WebDAV') {
credentials = await resolveWebDAVCredentials(db, env, metadata);
}
if (credentials?.source !== 'config') {
return;
}
stripSensitiveMetadataInPlace(metadata);
stripConfigDerivedMetadataInPlace(metadata);
}
// 移动 S3 渠道的图片
async function moveS3File(env, img, newFileId) {
const db = getDatabase(env);
+5 -3
View File
@@ -1,7 +1,8 @@
import { purgeCFCache } from "../../../utils/purgeCache.js";
import { addFileToIndex } from "../../../utils/indexManager.js";
import { getDatabase } from "../../../utils/databaseAdapter.js";
import { mergeTags, normalizeTags, validateTag } from "../../../utils/tagHelpers.js";
import { mergeTags, validateTag } from "../../../utils/tagHelpers.js";
import { cleanPersistedMetadata } from "../../../utils/metadata/metadataSecurity.js";
/**
* Tag Management API for Single Files
@@ -168,10 +169,11 @@ async function handleUpdateTags(context, db, fileId, hostname) {
// Update metadata
fileData.metadata.Tags = updatedTags;
const metadata = cleanPersistedMetadata(fileData.metadata);
// Save to database
await db.put(fileId, fileData.value, {
metadata: fileData.metadata
metadata
});
// Clear CDN cache asynchronously (don't wait for it to complete)
@@ -179,7 +181,7 @@ async function handleUpdateTags(context, db, fileId, hostname) {
waitUntil(purgeCFCache(context.env, cdnUrl));
// Update file index asynchronously
waitUntil(addFileToIndex(context, fileId, fileData.metadata));
waitUntil(addFileToIndex(context, fileId, metadata));
return new Response(JSON.stringify({
success: true,
+4 -2
View File
@@ -2,6 +2,7 @@ import { purgeCFCache } from "../../../utils/purgeCache.js";
import { batchAddFilesToIndex } from "../../../utils/indexManager.js";
import { getDatabase } from "../../../utils/databaseAdapter.js";
import { mergeTags, validateTag } from "../../../utils/tagHelpers.js";
import { cleanPersistedMetadata } from "../../../utils/metadata/metadataSecurity.js";
/**
* Batch Tag Management API
@@ -118,10 +119,11 @@ export async function onRequest(context) {
// Update metadata
fileData.metadata.Tags = updatedTags;
const metadata = cleanPersistedMetadata(fileData.metadata);
// Save to database
await db.put(fileId, fileData.value, {
metadata: fileData.metadata
metadata
});
// Clear CDN cache (async)
@@ -131,7 +133,7 @@ export async function onRequest(context) {
// Track updated file for batch index update
updatedFiles.push({
fileId: fileId,
metadata: fileData.metadata
metadata
});
results.updated++;
+6 -6
View File
@@ -1,6 +1,7 @@
import { purgeCFCache, purgeRandomFileListCache, purgePublicFileListCache } from "../../../utils/purgeCache";
import { addFileToIndex } from "../../../utils/indexManager.js";
import { getDatabase } from "../../../utils/databaseAdapter.js";
import { cleanPersistedMetadata } from "../../../utils/metadata/metadataSecurity.js";
export async function onRequest(context) {
// Contents of context object
@@ -9,8 +10,6 @@ export async function onRequest(context) {
env, // same as existing Worker API
params, // if filename includes [id] or [[path]]
waitUntil, // same as ctx.waitUntil in existing Worker API
next, // used for middleware or to fetch assets
data, // arbitrary space for passing data between middlewares
} = context;
// 组装 CDN URL
@@ -30,8 +29,9 @@ export async function onRequest(context) {
//change the metadata
value.metadata.ListType = "White"
await db.put(params.path, value.value, {metadata: value.metadata});
const info = JSON.stringify({ success: true, listType: value.metadata.ListType });
const metadata = cleanPersistedMetadata(value.metadata);
await db.put(params.path, value.value, {metadata});
const info = JSON.stringify({ success: true, listType: metadata.ListType });
// 清除CDN缓存
await purgeCFCache(env, cdnUrl);
@@ -42,7 +42,7 @@ export async function onRequest(context) {
await purgePublicFileListCache(url.origin, normalizedFolder);
// 更新索引
waitUntil(addFileToIndex(context, params.path, value.metadata));
waitUntil(addFileToIndex(context, params.path, metadata));
return new Response(info);
}
}
+2 -6
View File
@@ -751,14 +751,10 @@ async function getS3CdnFileUrl(env, metadata) {
const db = getDatabase(env);
const s3Credentials = await resolveS3Credentials(db, env, metadata);
const key = s3Credentials.key || metadata?.S3FileKey;
const configCdnFileUrl = buildCdnFileUrl(s3Credentials.cdnDomain, key);
if (s3Credentials.source === 'config') {
return configCdnFileUrl;
}
return configCdnFileUrl || metadata?.S3CdnFileUrl || '';
return buildCdnFileUrl(s3Credentials.cdnDomain, key);
} catch (error) {
console.warn('Failed to build S3 CDN file URL:', error.message);
return metadata?.S3CdnFileUrl || '';
return '';
}
}
+2 -2
View File
@@ -304,8 +304,8 @@ D1Database.prototype.extractMetadataFields = function(metadata) {
channel: metadata.Channel || null,
channelName: metadata.ChannelName || null,
tgFileId: metadata.TgFileId || null,
tgChatId: metadata.TgChatId || null,
tgBotToken: metadata.TgBotToken || null,
tgChatId: null,
tgBotToken: null,
isChunked: metadata.IsChunked || false
};
};
+47 -36
View File
@@ -2,7 +2,7 @@ import { getUploadConfig } from '../../api/manage/sysConfig/upload.js';
import { normalizeWebDAVHeaders } from '../storage/webdavAPI.js';
export async function resolveS3Credentials(db, env, metadata = {}) {
const channel = await findChannel(db, env, 's3', metadata.ChannelName);
const channel = await findChannel(db, env, 's3', metadata);
if (channel) {
return {
source: 'config',
@@ -18,20 +18,20 @@ export async function resolveS3Credentials(db, env, metadata = {}) {
}
return {
source: 'metadata',
endpoint: metadata.S3Endpoint,
region: metadata.S3Region || 'auto',
bucketName: metadata.S3BucketName,
pathStyle: metadata.S3PathStyle || false,
accessKeyId: metadata.S3AccessKeyId,
secretAccessKey: metadata.S3SecretAccessKey,
cdnDomain: metadata.S3CdnDomain || '',
source: 'missing',
endpoint: '',
region: 'auto',
bucketName: '',
pathStyle: false,
accessKeyId: '',
secretAccessKey: '',
cdnDomain: '',
key: metadata.S3FileKey,
};
}
export async function resolveTelegramCredentials(db, env, metadata = {}) {
const channel = await findChannel(db, env, 'telegram', metadata.ChannelName);
const channel = await findChannel(db, env, 'telegram', metadata);
if (channel) {
return {
source: 'config',
@@ -43,16 +43,16 @@ export async function resolveTelegramCredentials(db, env, metadata = {}) {
}
return {
source: 'metadata',
botToken: metadata.TgBotToken || env.TG_BOT_TOKEN,
chatId: metadata.TgChatId || env.TG_CHAT_ID,
proxyUrl: metadata.TgProxyUrl || '',
source: 'missing',
botToken: '',
chatId: '',
proxyUrl: '',
fileId: metadata.TgFileId,
};
}
export async function resolveDiscordCredentials(db, env, metadata = {}) {
const channel = await findChannel(db, env, 'discord', metadata.ChannelName);
const channel = await findChannel(db, env, 'discord', metadata);
if (channel) {
return {
source: 'config',
@@ -64,16 +64,16 @@ export async function resolveDiscordCredentials(db, env, metadata = {}) {
}
return {
source: 'metadata',
botToken: metadata.DiscordBotToken,
channelId: metadata.DiscordChannelId,
proxyUrl: metadata.DiscordProxyUrl || '',
source: 'missing',
botToken: '',
channelId: '',
proxyUrl: '',
messageId: metadata.DiscordMessageId,
};
}
export async function resolveHuggingFaceCredentials(db, env, metadata = {}) {
const channel = await findChannel(db, env, 'huggingface', metadata.ChannelName);
const channel = await findChannel(db, env, 'huggingface', metadata);
if (channel) {
return {
source: 'config',
@@ -86,17 +86,17 @@ export async function resolveHuggingFaceCredentials(db, env, metadata = {}) {
}
return {
source: 'metadata',
token: metadata.HfToken,
repo: metadata.HfRepo,
isPrivate: metadata.HfIsPrivate || false,
source: 'missing',
token: '',
repo: '',
isPrivate: false,
filePath: metadata.HfFilePath,
fileUrl: metadata.HfFileUrl,
fileUrl: '',
};
}
export async function resolveWebDAVCredentials(db, env, metadata = {}) {
const channel = await findChannel(db, env, 'webdav', metadata.ChannelName);
const channel = await findChannel(db, env, 'webdav', metadata);
if (channel) {
return normalizeWebDAVCredentials({
source: 'config',
@@ -112,19 +112,20 @@ export async function resolveWebDAVCredentials(db, env, metadata = {}) {
}
return normalizeWebDAVCredentials({
source: 'metadata',
baseUrl: metadata.WebDAVBaseUrl,
username: metadata.WebDAVUsername || '',
password: metadata.WebDAVPassword || '',
headers: metadata.WebDAVHeaders || {},
createDirectory: metadata.WebDAVCreateDirectory !== false,
publicUrl: metadata.WebDAVPublicBaseUrl || '',
source: 'missing',
baseUrl: '',
username: '',
password: '',
headers: {},
createDirectory: true,
publicUrl: '',
filePath: metadata.WebDAVFilePath,
publicFileUrl: metadata.WebDAVPublicUrl,
publicFileUrl: '',
});
}
async function findChannel(db, env, groupName, channelName) {
async function findChannel(db, env, groupName, metadata = {}) {
const channelName = getEffectiveChannelName(groupName, metadata);
if (!channelName) return null;
try {
@@ -137,9 +138,19 @@ async function findChannel(db, env, groupName, channelName) {
}
}
function getEffectiveChannelName(groupName, metadata = {}) {
if (metadata.ChannelName) return metadata.ChannelName;
if (groupName === 'telegram' && (metadata.Channel === 'Telegram' || metadata.Channel === 'TelegramNew')) {
return 'Telegram_env';
}
return '';
}
function normalizeWebDAVCredentials(config = {}) {
return {
source: config.source || 'metadata',
source: config.source || 'missing',
baseUrl: getWebDAVBaseUrl(config),
username: config.username || '',
password: config.password || '',
@@ -87,6 +87,16 @@ export function stripConfigDerivedMetadataInPlace(metadata = {}) {
return metadata;
}
export function cleanPersistedMetadata(metadata = {}) {
return stripConfigDerivedMetadata(stripSensitiveMetadata(metadata));
}
export function cleanPersistedMetadataInPlace(metadata = {}) {
stripSensitiveMetadataInPlace(metadata);
stripConfigDerivedMetadataInPlace(metadata);
return metadata;
}
function stripUrlUserinfo(value) {
try {
const url = new URL(value);
+24 -43
View File
@@ -1,5 +1,5 @@
import { getUploadConfig } from '../../api/manage/sysConfig/upload.js';
import { sanitizeFileMetadata } from './metadataSecurity.js';
import { sanitizeFileMetadata, stripConfigDerivedMetadata } from './metadataSecurity.js';
import { buildWebDAVUrl } from '../storage/webdavAPI.js';
export async function createMetadataViewContext(db, env) {
@@ -12,7 +12,7 @@ export async function createMetadataViewContext(db, env) {
export async function buildFileMetadataForManagement(db, env, metadata = {}, viewContext = null) {
const context = viewContext || await createMetadataViewContext(db, env);
const view = sanitizeFileMetadata(metadata);
const view = stripConfigDerivedMetadata(sanitizeFileMetadata(metadata));
enrichS3Metadata(context, metadata, view);
enrichHuggingFaceMetadata(context, metadata, view);
@@ -33,23 +33,16 @@ function enrichS3Metadata(context, sourceMetadata, view) {
try {
const channel = findChannel(context, 's3', sourceMetadata.ChannelName);
const credentials = channel
? {
endpoint: channel.endpoint,
region: channel.region || 'auto',
bucketName: channel.bucketName,
pathStyle: channel.pathStyle || false,
cdnDomain: channel.cdnDomain || '',
key: sourceMetadata.S3FileKey,
}
: {
endpoint: sourceMetadata.S3Endpoint,
region: sourceMetadata.S3Region || 'auto',
bucketName: sourceMetadata.S3BucketName,
pathStyle: sourceMetadata.S3PathStyle || false,
cdnDomain: sourceMetadata.S3CdnDomain || '',
key: sourceMetadata.S3FileKey,
};
if (!channel) return;
const credentials = {
endpoint: channel.endpoint,
region: channel.region || 'auto',
bucketName: channel.bucketName,
pathStyle: channel.pathStyle || false,
cdnDomain: channel.cdnDomain || '',
key: sourceMetadata.S3FileKey,
};
const key = credentials.key || sourceMetadata.S3FileKey;
if (!key) return;
@@ -58,9 +51,6 @@ function enrichS3Metadata(context, sourceMetadata, view) {
view.S3Location = buildS3Location(credentials, key);
}
if (channel) {
delete view.S3CdnFileUrl;
}
if (credentials.cdnDomain) {
view.S3CdnFileUrl = buildCdnFileUrl(credentials.cdnDomain, key);
}
@@ -74,15 +64,12 @@ function enrichHuggingFaceMetadata(context, sourceMetadata, view) {
try {
const channel = findChannel(context, 'huggingface', sourceMetadata.ChannelName);
const credentials = channel
? {
repo: channel.repo,
filePath: sourceMetadata.HfFilePath,
}
: {
repo: sourceMetadata.HfRepo,
filePath: sourceMetadata.HfFilePath,
};
if (!channel) return;
const credentials = {
repo: channel.repo,
filePath: sourceMetadata.HfFilePath,
};
if (credentials.repo && credentials.filePath) {
view.HfFileUrl = `https://huggingface.co/datasets/${credentials.repo}/resolve/main/${credentials.filePath}`;
}
@@ -96,19 +83,13 @@ function enrichWebDAVMetadata(context, sourceMetadata, view) {
try {
const channel = findChannel(context, 'webdav', sourceMetadata.ChannelName);
const credentials = channel
? {
publicUrl: channel.publicUrl || '',
filePath: sourceMetadata.WebDAVFilePath,
}
: {
publicUrl: sourceMetadata.WebDAVPublicBaseUrl || '',
filePath: sourceMetadata.WebDAVFilePath,
};
if (!channel) return;
const credentials = {
publicUrl: channel.publicUrl || '',
filePath: sourceMetadata.WebDAVFilePath,
};
const filePath = credentials.filePath || sourceMetadata.WebDAVFilePath;
if (channel) {
delete view.WebDAVPublicUrl;
}
if (credentials.publicUrl && filePath) {
view.WebDAVPublicUrl = buildWebDAVUrl(credentials.publicUrl, filePath);
}