mirror of
https://github.com/ZSCGR/CloudFlare-ImgBed.git
synced 2026-08-18 06:33:43 +08:00
2200 lines
71 KiB
JavaScript
2200 lines
71 KiB
JavaScript
/* 索引管理器 */
|
||
|
||
/**
|
||
* 文件索引结构（分块存储）：
|
||
*
|
||
* 索引元数据：
|
||
* - key: manage@index@meta
|
||
* - value: JSON.stringify(metadata)
|
||
* - metadata: {
|
||
* lastUpdated: 1640995200000,
|
||
* totalCount: 1000,
|
||
* lastOperationId: "operation_timestamp_uuid",
|
||
* chunkCount: 3,
|
||
* chunkSize: 10000
|
||
* }
|
||
*
|
||
* 索引分块：
|
||
* - key: manage@index_${chunkId} (例如: manage@index_0, manage@index_1, ...)
|
||
* - value: JSON.stringify(filesChunk)
|
||
* - filesChunk: [
|
||
* {
|
||
* id: "file_unique_id",
|
||
* metadata: {}
|
||
* },
|
||
* ...
|
||
* ]
|
||
*
|
||
* 原子操作结构（保持不变）：
|
||
* - key: manage@index@operation_${timestamp}_${uuid}
|
||
* - value: JSON.stringify(operation)
|
||
* - operation: {
|
||
* type: "add" | "remove" | "move" | "batch_add" | "batch_remove" | "batch_move",
|
||
* timestamp: 1640995200000,
|
||
* data: {
|
||
* // 根据操作类型包含不同的数据
|
||
* }
|
||
* }
|
||
*/
|
||
|
||
import { getDatabase, checkDatabaseConfig } from './databaseAdapter.js';
|
||
|
||
const INDEX_KEY = 'manage@index';
|
||
const INDEX_META_KEY = 'manage@index@meta'; // 索引元数据键
|
||
const OPERATION_KEY_PREFIX = 'manage@index@operation_';
|
||
// D1 单字段限制 2MB，KV 限制 25MB，根据数据库类型动态设置
|
||
const INDEX_CHUNK_SIZE_D1 = 500; // D1 数据库分块大小
|
||
const INDEX_CHUNK_SIZE_KV = 5000; // KV 存储分块大小
|
||
const KV_LIST_LIMIT = 1000; // 数据库列出批量大小
|
||
const BATCH_SIZE = 10; // 批量处理大小
|
||
|
||
/**
|
||
* 根据数据库类型获取索引分块大小
|
||
* @param {Object} env - 环境变量
|
||
* @returns {number} 分块大小
|
||
*/
|
||
export function getIndexChunkSize(env) {
|
||
const config = checkDatabaseConfig(env);
|
||
return config.usingD1 ? INDEX_CHUNK_SIZE_D1 : INDEX_CHUNK_SIZE_KV;
|
||
}
|
||
|
||
/**
|
||
* 添加文件到索引
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @param {string} fileId - 文件 ID
|
||
* @param {Object} metadata - 文件元数据
|
||
*/
|
||
export async function addFileToIndex(context, fileId, metadata = null) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
if (metadata === null) {
|
||
// 如果未传入metadata，尝试从数据库中获取
|
||
const fileData = await db.getWithMetadata(fileId);
|
||
metadata = fileData.metadata || {};
|
||
}
|
||
|
||
// 记录原子操作
|
||
const operationId = await recordOperation(context, 'add', {
|
||
fileId,
|
||
metadata
|
||
});
|
||
|
||
console.log(`File ${fileId} add operation recorded with ID: ${operationId}`);
|
||
return { success: true, operationId };
|
||
} catch (error) {
|
||
console.error('Error recording add file operation:', error);
|
||
return { success: false, error: error.message };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 批量添加文件到索引
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @param {Array} files - 文件数组，每个元素包含 { fileId, metadata }
|
||
* @param {Object} options - 选项
|
||
* @param {boolean} options.skipExisting - 是否跳过已存在的文件，默认为 false（更新已存在的文件）
|
||
* @returns {Object} 返回操作结果 { operationId, totalProcessed }
|
||
*/
|
||
export async function batchAddFilesToIndex(context, files, options = {}) {
|
||
try {
|
||
const { env } = context;
|
||
const { skipExisting = false } = options;
|
||
const db = getDatabase(env);
|
||
|
||
// 处理每个文件的metadata
|
||
const processedFiles = [];
|
||
for (const fileItem of files) {
|
||
const { fileId, metadata } = fileItem;
|
||
let finalMetadata = metadata;
|
||
|
||
// 如果没有提供metadata，尝试从数据库中获取
|
||
if (!finalMetadata) {
|
||
try {
|
||
const fileData = await db.getWithMetadata(fileId);
|
||
finalMetadata = fileData.metadata || {};
|
||
} catch (error) {
|
||
console.warn(`Failed to get metadata for file ${fileId}:`, error);
|
||
finalMetadata = {};
|
||
}
|
||
}
|
||
|
||
processedFiles.push({
|
||
fileId,
|
||
metadata: finalMetadata
|
||
});
|
||
}
|
||
|
||
// 记录批量添加操作
|
||
const operationId = await recordOperation(context, 'batch_add', {
|
||
files: processedFiles,
|
||
options: { skipExisting }
|
||
});
|
||
|
||
console.log(`Batch add operation recorded with ID: ${operationId}, ${files.length} files`);
|
||
return {
|
||
success: true,
|
||
operationId,
|
||
totalProcessed: files.length
|
||
};
|
||
} catch (error) {
|
||
console.error('Error recording batch add files operation:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message,
|
||
totalProcessed: 0
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 从索引中删除文件
|
||
* @param {Object} context - 上下文对象
|
||
* @param {string} fileId - 文件 ID
|
||
*/
|
||
export async function removeFileFromIndex(context, fileId) {
|
||
try {
|
||
// 记录删除操作
|
||
const operationId = await recordOperation(context, 'remove', {
|
||
fileId
|
||
});
|
||
|
||
console.log(`File ${fileId} remove operation recorded with ID: ${operationId}`);
|
||
return { success: true, operationId };
|
||
} catch (error) {
|
||
console.error('Error recording remove file operation:', error);
|
||
return { success: false, error: error.message };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 批量删除文件
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Array} fileIds - 文件 ID 数组
|
||
*/
|
||
export async function batchRemoveFilesFromIndex(context, fileIds) {
|
||
try {
|
||
// 记录批量删除操作
|
||
const operationId = await recordOperation(context, 'batch_remove', {
|
||
fileIds
|
||
});
|
||
|
||
console.log(`Batch remove operation recorded with ID: ${operationId}, ${fileIds.length} files`);
|
||
return {
|
||
success: true,
|
||
operationId,
|
||
totalProcessed: fileIds.length
|
||
};
|
||
} catch (error) {
|
||
console.error('Error recording batch remove files operation:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message,
|
||
totalProcessed: 0
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 移动文件（修改文件ID）
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @param {string} originalFileId - 原文件 ID
|
||
* @param {string} newFileId - 新文件 ID
|
||
* @param {Object} newMetadata - 新的元数据，如果为null则获取原文件的metadata
|
||
* @returns {Object} 返回操作结果 { success, operationId?, error? }
|
||
*/
|
||
export async function moveFileInIndex(context, originalFileId, newFileId, newMetadata = null) {
|
||
try {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
// 确定最终的metadata
|
||
let finalMetadata = newMetadata;
|
||
if (finalMetadata === null) {
|
||
// 如果没有提供新metadata，尝试从数据库中获取
|
||
try {
|
||
const fileData = await db.getWithMetadata(newFileId);
|
||
finalMetadata = fileData.metadata || {};
|
||
} catch (error) {
|
||
console.warn(`Failed to get metadata for new file ${newFileId}:`, error);
|
||
finalMetadata = {};
|
||
}
|
||
}
|
||
|
||
// 记录移动操作
|
||
const operationId = await recordOperation(context, 'move', {
|
||
originalFileId,
|
||
newFileId,
|
||
metadata: finalMetadata
|
||
});
|
||
|
||
console.log(`File move operation from ${originalFileId} to ${newFileId} recorded with ID: ${operationId}`);
|
||
return { success: true, operationId };
|
||
} catch (error) {
|
||
console.error('Error recording move file operation:', error);
|
||
return { success: false, error: error.message };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 批量移动文件
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @param {Array} moveOperations - 移动操作数组，每个元素包含 { originalFileId, newFileId, metadata? }
|
||
* @returns {Object} 返回操作结果 { operationId, totalProcessed }
|
||
*/
|
||
export async function batchMoveFilesInIndex(context, moveOperations) {
|
||
try {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
// 处理每个移动操作的metadata
|
||
const processedOperations = [];
|
||
for (const operation of moveOperations) {
|
||
const { originalFileId, newFileId, metadata } = operation;
|
||
|
||
// 确定最终的metadata
|
||
let finalMetadata = metadata;
|
||
if (finalMetadata === null || finalMetadata === undefined) {
|
||
// 如果没有提供新metadata，尝试从数据库中获取
|
||
try {
|
||
const fileData = await db.getWithMetadata(newFileId);
|
||
finalMetadata = fileData.metadata || {};
|
||
} catch (error) {
|
||
console.warn(`Failed to get metadata for new file ${newFileId}:`, error);
|
||
finalMetadata = {};
|
||
}
|
||
}
|
||
|
||
processedOperations.push({
|
||
originalFileId,
|
||
newFileId,
|
||
metadata: finalMetadata
|
||
});
|
||
}
|
||
|
||
// 记录批量移动操作
|
||
const operationId = await recordOperation(context, 'batch_move', {
|
||
operations: processedOperations
|
||
});
|
||
|
||
console.log(`Batch move operation recorded with ID: ${operationId}, ${moveOperations.length} operations`);
|
||
return {
|
||
success: true,
|
||
operationId,
|
||
totalProcessed: moveOperations.length
|
||
};
|
||
} catch (error) {
|
||
console.error('Error recording batch move files operation:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message,
|
||
totalProcessed: 0
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 合并所有挂起的操作到索引中
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Object} options - 选项
|
||
* @param {boolean} options.cleanupAfterMerge - 合并后是否清理操作记录，默认为 true
|
||
* @returns {Object} 合并结果
|
||
*/
|
||
export async function mergeOperationsToIndex(context, options = {}) {
|
||
const { request } = context;
|
||
const { cleanupAfterMerge = true } = options;
|
||
|
||
try {
|
||
console.log('Starting operations merge...');
|
||
|
||
// 获取当前索引
|
||
const currentIndex = await getIndex(context);
|
||
if (currentIndex.success === false) {
|
||
console.error('Failed to get current index for merge');
|
||
return {
|
||
success: false,
|
||
error: 'Failed to get current index'
|
||
};
|
||
}
|
||
|
||
// 获取所有待处理的操作
|
||
const operationsResult = await getAllPendingOperations(context, currentIndex.lastOperationId);
|
||
|
||
const operations = operationsResult.operations;
|
||
const isALLOperations = operationsResult.isAll;
|
||
|
||
if (operations.length === 0) {
|
||
console.log('No pending operations to merge');
|
||
return {
|
||
success: true,
|
||
processedOperations: 0,
|
||
message: 'No pending operations'
|
||
};
|
||
}
|
||
|
||
console.log(`Found ${operations.length} pending operations to merge. Is all operations: ${isALLOperations}, if there are remaining operations they will be processed in the next merge.`);
|
||
|
||
// 按时间戳排序操作，确保按正确顺序应用
|
||
operations.sort((a, b) => a.timestamp - b.timestamp);
|
||
|
||
// 创建索引的副本进行操作
|
||
const workingIndex = currentIndex;
|
||
let operationsProcessed = 0;
|
||
let addedCount = 0;
|
||
let removedCount = 0;
|
||
let movedCount = 0;
|
||
let updatedCount = 0;
|
||
const processedOperationIds = [];
|
||
|
||
// 应用每个操作
|
||
for (const operation of operations) {
|
||
try {
|
||
switch (operation.type) {
|
||
case 'add':
|
||
const addResult = applyAddOperation(workingIndex, operation.data);
|
||
if (addResult.added) addedCount++;
|
||
if (addResult.updated) updatedCount++;
|
||
break;
|
||
|
||
case 'remove':
|
||
if (applyRemoveOperation(workingIndex, operation.data)) {
|
||
removedCount++;
|
||
}
|
||
break;
|
||
|
||
case 'move':
|
||
if (applyMoveOperation(workingIndex, operation.data)) {
|
||
movedCount++;
|
||
}
|
||
break;
|
||
|
||
case 'batch_add':
|
||
const batchAddResult = applyBatchAddOperation(workingIndex, operation.data);
|
||
addedCount += batchAddResult.addedCount;
|
||
updatedCount += batchAddResult.updatedCount;
|
||
break;
|
||
|
||
case 'batch_remove':
|
||
removedCount += applyBatchRemoveOperation(workingIndex, operation.data);
|
||
break;
|
||
|
||
case 'batch_move':
|
||
movedCount += applyBatchMoveOperation(workingIndex, operation.data);
|
||
break;
|
||
|
||
default:
|
||
console.warn(`Unknown operation type: ${operation.type}`);
|
||
continue;
|
||
}
|
||
|
||
operationsProcessed++;
|
||
processedOperationIds.push(operation.id);
|
||
|
||
// 增加协作点
|
||
if (operationsProcessed % 3 === 0) {
|
||
await new Promise(resolve => setTimeout(resolve, 0));
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error(`Error applying operation ${operation.id}:`, error);
|
||
}
|
||
}
|
||
|
||
// 如果有任何修改，保存索引
|
||
if (operationsProcessed > 0) {
|
||
workingIndex.lastUpdated = Date.now();
|
||
workingIndex.totalCount = workingIndex.files.length;
|
||
|
||
// 记录最后处理的操作ID
|
||
if (processedOperationIds.length > 0) {
|
||
workingIndex.lastOperationId = processedOperationIds[processedOperationIds.length - 1];
|
||
}
|
||
|
||
// 保存更新后的索引（使用分块格式）
|
||
const saveSuccess = await saveChunkedIndex(context, workingIndex);
|
||
if (!saveSuccess) {
|
||
console.error('Failed to save chunked index');
|
||
return {
|
||
success: false,
|
||
error: 'Failed to save index'
|
||
};
|
||
}
|
||
|
||
console.log(`Index updated: ${addedCount} added, ${updatedCount} updated, ${removedCount} removed, ${movedCount} moved`);
|
||
}
|
||
|
||
// 清理已处理的操作记录
|
||
if (cleanupAfterMerge && processedOperationIds.length > 0) {
|
||
await cleanupOperations(context, processedOperationIds);
|
||
}
|
||
|
||
// 如果未处理完所有操作，调用 merge-operations API 递归处理
|
||
if (!isALLOperations) {
|
||
console.log('There are remaining operations, will process them in subsequent calls.');
|
||
|
||
const headers = new Headers(request.headers);
|
||
const originUrl = new URL(request.url);
|
||
const mergeUrl = `${originUrl.protocol}//${originUrl.host}/api/manage/list?action=merge-operations`;
|
||
|
||
await fetch(mergeUrl, { method: 'GET', headers });
|
||
|
||
return {
|
||
success: false,
|
||
error: 'There are remaining operations, will process them in subsequent calls.'
|
||
};
|
||
}
|
||
|
||
const result = {
|
||
success: true,
|
||
processedOperations: operationsProcessed,
|
||
addedCount,
|
||
updatedCount,
|
||
removedCount,
|
||
movedCount,
|
||
totalFiles: workingIndex.totalCount
|
||
};
|
||
|
||
console.log('Operations merge completed:', result);
|
||
return result;
|
||
|
||
} catch (error) {
|
||
console.error('Error merging operations:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 读取文件索引，支持搜索和分页
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Object} options - 查询选项
|
||
* @param {string} options.search - 搜索关键字
|
||
* @param {string} options.directory - 目录过滤
|
||
* @param {number} options.start - 起始位置
|
||
* @param {number} options.count - 返回数量，-1 表示返回所有
|
||
* @param {Array<string>|string} options.channel - 渠道过滤（支持数组多选）
|
||
* @param {Array<string>|string} options.listType - 列表类型过滤（支持数组多选）
|
||
* @param {Array<string>|string} options.accessStatus - 访问状态筛选（支持数组多选）：'normal'=正常, 'blocked'=已屏蔽
|
||
* @param {Array<string>|string} options.label - 审查结果筛选（支持数组多选）
|
||
* @param {Array<string>|string} options.fileType - 文件类型筛选（支持数组多选）
|
||
* @param {Array<string>|string} options.channelName - 渠道名称筛选（支持数组多选）
|
||
* @param {Array<string>} options.includeTags - 必须包含的标签数组
|
||
* @param {Array<string>} options.excludeTags - 必须排除的标签数组
|
||
* @param {boolean} options.countOnly - 仅返回总数
|
||
* @param {boolean} options.includeSubdirFiles - 是否包含子目录下的文件
|
||
*/
|
||
export async function readIndex(context, options = {}) {
|
||
try {
|
||
const {
|
||
search = '',
|
||
directory = '',
|
||
start = 0,
|
||
count = 50,
|
||
channel = [],
|
||
listType = [],
|
||
accessStatus = [],
|
||
label = [],
|
||
fileType = [],
|
||
channelName = [],
|
||
includeTags = [],
|
||
excludeTags = [],
|
||
countOnly = false,
|
||
includeSubdirFiles = false
|
||
} = options;
|
||
|
||
// 将参数统一转换为数组形式
|
||
const channelArr = Array.isArray(channel) ? channel : (channel ? [channel] : []);
|
||
const listTypeArr = Array.isArray(listType) ? listType : (listType ? [listType] : []);
|
||
const accessStatusArr = Array.isArray(accessStatus) ? accessStatus : (accessStatus ? [accessStatus] : []);
|
||
const labelArr = Array.isArray(label) ? label : (label ? [label] : []);
|
||
const fileTypeArr = Array.isArray(fileType) ? fileType : (fileType ? [fileType] : []);
|
||
const channelNameArr = Array.isArray(channelName) ? channelName : (channelName ? [channelName] : []);
|
||
|
||
// 处理目录满足无头有尾的格式，根目录为空
|
||
const dirPrefix = directory === '' || directory.endsWith('/') ? directory : directory + '/';
|
||
|
||
// 处理挂起的操作
|
||
const mergeResult = await mergeOperationsToIndex(context);
|
||
if (!mergeResult.success) {
|
||
throw new Error('Failed to merge operations: ' + mergeResult.error);
|
||
}
|
||
|
||
// 获取当前索引
|
||
const index = await getIndex(context);
|
||
if (!index.success) {
|
||
throw new Error('Failed to get index');
|
||
}
|
||
|
||
let filteredFiles = index.files;
|
||
|
||
// 目录过滤
|
||
if (directory) {
|
||
const normalizedDir = directory.endsWith('/') ? directory : directory + '/';
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileDir = file.metadata.Directory ? file.metadata.Directory : extractDirectory(file.id);
|
||
return fileDir.startsWith(normalizedDir) || file.metadata.Directory === directory;
|
||
});
|
||
}
|
||
|
||
// 渠道过滤（支持多选，OR 逻辑）
|
||
if (channelArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file =>
|
||
channelArr.some(ch => file.metadata.Channel?.toLowerCase() === ch.toLowerCase())
|
||
);
|
||
}
|
||
|
||
// 列表类型过滤（黑白名单，支持多选，OR 逻辑）
|
||
// White=白名单, Block=黑名单, None=未设置
|
||
if (listTypeArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileListType = file.metadata.ListType;
|
||
return listTypeArr.some(lt => {
|
||
if (lt === 'None') {
|
||
// 未设置：ListType 为空、undefined、null 或字符串 'None'
|
||
return !fileListType || fileListType === '' || fileListType === 'None';
|
||
}
|
||
return fileListType === lt;
|
||
});
|
||
});
|
||
}
|
||
|
||
// 访问状态筛选（综合判断 ListType 和 Label，支持多选，OR 逻辑）
|
||
// 'normal' = 正常：非已屏蔽状态
|
||
// 'blocked' = 已屏蔽：ListType === 'Block' || (Label === 'adult' && ListType !== 'White')
|
||
// 注意：白名单优先，即使 Label 是 adult，只要 ListType 是 White 就是正常
|
||
if (accessStatusArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileListType = file.metadata.ListType;
|
||
const fileLabel = file.metadata.Label;
|
||
const isBlocked = fileListType === 'Block' || (fileLabel === 'adult' && fileListType !== 'White');
|
||
|
||
return accessStatusArr.some(status => {
|
||
if (status === 'normal') {
|
||
return !isBlocked;
|
||
} else if (status === 'blocked') {
|
||
return isBlocked;
|
||
}
|
||
return false;
|
||
});
|
||
});
|
||
}
|
||
|
||
// 审查结果筛选 (label)（支持多选，OR 逻辑）
|
||
// 'normal' 匹配 Label 为 'everyone', 'None', '', null, undefined
|
||
// 'teen' 匹配 Label 为 'teen'
|
||
// 'adult' 匹配 Label 为 'adult'
|
||
if (labelArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileLabel = file.metadata.Label;
|
||
return labelArr.some(lbl => {
|
||
if (lbl === 'normal') {
|
||
return !fileLabel || fileLabel === '' || fileLabel === 'None' || fileLabel === 'everyone';
|
||
} else if (lbl === 'teen') {
|
||
return fileLabel === 'teen';
|
||
} else if (lbl === 'adult') {
|
||
return fileLabel === 'adult';
|
||
}
|
||
return false;
|
||
});
|
||
});
|
||
}
|
||
|
||
// 文件类型筛选 (fileType)（支持多选，OR 逻辑）
|
||
// 'image' 匹配 FileType 以 'image/' 开头
|
||
// 'video' 匹配 FileType 以 'video/' 开头
|
||
// 'audio' 匹配 FileType 以 'audio/' 开头
|
||
// 'other' 匹配不属于以上三类的文件
|
||
if (fileTypeArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const mimeType = file.metadata.FileType || '';
|
||
return fileTypeArr.some(ft => {
|
||
if (ft === 'image') {
|
||
return mimeType.startsWith('image/');
|
||
} else if (ft === 'video') {
|
||
return mimeType.startsWith('video/');
|
||
} else if (ft === 'audio') {
|
||
return mimeType.startsWith('audio/');
|
||
} else if (ft === 'other') {
|
||
return !mimeType.startsWith('image/') &&
|
||
!mimeType.startsWith('video/') &&
|
||
!mimeType.startsWith('audio/');
|
||
}
|
||
return false;
|
||
});
|
||
});
|
||
}
|
||
|
||
// 渠道名称筛选 (channelName)（支持多选，OR 逻辑）
|
||
// 支持 "type:name" 格式（如 "TelegramNew:default"）或单独的名称
|
||
if (channelNameArr.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileChannel = file.metadata.Channel;
|
||
const fileChannelName = file.metadata.ChannelName;
|
||
|
||
return channelNameArr.some(filterValue => {
|
||
// 检查是否是 "type:name" 格式
|
||
if (filterValue.includes(':')) {
|
||
const [type, name] = filterValue.split(':', 2);
|
||
// 同时匹配渠道类型和名称（大小写敏感）
|
||
return fileChannel === type && fileChannelName === name;
|
||
} else {
|
||
// 只匹配名称（向后兼容）
|
||
return fileChannelName === filterValue;
|
||
}
|
||
});
|
||
});
|
||
}
|
||
|
||
// 标签过滤（独立于搜索关键字）
|
||
if (includeTags.length > 0 || excludeTags.length > 0) {
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const fileTags = (file.metadata.Tags || []).map(t => t.toLowerCase());
|
||
|
||
// 检查必须包含的标签
|
||
if (includeTags.length > 0) {
|
||
const hasAllIncludeTags = includeTags.every(tag =>
|
||
fileTags.includes(tag.toLowerCase())
|
||
);
|
||
if (!hasAllIncludeTags) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// 检查必须排除的标签
|
||
if (excludeTags.length > 0) {
|
||
const hasAnyExcludeTag = excludeTags.some(tag =>
|
||
fileTags.includes(tag.toLowerCase())
|
||
);
|
||
if (hasAnyExcludeTag) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
return true;
|
||
});
|
||
}
|
||
|
||
// 搜索过滤（仅关键字）
|
||
if (search) {
|
||
const searchLower = search.toLowerCase();
|
||
filteredFiles = filteredFiles.filter(file => {
|
||
const matchesKeyword =
|
||
file.metadata.FileName?.toLowerCase().includes(searchLower) ||
|
||
file.id.toLowerCase().includes(searchLower);
|
||
return matchesKeyword;
|
||
});
|
||
}
|
||
|
||
// 如果只需要总数
|
||
if (countOnly) {
|
||
return {
|
||
totalCount: filteredFiles.length,
|
||
indexLastUpdated: index.lastUpdated
|
||
};
|
||
}
|
||
|
||
// 分页处理
|
||
const totalCount = filteredFiles.length;
|
||
|
||
let resultFiles = filteredFiles;
|
||
|
||
// 计算当前目录下的直接文件（不包含子目录文件）
|
||
const directFiles = filteredFiles.filter(file => {
|
||
const fileDir = file.metadata.Directory ? file.metadata.Directory : extractDirectory(file.id);
|
||
return fileDir === dirPrefix;
|
||
});
|
||
const directFileCount = directFiles.length;
|
||
|
||
// 如果不包含子目录文件，获取当前目录下的直接文件
|
||
if (!includeSubdirFiles) {
|
||
resultFiles = directFiles;
|
||
}
|
||
|
||
if (count !== -1) {
|
||
const startIndex = Math.max(0, start);
|
||
const endIndex = startIndex + Math.max(1, count);
|
||
resultFiles = resultFiles.slice(startIndex, endIndex);
|
||
}
|
||
|
||
// 提取目录信息
|
||
const directories = new Set();
|
||
filteredFiles.forEach(file => {
|
||
const fileDir = file.metadata.Directory ? file.metadata.Directory : extractDirectory(file.id);
|
||
if (fileDir && fileDir.startsWith(dirPrefix)) {
|
||
const relativePath = fileDir.substring(dirPrefix.length);
|
||
const firstSlashIndex = relativePath.indexOf('/');
|
||
if (firstSlashIndex !== -1) {
|
||
const subDir = dirPrefix + relativePath.substring(0, firstSlashIndex);
|
||
directories.add(subDir);
|
||
}
|
||
}
|
||
});
|
||
|
||
// 直接子文件夹数目
|
||
const directFolderCount = directories.size;
|
||
|
||
return {
|
||
files: resultFiles,
|
||
directories: Array.from(directories),
|
||
totalCount: totalCount,
|
||
directFileCount: directFileCount,
|
||
directFolderCount: directFolderCount,
|
||
indexLastUpdated: index.lastUpdated,
|
||
returnedCount: resultFiles.length,
|
||
success: true
|
||
};
|
||
|
||
} catch (error) {
|
||
console.error('Error reading index:', error);
|
||
return {
|
||
files: [],
|
||
directories: [],
|
||
totalCount: 0,
|
||
indexLastUpdated: Date.now(),
|
||
returnedCount: 0,
|
||
success: false,
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 重建索引（从数据库中的所有文件重新构建索引）
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Function} progressCallback - 进度回调函数
|
||
*/
|
||
export async function rebuildIndex(context, progressCallback = null) {
|
||
const { env, waitUntil } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
console.log('Starting index rebuild...');
|
||
|
||
let cursor = null;
|
||
let processedCount = 0;
|
||
const newIndex = {
|
||
files: [],
|
||
lastUpdated: Date.now(),
|
||
totalCount: 0,
|
||
lastOperationId: null
|
||
};
|
||
|
||
// 分批读取所有文件
|
||
while (true) {
|
||
const response = await db.list({
|
||
limit: KV_LIST_LIMIT,
|
||
cursor: cursor
|
||
});
|
||
|
||
cursor = response.cursor;
|
||
|
||
for (const item of response.keys) {
|
||
// 跳过管理相关的键
|
||
if (item.name.startsWith('manage@') || item.name.startsWith('chunk_')) {
|
||
continue;
|
||
}
|
||
|
||
// 跳过没有元数据的文件
|
||
if (!item.metadata || !item.metadata.TimeStamp) {
|
||
continue;
|
||
}
|
||
|
||
// 构建文件索引项
|
||
const fileItem = {
|
||
id: item.name,
|
||
metadata: item.metadata || {}
|
||
};
|
||
|
||
newIndex.files.push(fileItem);
|
||
processedCount++;
|
||
|
||
// 报告进度
|
||
if (progressCallback && processedCount % 100 === 0) {
|
||
progressCallback(processedCount);
|
||
}
|
||
}
|
||
|
||
if (!cursor) break;
|
||
|
||
// 添加协作点
|
||
await new Promise(resolve => setTimeout(resolve, 10));
|
||
}
|
||
|
||
// 按时间戳倒序排序
|
||
newIndex.files.sort((a, b) => b.metadata.TimeStamp - a.metadata.TimeStamp);
|
||
|
||
newIndex.totalCount = newIndex.files.length;
|
||
|
||
// 保存新索引（使用分块格式）
|
||
const saveSuccess = await saveChunkedIndex(context, newIndex);
|
||
if (!saveSuccess) {
|
||
console.error('Failed to save chunked index during rebuild');
|
||
return {
|
||
success: false,
|
||
error: 'Failed to save rebuilt index'
|
||
};
|
||
}
|
||
|
||
// 清除旧的操作记录和多余索引
|
||
waitUntil(deleteAllOperations(context));
|
||
waitUntil(clearChunkedIndex(context, true));
|
||
|
||
|
||
console.log(`Index rebuild completed. Processed ${processedCount} files, indexed ${newIndex.totalCount} files.`);
|
||
return {
|
||
success: true,
|
||
processedCount,
|
||
indexedCount: newIndex.totalCount
|
||
};
|
||
|
||
} catch (error) {
|
||
console.error('Error rebuilding index:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取索引信息
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Object} options - 统计选项
|
||
*/
|
||
export async function getIndexInfo(context, options = {}) {
|
||
try {
|
||
const index = await getIndex(context);
|
||
|
||
// 检查索引是否成功获取
|
||
if (index.success === false) {
|
||
return {
|
||
success: false,
|
||
error: 'Failed to retrieve index',
|
||
message: 'Index is not available or corrupted'
|
||
}
|
||
}
|
||
|
||
// 统计各渠道文件数量
|
||
const channelStats = Object.create(null);
|
||
const directoryStats = Object.create(null);
|
||
const typeStats = Object.create(null);
|
||
const uploadTrend = createUploadTrendAccumulator(index.files, options);
|
||
|
||
index.files.forEach(file => {
|
||
const metadata = file.metadata || {};
|
||
|
||
// 渠道统计
|
||
const channel = normalizeChannel(metadata.Channel);
|
||
incrementStat(channelStats, channel);
|
||
|
||
// 目录统计
|
||
const dir = metadata.Directory || extractDirectory(file.id) || '/';
|
||
incrementStat(directoryStats, dir);
|
||
|
||
// 类型统计
|
||
let listType = metadata.ListType || 'None';
|
||
const label = metadata.Label || 'None';
|
||
if (listType !== 'White' && label === 'adult') {
|
||
listType = 'Block';
|
||
}
|
||
incrementStat(typeStats, listType);
|
||
|
||
addUploadTrendPoint(uploadTrend, metadata, channel);
|
||
});
|
||
|
||
return {
|
||
success: true,
|
||
totalFiles: index.totalCount,
|
||
lastUpdated: index.lastUpdated,
|
||
channelStats,
|
||
directoryStats,
|
||
typeStats,
|
||
uploadTrend: finalizeUploadTrend(uploadTrend),
|
||
oldestFile: index.files[index.files.length - 1],
|
||
newestFile: index.files[0]
|
||
};
|
||
} catch (error) {
|
||
console.error('Error getting index info:', error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||
const DEFAULT_TREND_MAX_POINTS = 90;
|
||
const MAX_TREND_POINTS = 366;
|
||
const DEFAULT_TREND_SERIES_LIMIT = 8;
|
||
const MAX_TREND_SERIES_LIMIT = 20;
|
||
|
||
function incrementStat(stats, key) {
|
||
const normalizedKey = normalizeTrendKey(key);
|
||
stats[normalizedKey] = (stats[normalizedKey] || 0) + 1;
|
||
}
|
||
|
||
function normalizeChannel(channel) {
|
||
if (channel === 'TelegramNew') {
|
||
return 'Telegram';
|
||
}
|
||
return normalizeTrendKey(channel, 'Telegraph');
|
||
}
|
||
|
||
function normalizeTrendKey(value, fallback = 'Unknown') {
|
||
if (typeof value === 'string') {
|
||
const trimmed = value.trim();
|
||
return trimmed || fallback;
|
||
}
|
||
if (value === null || value === undefined) {
|
||
return fallback;
|
||
}
|
||
return String(value);
|
||
}
|
||
|
||
function normalizeInteger(value, defaultValue, min, max) {
|
||
const parsed = Number.parseInt(value, 10);
|
||
if (!Number.isFinite(parsed)) {
|
||
return defaultValue;
|
||
}
|
||
return Math.min(max, Math.max(min, parsed));
|
||
}
|
||
|
||
function getLocalDayNumber(timestamp, timezoneOffset) {
|
||
return Math.floor((timestamp - timezoneOffset * 60 * 1000) / DAY_MS);
|
||
}
|
||
|
||
function getValidTimestamp(metadata) {
|
||
const timestamp = Number(metadata?.TimeStamp);
|
||
return Number.isFinite(timestamp) && timestamp > 0 ? timestamp : null;
|
||
}
|
||
|
||
function formatTrendDay(dayNumber) {
|
||
return new Date(dayNumber * DAY_MS).toISOString().slice(0, 10);
|
||
}
|
||
|
||
function parseTrendDate(value) {
|
||
if (typeof value !== 'string') {
|
||
return null;
|
||
}
|
||
|
||
const match = value.trim().match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||
if (!match) {
|
||
return null;
|
||
}
|
||
|
||
const year = Number.parseInt(match[1], 10);
|
||
const month = Number.parseInt(match[2], 10);
|
||
const day = Number.parseInt(match[3], 10);
|
||
const date = new Date(Date.UTC(year, month - 1, day));
|
||
|
||
if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) {
|
||
return null;
|
||
}
|
||
|
||
return Math.floor(date.getTime() / DAY_MS);
|
||
}
|
||
|
||
function getTrendDayRange(files, timezoneOffset, options = {}) {
|
||
let newestDay = null;
|
||
let oldestDay = null;
|
||
const optionStartDay = parseTrendDate(options.startDate);
|
||
const optionEndDay = parseTrendDate(options.endDate);
|
||
|
||
// The index is maintained in timestamp-desc order, so this avoids sorting.
|
||
for (let i = 0; i < files.length; i++) {
|
||
const timestamp = getValidTimestamp(files[i].metadata);
|
||
if (timestamp !== null) {
|
||
newestDay = getLocalDayNumber(timestamp, timezoneOffset);
|
||
break;
|
||
}
|
||
}
|
||
|
||
for (let i = files.length - 1; i >= 0; i--) {
|
||
const timestamp = getValidTimestamp(files[i].metadata);
|
||
if (timestamp !== null) {
|
||
oldestDay = getLocalDayNumber(timestamp, timezoneOffset);
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (optionStartDay !== null || optionEndDay !== null) {
|
||
const startDay = optionStartDay !== null ? optionStartDay : (oldestDay !== null ? oldestDay : optionEndDay);
|
||
const endDay = optionEndDay !== null ? optionEndDay : (newestDay !== null ? newestDay : optionStartDay);
|
||
return startDay <= endDay
|
||
? { startDay, endDay }
|
||
: { startDay: endDay, endDay: startDay };
|
||
}
|
||
|
||
if (newestDay === null || oldestDay === null) {
|
||
return null;
|
||
}
|
||
|
||
if (oldestDay > newestDay) {
|
||
return { startDay: newestDay, endDay: oldestDay };
|
||
}
|
||
|
||
return { startDay: oldestDay, endDay: newestDay };
|
||
}
|
||
|
||
function buildTrendBucketLabels(startDay, endDay, bucketSizeDays, bucketCount) {
|
||
const labels = [];
|
||
for (let i = 0; i < bucketCount; i++) {
|
||
const bucketStartDay = startDay + i * bucketSizeDays;
|
||
const bucketEndDay = Math.min(bucketStartDay + bucketSizeDays - 1, endDay);
|
||
labels.push(bucketStartDay === bucketEndDay
|
||
? formatTrendDay(bucketStartDay)
|
||
: `${formatTrendDay(bucketStartDay)} - ${formatTrendDay(bucketEndDay)}`);
|
||
}
|
||
return labels;
|
||
}
|
||
|
||
function createUploadTrendAccumulator(files, options) {
|
||
const timezoneOffset = normalizeInteger(options.timezoneOffset, 0, -14 * 60, 14 * 60);
|
||
const maxPoints = normalizeInteger(options.maxPoints, DEFAULT_TREND_MAX_POINTS, 7, MAX_TREND_POINTS);
|
||
const seriesLimit = normalizeInteger(options.seriesLimit, DEFAULT_TREND_SERIES_LIMIT, 1, MAX_TREND_SERIES_LIMIT);
|
||
const range = getTrendDayRange(files, timezoneOffset, options);
|
||
|
||
if (!range) {
|
||
return {
|
||
enabled: false,
|
||
timezoneOffset,
|
||
maxPoints,
|
||
seriesLimit
|
||
};
|
||
}
|
||
|
||
const spanDays = range.endDay - range.startDay + 1;
|
||
const bucketSizeDays = Math.max(1, Math.ceil(spanDays / maxPoints));
|
||
const bucketCount = Math.ceil(spanDays / bucketSizeDays);
|
||
|
||
return {
|
||
enabled: true,
|
||
timezoneOffset,
|
||
maxPoints,
|
||
seriesLimit,
|
||
startDay: range.startDay,
|
||
endDay: range.endDay,
|
||
bucketSizeDays,
|
||
bucketCount,
|
||
labels: buildTrendBucketLabels(range.startDay, range.endDay, bucketSizeDays, bucketCount),
|
||
total: Array(bucketCount).fill(0),
|
||
channelGroups: new Map(),
|
||
channelNameGroups: new Map()
|
||
};
|
||
}
|
||
|
||
function addUploadTrendPoint(accumulator, metadata, channel) {
|
||
if (!accumulator.enabled) {
|
||
return;
|
||
}
|
||
|
||
const timestamp = getValidTimestamp(metadata);
|
||
if (timestamp === null) {
|
||
return;
|
||
}
|
||
|
||
const dayNumber = getLocalDayNumber(timestamp, accumulator.timezoneOffset);
|
||
const bucketIndex = Math.floor((dayNumber - accumulator.startDay) / accumulator.bucketSizeDays);
|
||
if (bucketIndex < 0 || bucketIndex >= accumulator.bucketCount) {
|
||
return;
|
||
}
|
||
|
||
const channelName = normalizeTrendKey(metadata.ChannelName, channel);
|
||
|
||
accumulator.total[bucketIndex] += 1;
|
||
addTrendGroupPoint(accumulator.channelGroups, channel, bucketIndex);
|
||
addTrendGroupPoint(accumulator.channelNameGroups, channelName, bucketIndex);
|
||
}
|
||
|
||
function addTrendGroupPoint(groups, key, bucketIndex) {
|
||
const normalizedKey = normalizeTrendKey(key);
|
||
let entry = groups.get(normalizedKey);
|
||
if (!entry) {
|
||
entry = {
|
||
total: 0,
|
||
buckets: new Map()
|
||
};
|
||
groups.set(normalizedKey, entry);
|
||
}
|
||
|
||
entry.total += 1;
|
||
entry.buckets.set(bucketIndex, (entry.buckets.get(bucketIndex) || 0) + 1);
|
||
}
|
||
|
||
function buildTrendSeries(groups, bucketCount, seriesLimit) {
|
||
const selectedEntries = selectTopTrendEntries(groups, seriesLimit);
|
||
const selectedNames = new Set(selectedEntries.map(([name]) => name));
|
||
const series = selectedEntries.map(([name, entry]) => ({
|
||
name,
|
||
total: entry.total,
|
||
data: buildTrendSeriesData(entry, bucketCount)
|
||
}));
|
||
|
||
if (groups.size > selectedEntries.length) {
|
||
const otherData = Array(bucketCount).fill(0);
|
||
let otherTotal = 0;
|
||
|
||
for (const [name, entry] of groups.entries()) {
|
||
if (selectedNames.has(name)) {
|
||
continue;
|
||
}
|
||
otherTotal += entry.total;
|
||
entry.buckets.forEach((count, bucketIndex) => {
|
||
otherData[bucketIndex] += count;
|
||
});
|
||
}
|
||
|
||
series.push({
|
||
name: '__other__',
|
||
isOther: true,
|
||
total: otherTotal,
|
||
data: otherData
|
||
});
|
||
}
|
||
|
||
return {
|
||
series,
|
||
totalSeries: groups.size,
|
||
limited: groups.size > selectedEntries.length
|
||
};
|
||
}
|
||
|
||
function selectTopTrendEntries(groups, seriesLimit) {
|
||
const topEntries = [];
|
||
|
||
for (const entry of groups.entries()) {
|
||
insertTopTrendEntry(topEntries, entry, seriesLimit);
|
||
}
|
||
|
||
return topEntries;
|
||
}
|
||
|
||
function insertTopTrendEntry(topEntries, candidate, seriesLimit) {
|
||
let insertIndex = -1;
|
||
|
||
for (let i = 0; i < topEntries.length; i++) {
|
||
if (compareTrendEntry(candidate, topEntries[i]) < 0) {
|
||
insertIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (insertIndex === -1) {
|
||
if (topEntries.length < seriesLimit) {
|
||
topEntries.push(candidate);
|
||
}
|
||
return;
|
||
}
|
||
|
||
topEntries.splice(insertIndex, 0, candidate);
|
||
if (topEntries.length > seriesLimit) {
|
||
topEntries.pop();
|
||
}
|
||
}
|
||
|
||
function compareTrendEntry(left, right) {
|
||
if (right[1].total !== left[1].total) {
|
||
return right[1].total - left[1].total;
|
||
}
|
||
return left[0].localeCompare(right[0]);
|
||
}
|
||
|
||
function buildTrendSeriesData(entry, bucketCount) {
|
||
const data = Array(bucketCount).fill(0);
|
||
entry.buckets.forEach((count, bucketIndex) => {
|
||
data[bucketIndex] = count;
|
||
});
|
||
return data;
|
||
}
|
||
|
||
function finalizeUploadTrend(accumulator) {
|
||
const emptyGroup = {
|
||
series: [],
|
||
totalSeries: 0,
|
||
limited: false
|
||
};
|
||
|
||
if (!accumulator.enabled) {
|
||
return {
|
||
labels: [],
|
||
total: [],
|
||
bucketSizeDays: 1,
|
||
maxPoints: accumulator.maxPoints,
|
||
seriesLimit: accumulator.seriesLimit,
|
||
range: null,
|
||
groupBy: {
|
||
channel: emptyGroup,
|
||
channelName: emptyGroup
|
||
}
|
||
};
|
||
}
|
||
|
||
return {
|
||
labels: accumulator.labels,
|
||
total: accumulator.total,
|
||
bucketSizeDays: accumulator.bucketSizeDays,
|
||
maxPoints: accumulator.maxPoints,
|
||
seriesLimit: accumulator.seriesLimit,
|
||
range: {
|
||
startDate: formatTrendDay(accumulator.startDay),
|
||
endDate: formatTrendDay(accumulator.endDay),
|
||
timezoneOffset: accumulator.timezoneOffset
|
||
},
|
||
groupBy: {
|
||
channel: buildTrendSeries(accumulator.channelGroups, accumulator.bucketCount, accumulator.seriesLimit),
|
||
channelName: buildTrendSeries(accumulator.channelNameGroups, accumulator.bucketCount, accumulator.seriesLimit)
|
||
}
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取索引元数据（轻量级，只读取 meta，不读取整个索引）
|
||
* 用于容量检查等场景，避免读取整个索引
|
||
* @param {Object} context - 上下文对象
|
||
* @returns {Object} 索引元数据，包含 totalCount, totalSizeMB, channelStats 等
|
||
*/
|
||
export async function getIndexMeta(context) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
const metadataStr = await db.get(INDEX_META_KEY);
|
||
if (!metadataStr) {
|
||
return {
|
||
success: false,
|
||
totalCount: 0,
|
||
totalSizeMB: 0,
|
||
channelStats: {}
|
||
};
|
||
}
|
||
|
||
const metadata = JSON.parse(metadataStr);
|
||
return {
|
||
success: true,
|
||
totalCount: metadata.totalCount || 0,
|
||
totalSizeMB: metadata.totalSizeMB || 0,
|
||
channelStats: metadata.channelStats || {},
|
||
lastUpdated: metadata.lastUpdated
|
||
};
|
||
} catch (error) {
|
||
console.error('Error getting index meta:', error);
|
||
return {
|
||
success: false,
|
||
totalCount: 0,
|
||
totalSizeMB: 0,
|
||
channelStats: {}
|
||
};
|
||
}
|
||
}
|
||
|
||
/* ============= 原子操作相关函数 ============= */
|
||
|
||
/**
|
||
* 生成唯一的操作ID
|
||
*/
|
||
function generateOperationId() {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).substring(2, 9);
|
||
return `${timestamp}_${random}`;
|
||
}
|
||
|
||
/**
|
||
* 记录原子操作
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @param {string} type - 操作类型
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
async function recordOperation(context, type, data) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
const operationId = generateOperationId();
|
||
const operation = {
|
||
type,
|
||
timestamp: Date.now(),
|
||
data
|
||
};
|
||
|
||
const operationKey = OPERATION_KEY_PREFIX + operationId;
|
||
await db.put(operationKey, JSON.stringify(operation));
|
||
|
||
return operationId;
|
||
}
|
||
|
||
/**
|
||
* 获取所有待处理的操作
|
||
* @param {Object} context - 上下文对象
|
||
* @param {string} lastOperationId - 最后处理的操作ID
|
||
*/
|
||
async function getAllPendingOperations(context, lastOperationId = null) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
const operations = [];
|
||
|
||
let cursor = null;
|
||
const MAX_OPERATION_COUNT = 30; // 单次获取的最大操作数量
|
||
let isALL = true; // 是否获取了所有操作
|
||
let operationCount = 0;
|
||
|
||
try {
|
||
while (true) {
|
||
const response = await db.list({
|
||
prefix: OPERATION_KEY_PREFIX,
|
||
limit: KV_LIST_LIMIT,
|
||
cursor: cursor
|
||
});
|
||
|
||
for (const item of response.keys) {
|
||
// 如果指定了lastOperationId，跳过已处理的操作
|
||
if (lastOperationId && item.name <= OPERATION_KEY_PREFIX + lastOperationId) {
|
||
continue;
|
||
}
|
||
|
||
if (operationCount >= MAX_OPERATION_COUNT) {
|
||
isALL = false; // 达到最大操作数量，停止获取
|
||
break;
|
||
}
|
||
|
||
try {
|
||
const operationData = await db.get(item.name);
|
||
if (operationData) {
|
||
const operation = JSON.parse(operationData);
|
||
operation.id = item.name.substring(OPERATION_KEY_PREFIX.length);
|
||
operations.push(operation);
|
||
operationCount++;
|
||
}
|
||
} catch (error) {
|
||
isALL = false;
|
||
console.warn(`Failed to parse operation ${item.name}:`, error);
|
||
}
|
||
}
|
||
|
||
cursor = response.cursor;
|
||
if (!cursor || operationCount >= MAX_OPERATION_COUNT) break;
|
||
}
|
||
} catch (error) {
|
||
console.error('Error getting pending operations:', error);
|
||
}
|
||
|
||
return {
|
||
operations,
|
||
isAll: isALL,
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 应用添加操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyAddOperation(index, data) {
|
||
const { fileId, metadata } = data;
|
||
|
||
// 检查文件是否已存在
|
||
const existingIndex = index.files.findIndex(file => file.id === fileId);
|
||
|
||
const fileItem = {
|
||
id: fileId,
|
||
metadata: metadata || {}
|
||
};
|
||
|
||
if (existingIndex !== -1) {
|
||
// 更新现有文件
|
||
index.files[existingIndex] = fileItem;
|
||
return { added: false, updated: true };
|
||
} else {
|
||
// 添加新文件
|
||
insertFileInOrder(index.files, fileItem);
|
||
return { added: true, updated: false };
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 应用删除操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyRemoveOperation(index, data) {
|
||
const { fileId } = data;
|
||
const initialLength = index.files.length;
|
||
index.files = index.files.filter(file => file.id !== fileId);
|
||
return index.files.length < initialLength;
|
||
}
|
||
|
||
/**
|
||
* 应用移动操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyMoveOperation(index, data) {
|
||
const { originalFileId, newFileId, metadata } = data;
|
||
|
||
const originalIndex = index.files.findIndex(file => file.id === originalFileId);
|
||
if (originalIndex === -1) {
|
||
return false; // 原文件不存在
|
||
}
|
||
|
||
// 更新文件ID和元数据
|
||
index.files[originalIndex] = {
|
||
id: newFileId,
|
||
metadata: metadata || index.files[originalIndex].metadata
|
||
};
|
||
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* 应用批量添加操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyBatchAddOperation(index, data) {
|
||
const { files, options } = data;
|
||
const { skipExisting = false } = options || {};
|
||
|
||
let addedCount = 0;
|
||
let updatedCount = 0;
|
||
|
||
// 创建现有文件ID的映射以提高查找效率
|
||
const existingFilesMap = new Map();
|
||
index.files.forEach((file, idx) => {
|
||
existingFilesMap.set(file.id, idx);
|
||
});
|
||
|
||
for (const fileData of files) {
|
||
const { fileId, metadata } = fileData;
|
||
const fileItem = {
|
||
id: fileId,
|
||
metadata: metadata || {}
|
||
};
|
||
|
||
const existingIndex = existingFilesMap.get(fileId);
|
||
|
||
if (existingIndex !== undefined) {
|
||
if (!skipExisting) {
|
||
// 更新现有文件
|
||
index.files[existingIndex] = fileItem;
|
||
updatedCount++;
|
||
}
|
||
} else {
|
||
// 添加新文件
|
||
insertFileInOrder(index.files, fileItem);
|
||
// 更新映射
|
||
index.files.forEach((file, idx) => {
|
||
existingFilesMap.set(file.id, idx);
|
||
});
|
||
|
||
addedCount++;
|
||
}
|
||
}
|
||
|
||
return { addedCount, updatedCount };
|
||
}
|
||
|
||
/**
|
||
* 应用批量删除操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyBatchRemoveOperation(index, data) {
|
||
const { fileIds } = data;
|
||
const fileIdSet = new Set(fileIds);
|
||
const initialLength = index.files.length;
|
||
|
||
index.files = index.files.filter(file => !fileIdSet.has(file.id));
|
||
|
||
return initialLength - index.files.length;
|
||
}
|
||
|
||
/**
|
||
* 应用批量移动操作
|
||
* @param {Object} index - 索引对象
|
||
* @param {Object} data - 操作数据
|
||
*/
|
||
function applyBatchMoveOperation(index, data) {
|
||
const { operations } = data;
|
||
let movedCount = 0;
|
||
|
||
// 创建现有文件ID的映射以提高查找效率
|
||
const existingFilesMap = new Map();
|
||
index.files.forEach((file, idx) => {
|
||
existingFilesMap.set(file.id, idx);
|
||
});
|
||
|
||
for (const operation of operations) {
|
||
const { originalFileId, newFileId, metadata } = operation;
|
||
|
||
const originalIndex = existingFilesMap.get(originalFileId);
|
||
if (originalIndex !== undefined) {
|
||
// 更新映射
|
||
existingFilesMap.delete(originalFileId);
|
||
existingFilesMap.set(newFileId, originalIndex);
|
||
|
||
// 更新文件信息
|
||
index.files[originalIndex] = {
|
||
id: newFileId,
|
||
metadata: metadata || index.files[originalIndex].metadata
|
||
};
|
||
|
||
movedCount++;
|
||
}
|
||
}
|
||
|
||
return movedCount;
|
||
}
|
||
|
||
/**
|
||
* 并发清理指定的原子操作记录
|
||
* @param {Object} context - 上下文对象
|
||
* @param {Array} operationIds - 要清理的操作ID数组
|
||
* @param {number} concurrency - 并发数量，默认为10
|
||
*/
|
||
async function cleanupOperations(context, operationIds, concurrency = 10) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
console.log(`Cleaning up ${operationIds.length} processed operations with concurrency ${concurrency}...`);
|
||
|
||
let deletedCount = 0;
|
||
let errorCount = 0;
|
||
|
||
// 创建删除任务数组
|
||
const deleteTasks = operationIds.map(operationId => {
|
||
const operationKey = OPERATION_KEY_PREFIX + operationId;
|
||
return async () => {
|
||
try {
|
||
await db.delete(operationKey);
|
||
deletedCount++;
|
||
} catch (error) {
|
||
console.error(`Error deleting operation ${operationId}:`, error);
|
||
errorCount++;
|
||
}
|
||
};
|
||
});
|
||
|
||
// 使用并发控制执行删除操作
|
||
await promiseLimit(deleteTasks, concurrency);
|
||
|
||
console.log(`Successfully cleaned up ${deletedCount} operations, ${errorCount} operations failed.`);
|
||
return {
|
||
success: true,
|
||
deletedCount: deletedCount,
|
||
errorCount: errorCount,
|
||
};
|
||
|
||
} catch (error) {
|
||
console.error('Error cleaning up operations:', error);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 删除所有原子操作记录
|
||
* @param {Object} context - 上下文对象，包含 env 和其他信息
|
||
* @returns {Object} 删除结果 { success, deletedCount, errors?, totalFound? }
|
||
*/
|
||
export async function deleteAllOperations(context) {
|
||
const { request, env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
console.log('Starting to delete all atomic operations...');
|
||
|
||
// 获取所有原子操作
|
||
const allOperationIds = [];
|
||
let cursor = null;
|
||
let totalFound = 0;
|
||
|
||
// 首先收集所有操作键
|
||
while (true) {
|
||
const response = await db.list({
|
||
prefix: OPERATION_KEY_PREFIX,
|
||
limit: KV_LIST_LIMIT,
|
||
cursor: cursor
|
||
});
|
||
|
||
for (const item of response.keys) {
|
||
allOperationIds.push(item.name.substring(OPERATION_KEY_PREFIX.length));
|
||
totalFound++;
|
||
}
|
||
|
||
cursor = response.cursor;
|
||
if (!cursor) break;
|
||
}
|
||
|
||
if (totalFound === 0) {
|
||
console.log('No atomic operations found to delete');
|
||
return {
|
||
success: true,
|
||
deletedCount: 0,
|
||
totalFound: 0,
|
||
message: 'No operations to delete'
|
||
};
|
||
}
|
||
|
||
console.log(`Found ${totalFound} atomic operations to delete`);
|
||
|
||
// 限制单次删除的数量
|
||
const MAX_DELETE_BATCH = 40;
|
||
const toDeleteOperationIds = allOperationIds.slice(0, MAX_DELETE_BATCH);
|
||
|
||
// 批量删除原子操作
|
||
const cleanupResult = await cleanupOperations(context, toDeleteOperationIds);
|
||
|
||
// 剩余未删除的操作，调用 delete-operations API 进行递归删除
|
||
if (allOperationIds.length > MAX_DELETE_BATCH || cleanupResult.errorCount > 0) {
|
||
console.warn(`Too many operations (${allOperationIds.length}), only deleting first ${cleanupResult.deletedCount}. The remaining operations will be deleted in subsequent calls.`);
|
||
// 复制请求头，用于鉴权
|
||
const headers = new Headers(request.headers);
|
||
|
||
const originUrl = new URL(request.url);
|
||
const deleteUrl = `${originUrl.protocol}//${originUrl.host}/api/manage/list?action=delete-operations`
|
||
|
||
await fetch(deleteUrl, {
|
||
method: 'GET',
|
||
headers: headers
|
||
});
|
||
|
||
} else {
|
||
console.log(`Delete all operations completed`);
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('Error deleting all operations:', error);
|
||
}
|
||
}
|
||
|
||
/* ============= 工具函数 ============= */
|
||
|
||
/**
|
||
* 获取索引（内部函数）
|
||
* @param {Object} context - 上下文对象
|
||
*/
|
||
async function getIndex(context) {
|
||
const { waitUntil } = context;
|
||
try {
|
||
// 首先尝试加载分块索引
|
||
const index = await loadChunkedIndex(context);
|
||
if (index.success) {
|
||
return index;
|
||
} else {
|
||
// 如果加载失败，触发重建索引
|
||
waitUntil(rebuildIndex(context));
|
||
}
|
||
} catch (error) {
|
||
console.warn('Error reading index, creating new one:', error);
|
||
waitUntil(rebuildIndex(context));
|
||
}
|
||
|
||
// 返回空的索引结构
|
||
return {
|
||
files: [],
|
||
lastUpdated: Date.now(),
|
||
totalCount: 0,
|
||
lastOperationId: null,
|
||
success: false,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 从文件路径提取目录（内部函数）
|
||
* @param {string} filePath - 文件路径
|
||
*/
|
||
function extractDirectory(filePath) {
|
||
const lastSlashIndex = filePath.lastIndexOf('/');
|
||
if (lastSlashIndex === -1) {
|
||
return ''; // 根目录
|
||
}
|
||
return filePath.substring(0, lastSlashIndex + 1); // 包含最后的斜杠
|
||
}
|
||
|
||
/**
|
||
* 将扁平目录路径列表转换为嵌套树结构
|
||
* @param {Array<string>} directories - 目录路径数组，如 ['photos/', 'photos/2024/', 'documents/']
|
||
* @returns {Object} 树形结构 { name, path, children }
|
||
*
|
||
* 示例输出：
|
||
* {
|
||
* name: "/",
|
||
* path: "",
|
||
* children: [
|
||
* {
|
||
* name: "photos",
|
||
* path: "photos/",
|
||
* children: [
|
||
* { name: "2024", path: "photos/2024/", children: [] }
|
||
* ]
|
||
* },
|
||
* { name: "documents", path: "documents/", children: [] }
|
||
* ]
|
||
* }
|
||
*/
|
||
function buildTree(directories) {
|
||
// 创建根节点
|
||
const root = {
|
||
name: "/",
|
||
path: "",
|
||
children: []
|
||
};
|
||
|
||
// 如果没有目录，返回仅包含根节点的空树
|
||
if (!directories || directories.length === 0) {
|
||
return root;
|
||
}
|
||
|
||
// 使用 Map 存储已创建的节点，key 为路径
|
||
const nodeMap = new Map();
|
||
nodeMap.set("", root);
|
||
|
||
// 对目录进行排序，确保父目录在子目录之前处理
|
||
const sortedDirs = [...directories].sort();
|
||
|
||
for (const dirPath of sortedDirs) {
|
||
// 跳过空路径（根目录已创建）
|
||
if (!dirPath) continue;
|
||
|
||
// 规范化路径：确保以 / 结尾
|
||
const normalizedPath = dirPath.endsWith('/') ? dirPath : dirPath + '/';
|
||
|
||
// 如果节点已存在，跳过
|
||
if (nodeMap.has(normalizedPath)) continue;
|
||
|
||
// 分割路径获取各级目录名
|
||
const parts = normalizedPath.split('/').filter(part => part !== '');
|
||
|
||
// 逐级创建节点
|
||
let currentPath = "";
|
||
let parentNode = root;
|
||
|
||
for (let i = 0; i < parts.length; i++) {
|
||
const part = parts[i];
|
||
currentPath = currentPath + part + '/';
|
||
|
||
// 检查当前路径的节点是否已存在
|
||
if (nodeMap.has(currentPath)) {
|
||
parentNode = nodeMap.get(currentPath);
|
||
} else {
|
||
// 创建新节点
|
||
const newNode = {
|
||
name: part,
|
||
path: currentPath,
|
||
children: []
|
||
};
|
||
|
||
// 添加到父节点的 children 中
|
||
parentNode.children.push(newNode);
|
||
|
||
// 存储到 Map 中
|
||
nodeMap.set(currentPath, newNode);
|
||
|
||
// 更新父节点引用
|
||
parentNode = newNode;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 对每个节点的 children 按名称排序
|
||
const sortChildren = (node) => {
|
||
node.children.sort((a, b) => a.name.localeCompare(b.name));
|
||
node.children.forEach(sortChildren);
|
||
};
|
||
sortChildren(root);
|
||
|
||
return root;
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 将文件按时间戳倒序插入到已排序的数组中
|
||
* @param {Array} sortedFiles - 已按时间戳倒序排序的文件数组
|
||
* @param {Object} fileItem - 要插入的文件项
|
||
*/
|
||
function insertFileInOrder(sortedFiles, fileItem) {
|
||
const fileTimestamp = fileItem.metadata.TimeStamp || 0;
|
||
|
||
// 如果数组为空或新文件时间戳比第一个文件更新，直接插入到开头
|
||
if (sortedFiles.length === 0 || fileTimestamp >= (sortedFiles[0].metadata.TimeStamp || 0)) {
|
||
sortedFiles.unshift(fileItem);
|
||
return;
|
||
}
|
||
|
||
// 如果新文件时间戳比最后一个文件更旧，直接添加到末尾
|
||
if (fileTimestamp <= (sortedFiles[sortedFiles.length - 1].metadata.TimeStamp || 0)) {
|
||
sortedFiles.push(fileItem);
|
||
return;
|
||
}
|
||
|
||
// 使用二分查找找到正确的插入位置
|
||
let left = 0;
|
||
let right = sortedFiles.length;
|
||
|
||
while (left < right) {
|
||
const mid = Math.floor((left + right) / 2);
|
||
const midTimestamp = sortedFiles[mid].metadata.TimeStamp || 0;
|
||
|
||
if (fileTimestamp >= midTimestamp) {
|
||
right = mid;
|
||
} else {
|
||
left = mid + 1;
|
||
}
|
||
}
|
||
|
||
// 在找到的位置插入文件
|
||
sortedFiles.splice(left, 0, fileItem);
|
||
}
|
||
|
||
/**
|
||
* 并发控制工具函数 - 限制同时执行的Promise数量
|
||
* @param {Array} tasks - 任务数组，每个任务是一个返回Promise的函数
|
||
* @param {number} concurrency - 并发数量
|
||
* @returns {Promise<Array>} 所有任务的结果数组
|
||
*/
|
||
async function promiseLimit(tasks, concurrency = BATCH_SIZE) {
|
||
const results = [];
|
||
const executing = [];
|
||
|
||
for (let i = 0; i < tasks.length; i++) {
|
||
const task = tasks[i];
|
||
const promise = Promise.resolve().then(() => task()).then(result => {
|
||
results[i] = result;
|
||
return result;
|
||
}).finally(() => {
|
||
const index = executing.indexOf(promise);
|
||
if (index >= 0) {
|
||
executing.splice(index, 1);
|
||
}
|
||
});
|
||
|
||
executing.push(promise);
|
||
|
||
if (executing.length >= concurrency) {
|
||
await Promise.race(executing);
|
||
}
|
||
}
|
||
|
||
// 等待所有剩余的Promise完成
|
||
await Promise.all(executing);
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* 保存分块索引到数据库
|
||
* @param {Object} context - 上下文对象，包含 env
|
||
* @param {Object} index - 完整的索引对象
|
||
* @returns {Promise<boolean>} 是否保存成功
|
||
*/
|
||
async function saveChunkedIndex(context, index) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
const chunkSize = getIndexChunkSize(env);
|
||
|
||
try {
|
||
const files = index.files || [];
|
||
const chunks = [];
|
||
|
||
// 将文件数组分块
|
||
for (let i = 0; i < files.length; i += chunkSize) {
|
||
const chunk = files.slice(i, i + chunkSize);
|
||
chunks.push(chunk);
|
||
}
|
||
|
||
// 计算各渠道容量统计
|
||
const channelStats = {};
|
||
let totalSizeMB = 0;
|
||
|
||
for (const file of files) {
|
||
const channelName = file.metadata?.ChannelName;
|
||
const fileSize = parseFloat(file.metadata?.FileSize) || 0;
|
||
|
||
totalSizeMB += fileSize;
|
||
|
||
if (channelName) {
|
||
if (!channelStats[channelName]) {
|
||
channelStats[channelName] = { usedMB: 0, fileCount: 0 };
|
||
}
|
||
channelStats[channelName].usedMB += fileSize;
|
||
channelStats[channelName].fileCount += 1;
|
||
}
|
||
}
|
||
|
||
// 保存索引元数据（包含容量统计）
|
||
const metadata = {
|
||
lastUpdated: index.lastUpdated,
|
||
totalCount: index.totalCount,
|
||
totalSizeMB: Math.round(totalSizeMB * 100) / 100,
|
||
channelStats,
|
||
lastOperationId: index.lastOperationId,
|
||
chunkCount: chunks.length,
|
||
chunkSize: chunkSize
|
||
};
|
||
|
||
await db.put(INDEX_META_KEY, JSON.stringify(metadata));
|
||
|
||
// 保存各个分块
|
||
const savePromises = chunks.map((chunk, chunkId) => {
|
||
const chunkKey = `${INDEX_KEY}_${chunkId}`;
|
||
return db.put(chunkKey, JSON.stringify(chunk));
|
||
});
|
||
|
||
await Promise.all(savePromises);
|
||
|
||
console.log(`Saved chunked index: ${chunks.length} chunks, ${files.length} total files, ${totalSizeMB.toFixed(2)} MB`);
|
||
return true;
|
||
|
||
} catch (error) {
|
||
console.error('Error saving chunked index:', error);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 从数据库加载分块索引
|
||
* @param {Object} context - 上下文对象，包含 env
|
||
* @returns {Promise<Object>} 完整的索引对象
|
||
*/
|
||
async function loadChunkedIndex(context) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
// 首先获取元数据
|
||
const metadataStr = await db.get(INDEX_META_KEY);
|
||
if (!metadataStr) {
|
||
throw new Error('Index metadata not found');
|
||
}
|
||
|
||
const metadata = JSON.parse(metadataStr);
|
||
const files = [];
|
||
|
||
// 并行加载所有分块
|
||
const loadPromises = [];
|
||
for (let chunkId = 0; chunkId < metadata.chunkCount; chunkId++) {
|
||
const chunkKey = `${INDEX_KEY}_${chunkId}`;
|
||
loadPromises.push(
|
||
db.get(chunkKey).then(chunkStr => {
|
||
if (chunkStr) {
|
||
return JSON.parse(chunkStr);
|
||
}
|
||
return [];
|
||
})
|
||
);
|
||
}
|
||
|
||
const chunks = await Promise.all(loadPromises);
|
||
|
||
// 合并所有分块
|
||
chunks.forEach(chunk => {
|
||
if (Array.isArray(chunk)) {
|
||
files.push(...chunk);
|
||
}
|
||
});
|
||
|
||
const index = {
|
||
files,
|
||
lastUpdated: metadata.lastUpdated,
|
||
totalCount: metadata.totalCount,
|
||
lastOperationId: metadata.lastOperationId,
|
||
success: true
|
||
};
|
||
|
||
console.log(`Loaded chunked index: ${metadata.chunkCount} chunks, ${files.length} total files`);
|
||
return index;
|
||
|
||
} catch (error) {
|
||
console.error('Error loading chunked index:', error);
|
||
// 返回空的索引结构
|
||
return {
|
||
files: [],
|
||
lastUpdated: Date.now(),
|
||
totalCount: 0,
|
||
lastOperationId: null,
|
||
success: false,
|
||
};
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 清理分块索引
|
||
* @param {Object} context - 上下文对象，包含 env
|
||
* @param {boolean} onlyNonUsed - 是否仅清理未使用的分块索引，默认为 false
|
||
* @returns {Promise<boolean>} 是否清理成功
|
||
*/
|
||
export async function clearChunkedIndex(context, onlyNonUsed = false) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
console.log('Starting chunked index cleanup...');
|
||
|
||
// 获取元数据
|
||
const metadataStr = await db.get(INDEX_META_KEY);
|
||
let chunkCount = 0;
|
||
|
||
if (metadataStr) {
|
||
const metadata = JSON.parse(metadataStr);
|
||
chunkCount = metadata.chunkCount || 0;
|
||
|
||
if (!onlyNonUsed) {
|
||
// 删除元数据
|
||
await db.delete(INDEX_META_KEY).catch(() => {});
|
||
}
|
||
}
|
||
|
||
// 删除分块
|
||
const recordedChunks = []; // 现有的索引分块键
|
||
let cursor = null;
|
||
while (true) {
|
||
const response = await db.list({
|
||
prefix: INDEX_KEY,
|
||
limit: KV_LIST_LIMIT,
|
||
cursor: cursor
|
||
});
|
||
|
||
for (const item of response.keys) {
|
||
recordedChunks.push(item.name);
|
||
}
|
||
|
||
cursor = response.cursor;
|
||
if (!cursor) break;
|
||
}
|
||
|
||
const reservedChunks = [];
|
||
if (onlyNonUsed) {
|
||
// 如果仅清理未使用的分块索引，保留当前在使用的分块
|
||
for (let chunkId = 0; chunkId < chunkCount; chunkId++) {
|
||
reservedChunks.push(`${INDEX_KEY}_${chunkId}`);
|
||
}
|
||
}
|
||
|
||
const deletePromises = [];
|
||
for (let chunkKey of recordedChunks) {
|
||
if (reservedChunks.includes(chunkKey) || !chunkKey.startsWith(INDEX_KEY + '_')) {
|
||
// 保留的分块和非分块键不删除
|
||
continue;
|
||
}
|
||
|
||
deletePromises.push(
|
||
db.delete(chunkKey).catch(() => {})
|
||
);
|
||
}
|
||
|
||
if (recordedChunks.includes(INDEX_KEY)) {
|
||
deletePromises.push(
|
||
db.delete(INDEX_KEY).catch(() => {})
|
||
);
|
||
}
|
||
|
||
await Promise.all(deletePromises);
|
||
|
||
console.log(`Chunked index cleanup completed. Attempted to delete ${chunkCount} chunks.`);
|
||
return true;
|
||
|
||
} catch (error) {
|
||
console.error('Error during chunked index cleanup:', error);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取索引的存储统计信息
|
||
* @param {Object} context - 上下文对象，包含 env
|
||
* @returns {Object} 存储统计信息
|
||
*/
|
||
export async function getIndexStorageStats(context) {
|
||
const { env } = context;
|
||
const db = getDatabase(env);
|
||
|
||
try {
|
||
// 获取元数据
|
||
const metadataStr = await db.get(INDEX_META_KEY);
|
||
if (!metadataStr) {
|
||
return {
|
||
success: false,
|
||
error: 'No chunked index metadata found',
|
||
isChunked: false
|
||
};
|
||
}
|
||
|
||
const metadata = JSON.parse(metadataStr);
|
||
|
||
// 检查各个分块的存在情况
|
||
const chunkChecks = [];
|
||
for (let chunkId = 0; chunkId < metadata.chunkCount; chunkId++) {
|
||
const chunkKey = `${INDEX_KEY}_${chunkId}`;
|
||
chunkChecks.push(
|
||
db.get(chunkKey).then(data => ({
|
||
chunkId,
|
||
exists: !!data,
|
||
size: data ? data.length : 0
|
||
}))
|
||
);
|
||
}
|
||
|
||
const chunkResults = await Promise.all(chunkChecks);
|
||
|
||
const stats = {
|
||
success: true,
|
||
isChunked: true,
|
||
metadata,
|
||
chunks: chunkResults,
|
||
totalChunks: metadata.chunkCount,
|
||
existingChunks: chunkResults.filter(c => c.exists).length,
|
||
totalSize: chunkResults.reduce((sum, c) => sum + c.size, 0)
|
||
};
|
||
|
||
return stats;
|
||
|
||
} catch (error) {
|
||
console.error('Error getting index storage stats:', error);
|
||
return {
|
||
success: false,
|
||
error: error.message,
|
||
isChunked: false
|
||
};
|
||
}
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 从索引中提取目录树结构
|
||
* @param {Object} context - 上下文对象
|
||
* @returns {Object} 树形结构 { name, path, children }
|
||
*/
|
||
export async function getDirectoryTree(context) {
|
||
// 1. 合并挂起操作
|
||
await mergeOperationsToIndex(context);
|
||
|
||
// 2. 获取索引
|
||
const index = await getIndex(context);
|
||
|
||
// 3. 提取所有目录路径
|
||
const directorySet = new Set();
|
||
|
||
if (index.files && index.files.length > 0) {
|
||
for (const file of index.files) {
|
||
// 获取文件的目录路径
|
||
const dirPath = file.metadata?.Directory || extractDirectory(file.id);
|
||
|
||
if (dirPath) {
|
||
// 规范化路径：确保以 / 结尾
|
||
const normalizedDir = dirPath.endsWith('/') ? dirPath : dirPath + '/';
|
||
|
||
// 将路径按 / 分割，逐级添加到目录集合中（确保父目录也被包含）
|
||
const parts = normalizedDir.split('/').filter(part => part !== '');
|
||
let currentPath = '';
|
||
|
||
for (const part of parts) {
|
||
currentPath = currentPath + part + '/';
|
||
directorySet.add(currentPath);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 4. 构建树形结构
|
||
const directories = Array.from(directorySet);
|
||
return buildTree(directories);
|
||
}
|