mirror of
https://github.com/KuekHaoYang/KVideo.git
synced 2026-08-19 18:53:42 +08:00
新增M3U8启发式广告检测算法并支持动态关键词配置
This commit is contained in:
Troray
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Heuristic Ad Detection Module
|
||||
*
|
||||
* Provides block-based analysis for detecting ads in M3U8 playlists
|
||||
* using filename pattern matching and other heuristics.
|
||||
*/
|
||||
|
||||
// Ad-related path keywords for scoring
|
||||
export const AD_PATH_KEYWORDS = [
|
||||
'advert', 'preroll', 'midroll', 'postroll',
|
||||
'dai', 'vast', 'ima', 'adjump', 'commercial', 'sponsor'
|
||||
];
|
||||
|
||||
/**
|
||||
* Represents a segment in the playlist
|
||||
*/
|
||||
interface Segment {
|
||||
url: string;
|
||||
duration: number;
|
||||
lineIndex: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a block of segments between DISCONTINUITY markers
|
||||
*/
|
||||
interface Block {
|
||||
segments: Segment[];
|
||||
startLineIndex: number;
|
||||
endLineIndex: number;
|
||||
hasCueTag: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern extracted from main content for comparison
|
||||
*/
|
||||
interface MainPattern {
|
||||
filenameRegex: RegExp | null;
|
||||
avgDuration: number;
|
||||
commonPrefix: string;
|
||||
pathPrefix: string; // Directory path prefix (e.g., "/20230907/73PWifvT/1392kb/hls/")
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse M3U8 content into blocks separated by DISCONTINUITY markers
|
||||
*/
|
||||
export function parseBlocks(lines: string[]): Block[] {
|
||||
const blocks: Block[] = [];
|
||||
let currentBlock: Block = {
|
||||
segments: [],
|
||||
startLineIndex: 0,
|
||||
endLineIndex: 0,
|
||||
hasCueTag: false
|
||||
};
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
|
||||
// Check for CUE tags
|
||||
if (line.startsWith('#EXT-X-CUE-OUT') || line.startsWith('#EXT-X-CUE-IN')) {
|
||||
currentBlock.hasCueTag = true;
|
||||
}
|
||||
|
||||
// DISCONTINUITY marks block boundary
|
||||
if (line === '#EXT-X-DISCONTINUITY') {
|
||||
if (currentBlock.segments.length > 0) {
|
||||
currentBlock.endLineIndex = i - 1;
|
||||
blocks.push(currentBlock);
|
||||
}
|
||||
currentBlock = {
|
||||
segments: [],
|
||||
startLineIndex: i + 1,
|
||||
endLineIndex: 0,
|
||||
hasCueTag: false
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse EXTINF and the following URL
|
||||
if (line.startsWith('#EXTINF:')) {
|
||||
const durationMatch = line.match(/#EXTINF:([\d.]+)/);
|
||||
const duration = durationMatch ? parseFloat(durationMatch[1]) : 0;
|
||||
|
||||
// Next line should be the URL
|
||||
if (i + 1 < lines.length) {
|
||||
const url = lines[i + 1].trim();
|
||||
if (url && !url.startsWith('#')) {
|
||||
currentBlock.segments.push({
|
||||
url,
|
||||
duration,
|
||||
lineIndex: i + 1
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Don't forget the last block
|
||||
if (currentBlock.segments.length > 0) {
|
||||
currentBlock.endLineIndex = lines.length - 1;
|
||||
blocks.push(currentBlock);
|
||||
}
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from URL (handles both relative and absolute URLs)
|
||||
*/
|
||||
function extractFilename(url: string): string {
|
||||
try {
|
||||
const path = url.includes('://') ? new URL(url).pathname : url;
|
||||
const parts = path.split('/');
|
||||
return parts[parts.length - 1] || '';
|
||||
} catch {
|
||||
return url.split('/').pop() || '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find common prefix among an array of strings
|
||||
*/
|
||||
function findCommonPrefix(strings: string[]): string {
|
||||
if (!strings || strings.length < 2) return '';
|
||||
|
||||
let prefix = '';
|
||||
const first = strings[0];
|
||||
|
||||
for (let i = 0; i < first.length; i++) {
|
||||
const char = first[i];
|
||||
if (strings.every(s => s[i] === char)) {
|
||||
prefix += char;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return prefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract path prefix (directory) from URL
|
||||
* e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
|
||||
*/
|
||||
function extractPathPrefix(url: string): string {
|
||||
try {
|
||||
const path = url.includes('://') ? new URL(url).pathname : url;
|
||||
const lastSlash = path.lastIndexOf('/');
|
||||
return lastSlash >= 0 ? path.substring(0, lastSlash + 1) : '';
|
||||
} catch {
|
||||
const lastSlash = url.lastIndexOf('/');
|
||||
return lastSlash >= 0 ? url.substring(0, lastSlash + 1) : '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn pattern from the largest block (assumed to be main content)
|
||||
*/
|
||||
export function learnMainPattern(blocks: Block[]): MainPattern {
|
||||
// Find the largest block by segment count (likely main content)
|
||||
const mainBlock = blocks.length > 0 ? blocks.reduce((largest, block) =>
|
||||
block.segments.length > largest.segments.length ? block : largest
|
||||
) : null;
|
||||
|
||||
if (!mainBlock || mainBlock.segments.length === 0) {
|
||||
return { filenameRegex: null, avgDuration: 0, commonPrefix: '', pathPrefix: '' };
|
||||
}
|
||||
|
||||
// Extract filenames
|
||||
const filenames = mainBlock.segments.map(s => extractFilename(s.url));
|
||||
|
||||
// Find common prefix
|
||||
const commonPrefix = findCommonPrefix(filenames);
|
||||
|
||||
// Calculate average duration
|
||||
const totalDuration = mainBlock.segments.reduce((sum, s) => sum + s.duration, 0);
|
||||
const avgDuration = totalDuration / mainBlock.segments.length;
|
||||
|
||||
// Try to build a regex pattern from the filenames
|
||||
// Common patterns: "0000001.ts", "seg-1.ts", "segment_001.ts"
|
||||
let filenameRegex: RegExp | null = null;
|
||||
if (commonPrefix.length >= 2) {
|
||||
// Escape special regex characters in prefix
|
||||
const escapedPrefix = commonPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
filenameRegex = new RegExp(`^${escapedPrefix}`);
|
||||
}
|
||||
|
||||
// Extract path prefix (directory path without filename)
|
||||
// e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
|
||||
const firstUrl = mainBlock.segments[0].url;
|
||||
const pathPrefix = extractPathPrefix(firstUrl);
|
||||
|
||||
return { filenameRegex, avgDuration, commonPrefix, pathPrefix };
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a block for ad likelihood based on heuristics
|
||||
* Returns a score where higher = more likely to be an ad
|
||||
*/
|
||||
export function scoreBlock(block: Block, mainPattern: MainPattern, extraKeywords: string[] = []): number {
|
||||
let score = 0;
|
||||
|
||||
// If block has CUE tag, it's definitely an ad
|
||||
if (block.hasCueTag) {
|
||||
return 10; // Max score
|
||||
}
|
||||
|
||||
// Check path keywords (Built-in + Custom)
|
||||
// We filter out very short custom keywords to avoid false positives in scoring
|
||||
const safeExtraKeywords = extraKeywords.filter(k => k.length > 2);
|
||||
const allKeywords = [...AD_PATH_KEYWORDS, ...safeExtraKeywords];
|
||||
|
||||
for (const segment of block.segments) {
|
||||
const urlLower = segment.url.toLowerCase();
|
||||
for (const keyword of allKeywords) {
|
||||
if (urlLower.includes(keyword.toLowerCase())) {
|
||||
score += 2.5;
|
||||
break; // Only count once per segment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check filename pattern mismatch
|
||||
if (mainPattern.filenameRegex) {
|
||||
const mismatchCount = block.segments.filter(s => {
|
||||
if (!mainPattern.filenameRegex) return false; // No pattern to compare against
|
||||
const filename = extractFilename(s.url);
|
||||
return !mainPattern.filenameRegex.test(filename);
|
||||
}).length;
|
||||
|
||||
if (mismatchCount === block.segments.length && block.segments.length > 0) {
|
||||
score += 1.5; // All filenames differ from main pattern
|
||||
}
|
||||
}
|
||||
|
||||
// **KEY FEATURE**: Check path prefix mismatch (e.g., different date/folder/bitrate)
|
||||
// This is the most reliable indicator for ads that come from different CDN paths
|
||||
if (mainPattern.pathPrefix && block.segments.length > 0) {
|
||||
const pathMismatchCount = block.segments.filter(s => {
|
||||
const segmentPathPrefix = extractPathPrefix(s.url);
|
||||
return segmentPathPrefix !== mainPattern.pathPrefix;
|
||||
}).length;
|
||||
|
||||
if (pathMismatchCount === block.segments.length) {
|
||||
// ALL segments have different path prefix - strong ad indicator
|
||||
score += 5.0;
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Threshold configuration
|
||||
*/
|
||||
export const THRESHOLDS = {
|
||||
HIGH: 5.0, // Definitely an ad
|
||||
LOW: 3.0 // Possibly an ad (for future "fuzzy" mode)
|
||||
};
|
||||
|
||||
/**
|
||||
* Determine if a block should be filtered based on its score
|
||||
*/
|
||||
export function shouldFilterBlock(score: number, threshold: number = THRESHOLDS.HIGH): boolean {
|
||||
return score >= threshold;
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
|
||||
/**
|
||||
* Utility functions for M3U8 playlist manipulation
|
||||
*/
|
||||
|
||||
import { parseBlocks, learnMainPattern, scoreBlock, shouldFilterBlock } from './m3u8-ad-detector';
|
||||
|
||||
/**
|
||||
* Filters ads from specific M3U8 content using multiple detection strategies:
|
||||
* 1. Keyword matching (configurable via env)
|
||||
* 2. CUE-OUT/CUE-IN standard tags
|
||||
* 3. Heuristic block analysis (filename patterns, ad path keywords)
|
||||
*
|
||||
* Also converts relative URLs to absolute URLs for Blob playback.
|
||||
*
|
||||
* @param content The raw M3U8 content string
|
||||
* @param baseUrl The base URL of the M3U8 file (to resolve relative paths)
|
||||
* @returns The filtered M3U8 content
|
||||
*/
|
||||
export type AdFilterMode = 'off' | 'keyword' | 'heuristic' | 'aggressive';
|
||||
|
||||
export function filterM3u8Ad(content: string, baseUrl: string, mode: AdFilterMode = 'heuristic', customKeywords: string[] = []): string {
|
||||
if (!content) return '';
|
||||
|
||||
// 1. Performance: Parse Keywords & Base URL ONLY ONCE
|
||||
// Merge env keywords (build time) with custom keywords (runtime injected)
|
||||
const envKeywordsStr = process.env.NEXT_PUBLIC_AD_KEYWORDS || '';
|
||||
const envKeywords = envKeywordsStr.split(/[\n,]/).map(k => k.trim()).filter(k => k);
|
||||
|
||||
// Combine and deduplicate
|
||||
const keywords = Array.from(new Set([...envKeywords, ...customKeywords]));
|
||||
|
||||
const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/') + 1);
|
||||
let origin = '';
|
||||
try {
|
||||
origin = new URL(baseUrl).origin;
|
||||
} catch (e) { /* ignore */ }
|
||||
|
||||
// 2. Global Scan: Check if any ad keywords exist in the content
|
||||
const hasKeywordMatch = mode !== 'off' && keywords.some(k => content.includes(k));
|
||||
const hasCueTag = mode !== 'off' && (content.includes('#EXT-X-CUE-OUT') || content.includes('#EXT-X-CUE-IN'));
|
||||
|
||||
// 3. Heuristic Analysis: If no explicit ad signals, use block-based detection
|
||||
const lines = content.split(/\r?\n/);
|
||||
let adLineIndices = new Set<number>();
|
||||
|
||||
if (!hasCueTag && (mode === 'heuristic' || mode === 'aggressive')) {
|
||||
// No obvious ad signals - run heuristic analysis
|
||||
const blocks = parseBlocks(lines);
|
||||
if (blocks.length > 1) {
|
||||
const mainPattern = learnMainPattern(blocks);
|
||||
for (const block of blocks) {
|
||||
// Pass all keywords (including custom ones) to heuristic scorer
|
||||
const score = scoreBlock(block, mainPattern, keywords);
|
||||
const threshold = mode === 'aggressive' ? 3.0 : 5.0;
|
||||
if (shouldFilterBlock(score, threshold)) {
|
||||
// Mark all lines in this block for removal
|
||||
for (const segment of block.segments) {
|
||||
adLineIndices.add(segment.lineIndex);
|
||||
adLineIndices.add(segment.lineIndex - 1); // EXTINF line
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const processedLines: string[] = [];
|
||||
|
||||
// State machine for CUE-OUT/CUE-IN tracking
|
||||
let insideCueAdBlock = false;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const trimmedLine = line.trim();
|
||||
|
||||
// Skip lines marked by heuristic analysis
|
||||
if (adLineIndices.has(i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. CUE Tag Detection (SCTE-35 Standard)
|
||||
// EXT-X-CUE-OUT marks start of ad, EXT-X-CUE-IN marks end
|
||||
if (mode !== 'off' && trimmedLine.startsWith('#EXT-X-CUE-OUT')) {
|
||||
insideCueAdBlock = true;
|
||||
// Remove preceding DISCONTINUITY if present
|
||||
if (processedLines.length > 0 && processedLines[processedLines.length - 1].trim() === '#EXT-X-DISCONTINUITY') {
|
||||
processedLines.pop();
|
||||
}
|
||||
continue; // Skip the CUE-OUT tag itself
|
||||
}
|
||||
|
||||
if (trimmedLine.startsWith('#EXT-X-CUE-IN')) {
|
||||
insideCueAdBlock = false;
|
||||
// Also skip the next line if it's a DISCONTINUITY (ad block ending marker)
|
||||
if (i + 1 < lines.length && lines[i + 1].trim() === '#EXT-X-DISCONTINUITY') {
|
||||
i++; // Skip the following DISCONTINUITY
|
||||
}
|
||||
continue; // Skip the CUE-IN tag itself
|
||||
}
|
||||
|
||||
// Skip all content inside CUE ad block
|
||||
if (insideCueAdBlock) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 4. Keyword-based Ad Detection & Backtrack (skip if no keywords configured)
|
||||
if (keywords.length > 0 && hasKeywordMatch && keywords.some(keyword => trimmedLine.includes(keyword))) {
|
||||
// Found Ad: Remove it and backtrack to remove associated metadata
|
||||
while (processedLines.length > 0) {
|
||||
const lastIndex = processedLines.length - 1;
|
||||
const lastLine = processedLines[lastIndex].trim();
|
||||
|
||||
if (lastLine.startsWith('#EXTINF:') || lastLine === '#EXT-X-DISCONTINUITY') {
|
||||
processedLines.pop();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue; // Skip the ad line itself
|
||||
}
|
||||
|
||||
// 5. Discontinuity Handling (Conservative Mode)
|
||||
// Keep all Discontinuity tags by default.
|
||||
// They will ONLY be removed via backtracking when a confirmed ad segment is found.
|
||||
// This prevents false positives on legitimate concatenated streams.
|
||||
if (trimmedLine === '#EXT-X-DISCONTINUITY') {
|
||||
processedLines.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 6. General Cleanup & URL Normalization
|
||||
if (!trimmedLine || trimmedLine.startsWith('http') || trimmedLine.startsWith('blob:')) {
|
||||
processedLines.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (trimmedLine.startsWith('#')) {
|
||||
// Handle URI="..." in attributes (e.g. #EXT-X-KEY)
|
||||
if (trimmedLine.includes('URI="')) {
|
||||
processedLines.push(line.replace(/URI="([^"]+)"/g, (match, uri) => {
|
||||
if (uri.startsWith('http')) return match; // Already absolute
|
||||
if (uri.startsWith('/')) {
|
||||
return `URI="${origin}${uri}"`; // Root-relative
|
||||
}
|
||||
return `URI="${basePath}${uri}"`; // Path-relative
|
||||
}));
|
||||
} else {
|
||||
processedLines.push(line);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 7. Resolve Relative URLs (for Blob support)
|
||||
if (trimmedLine.startsWith('/')) {
|
||||
processedLines.push(origin ? `${origin}${trimmedLine}` : trimmedLine);
|
||||
} else {
|
||||
processedLines.push(`${basePath}${trimmedLine}`);
|
||||
}
|
||||
}
|
||||
|
||||
return processedLines.join('\n');
|
||||
}
|
||||
Reference in New Issue
Block a user