mirror of
https://github.com/KuekHaoYang/KVideo.git
synced 2026-08-22 04:03:42 +08:00
fix: harden M3U8 ad filtering
Reject unsafe dynamic filter execution, reduce heuristic false positives, add regression coverage, bump the app to 4.9.17, and update Next.js to the latest stable security patch.
This commit is contained in:
kuekhaoyang
+26
-381
@@ -1,102 +1,59 @@
|
||||
/**
|
||||
* Heuristic Ad Detection Module
|
||||
*
|
||||
* Provides block-based analysis for detecting ads in M3U8 playlists
|
||||
* using filename pattern matching and other heuristics.
|
||||
*/
|
||||
import type { Block } from './m3u8-ad-types';
|
||||
|
||||
// Ad-related path keywords for scoring
|
||||
export const AD_PATH_KEYWORDS = [
|
||||
'advert', 'preroll', 'midroll', 'postroll',
|
||||
'dai', 'vast', 'ima', 'adjump', 'commercial', 'sponsor'
|
||||
];
|
||||
export type { Block, MainPattern, Segment } from './m3u8-ad-types';
|
||||
export { learnMainPattern } from './m3u8-ad-pattern';
|
||||
export { findDuplicateSignatureBlockIndices } from './m3u8-ad-signatures';
|
||||
export {
|
||||
AD_PATH_KEYWORDS,
|
||||
scoreBlock,
|
||||
shouldFilterBlock,
|
||||
THRESHOLDS,
|
||||
} from './m3u8-ad-scoring';
|
||||
|
||||
/**
|
||||
* Represents a segment in the playlist
|
||||
*/
|
||||
interface Segment {
|
||||
url: string;
|
||||
duration: number;
|
||||
lineIndex: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a block of segments between DISCONTINUITY markers
|
||||
*/
|
||||
interface Block {
|
||||
segments: Segment[];
|
||||
startLineIndex: number;
|
||||
endLineIndex: number;
|
||||
hasCueTag: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pattern extracted from main content for comparison
|
||||
*/
|
||||
interface MainPattern {
|
||||
filenameRegex: RegExp | null;
|
||||
avgDuration: number;
|
||||
commonPrefix: string;
|
||||
pathPrefix: string; // Directory path prefix (e.g., "/20230907/73PWifvT/1392kb/hls/")
|
||||
isGlobalNTSC: boolean; // Whether >30% of total playlist segments use 30fps NTSC frame fractions
|
||||
isGlobal24fps: boolean; // Whether >35% of total playlist segments use 24fps/23.976fps frame fractions (.004, .002, .008)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse M3U8 content into blocks separated by DISCONTINUITY markers
|
||||
*/
|
||||
export function parseBlocks(lines: string[]): Block[] {
|
||||
const blocks: Block[] = [];
|
||||
let currentBlock: Block = {
|
||||
segments: [],
|
||||
startLineIndex: 0,
|
||||
endLineIndex: 0,
|
||||
hasCueTag: false
|
||||
hasCueTag: false,
|
||||
};
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index].trim();
|
||||
|
||||
// Check for CUE tags
|
||||
if (line.startsWith('#EXT-X-CUE-OUT') || line.startsWith('#EXT-X-CUE-IN')) {
|
||||
currentBlock.hasCueTag = true;
|
||||
}
|
||||
|
||||
// DISCONTINUITY marks block boundary
|
||||
if (line === '#EXT-X-DISCONTINUITY') {
|
||||
if (currentBlock.segments.length > 0) {
|
||||
currentBlock.endLineIndex = i - 1;
|
||||
currentBlock.endLineIndex = index - 1;
|
||||
blocks.push(currentBlock);
|
||||
}
|
||||
currentBlock = {
|
||||
segments: [],
|
||||
startLineIndex: i + 1,
|
||||
startLineIndex: index + 1,
|
||||
endLineIndex: 0,
|
||||
hasCueTag: false
|
||||
hasCueTag: false,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse EXTINF and the following URL
|
||||
if (line.startsWith('#EXTINF:')) {
|
||||
const durationMatch = line.match(/#EXTINF:([\d.]+)/);
|
||||
const duration = durationMatch ? parseFloat(durationMatch[1]) : 0;
|
||||
if (!line.startsWith('#EXTINF:')) continue;
|
||||
|
||||
// Next line should be the URL
|
||||
if (i + 1 < lines.length) {
|
||||
const url = lines[i + 1].trim();
|
||||
if (url && !url.startsWith('#')) {
|
||||
currentBlock.segments.push({
|
||||
url,
|
||||
duration,
|
||||
lineIndex: i + 1
|
||||
});
|
||||
}
|
||||
}
|
||||
const durationMatch = line.match(/#EXTINF:([\d.]+)/);
|
||||
const duration = durationMatch ? Number.parseFloat(durationMatch[1]) : 0;
|
||||
const url = lines[index + 1]?.trim();
|
||||
if (url && !url.startsWith('#')) {
|
||||
currentBlock.segments.push({
|
||||
url,
|
||||
duration,
|
||||
lineIndex: index + 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Don't forget the last block
|
||||
if (currentBlock.segments.length > 0) {
|
||||
currentBlock.endLineIndex = lines.length - 1;
|
||||
blocks.push(currentBlock);
|
||||
@@ -104,315 +61,3 @@ export function parseBlocks(lines: string[]): Block[] {
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwrap proxied URL to get original URL
|
||||
*/
|
||||
function unwrapProxyUrl(url: string): string {
|
||||
if (url.includes('/api/proxy?url=')) {
|
||||
try {
|
||||
const match = url.match(/[?&]url=([^&]+)/);
|
||||
if (match && match[1]) {
|
||||
return decodeURIComponent(match[1]);
|
||||
}
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract filename from URL (handles both relative and absolute URLs)
|
||||
*/
|
||||
function extractFilename(url: string): string {
|
||||
try {
|
||||
const unwrappedUrl = unwrapProxyUrl(url);
|
||||
const path = unwrappedUrl.includes('://') ? new URL(unwrappedUrl).pathname : unwrappedUrl;
|
||||
const parts = path.split('/');
|
||||
return parts[parts.length - 1] || '';
|
||||
} catch {
|
||||
return url.split('/').pop() || '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find common prefix among an array of strings
|
||||
*/
|
||||
function findCommonPrefix(strings: string[]): string {
|
||||
if (!strings || strings.length < 2) return '';
|
||||
|
||||
let prefix = '';
|
||||
const first = strings[0];
|
||||
|
||||
for (let i = 0; i < first.length; i++) {
|
||||
const char = first[i];
|
||||
if (strings.every(s => s[i] === char)) {
|
||||
prefix += char;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return prefix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract path prefix (directory) from URL
|
||||
* e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
|
||||
*/
|
||||
function extractPathPrefix(url: string): string {
|
||||
try {
|
||||
const unwrappedUrl = unwrapProxyUrl(url);
|
||||
const path = unwrappedUrl.includes('://') ? new URL(unwrappedUrl).pathname : unwrappedUrl;
|
||||
const lastSlash = path.lastIndexOf('/');
|
||||
return lastSlash >= 0 ? path.substring(0, lastSlash + 1) : '';
|
||||
} catch {
|
||||
const lastSlash = url.lastIndexOf('/');
|
||||
return lastSlash >= 0 ? url.substring(0, lastSlash + 1) : '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Learn pattern from the largest block (assumed to be main content)
|
||||
*/
|
||||
export function learnMainPattern(blocks: Block[]): MainPattern {
|
||||
// Find the largest block by segment count (likely main content)
|
||||
const mainBlock = blocks.length > 0 ? blocks.reduce((largest, block) =>
|
||||
block.segments.length > largest.segments.length ? block : largest
|
||||
) : null;
|
||||
|
||||
if (!mainBlock || mainBlock.segments.length === 0) {
|
||||
return { filenameRegex: null, avgDuration: 0, commonPrefix: '', pathPrefix: '', isGlobalNTSC: false, isGlobal24fps: false };
|
||||
}
|
||||
|
||||
// Extract filenames
|
||||
const filenames = mainBlock.segments.map(s => extractFilename(s.url));
|
||||
|
||||
// Find common prefix
|
||||
const commonPrefix = findCommonPrefix(filenames);
|
||||
|
||||
// Calculate average duration
|
||||
const totalDuration = mainBlock.segments.reduce((sum, s) => sum + s.duration, 0);
|
||||
const avgDuration = totalDuration / mainBlock.segments.length;
|
||||
|
||||
// Try to build a regex pattern from the filenames
|
||||
// Common patterns: "0000001.ts", "seg-1.ts", "segment_001.ts"
|
||||
let filenameRegex: RegExp | null = null;
|
||||
if (commonPrefix.length >= 2) {
|
||||
// Escape special regex characters in prefix
|
||||
const escapedPrefix = commonPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
filenameRegex = new RegExp(`^${escapedPrefix}`);
|
||||
}
|
||||
|
||||
// Extract path prefix (directory path without filename)
|
||||
// e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
|
||||
const firstUrl = mainBlock.segments[0].url;
|
||||
const pathPrefix = extractPathPrefix(firstUrl);
|
||||
|
||||
// Calculate global NTSC 30fps and 24fps/23.976fps fraction ratios across all blocks in the playlist
|
||||
const ntscFractions = new Set([33, 67, 133, 167, 233, 267, 333, 367, 433, 467, 533, 567, 633, 667, 733, 767, 833, 867, 933, 967]);
|
||||
const fps24Fractions = new Set([2, 4, 6, 8, 12, 16, 20, 24]);
|
||||
let totalSegCount = 0;
|
||||
let ntscSegCount = 0;
|
||||
let fps24SegCount = 0;
|
||||
|
||||
blocks.forEach(b => {
|
||||
b.segments.forEach(s => {
|
||||
totalSegCount++;
|
||||
const msFraction = Math.round((s.duration % 1) * 1000);
|
||||
if (ntscFractions.has(msFraction)) ntscSegCount++;
|
||||
if (fps24Fractions.has(msFraction)) fps24SegCount++;
|
||||
});
|
||||
});
|
||||
|
||||
const isGlobalNTSC = totalSegCount > 0 && (ntscSegCount / totalSegCount) > 0.3;
|
||||
const isGlobal24fps = totalSegCount > 0 && (fps24SegCount / totalSegCount) > 0.35;
|
||||
|
||||
return { filenameRegex, avgDuration, commonPrefix, pathPrefix, isGlobalNTSC, isGlobal24fps };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find blocks that share an identical sequence of segment durations (fingerprint)
|
||||
* with another block in the playlist.
|
||||
*
|
||||
* If a block (with >= 3 segments) has an identical duration signature
|
||||
* as another block in the playlist, and it is not the main content block,
|
||||
* it is extremely likely to be a repeated inserted ad block.
|
||||
*/
|
||||
export function findDuplicateSignatureBlockIndices(blocks: Block[]): Set<number> {
|
||||
const duplicateIndices = new Set<number>();
|
||||
if (!blocks || blocks.length < 2) return duplicateIndices;
|
||||
|
||||
// Find the largest block (assumed main content block)
|
||||
let mainBlockIndex = -1;
|
||||
let maxSegments = 0;
|
||||
|
||||
blocks.forEach((block, idx) => {
|
||||
if (block.segments.length > maxSegments) {
|
||||
maxSegments = block.segments.length;
|
||||
mainBlockIndex = idx;
|
||||
}
|
||||
});
|
||||
|
||||
let mainAvgDuration = 0;
|
||||
if (mainBlockIndex >= 0 && blocks[mainBlockIndex].segments.length > 0) {
|
||||
const mainSegs = blocks[mainBlockIndex].segments;
|
||||
mainAvgDuration = mainSegs.reduce((sum, s) => sum + s.duration, 0) / mainSegs.length;
|
||||
}
|
||||
|
||||
// Map signature -> array of block indices
|
||||
const signatureMap = new Map<string, number[]>();
|
||||
|
||||
blocks.forEach((block, idx) => {
|
||||
// Require at least 3 segments to form a signature to prevent accidental single-segment collisions
|
||||
if (block.segments.length < 3) return;
|
||||
|
||||
const firstDur = block.segments[0].duration;
|
||||
const isUniformBlock = block.segments.every(s => Math.abs(s.duration - firstDur) < 0.005);
|
||||
|
||||
// If the block is uniform (e.g. 2.0s, 2.0s, 2.0s) AND its duration matches the main content's duration (e.g. zuida.m3u8 where main content is also 2.0s),
|
||||
// it represents normal main video chunking and must NOT be flagged as an ad signature.
|
||||
if (isUniformBlock && mainAvgDuration > 0 && Math.abs(firstDur - mainAvgDuration) < 0.05) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Signature based on segment durations rounded to 3 decimal places (milliseconds precision)
|
||||
const signature = block.segments.map(s => s.duration.toFixed(3)).join(',');
|
||||
|
||||
const existing = signatureMap.get(signature) || [];
|
||||
existing.push(idx);
|
||||
signatureMap.set(signature, existing);
|
||||
});
|
||||
|
||||
// Flag blocks whose signatures appear 2 or more times (up to 30% of total blocks, at least 3)
|
||||
const maxAllowedAdOccurrences = Math.max(3, blocks.length * 0.3);
|
||||
signatureMap.forEach((indices) => {
|
||||
if (indices.length >= 2 && indices.length <= maxAllowedAdOccurrences) {
|
||||
indices.forEach(idx => {
|
||||
if (idx !== mainBlockIndex) {
|
||||
duplicateIndices.add(idx);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return duplicateIndices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a block for ad likelihood based on heuristics
|
||||
* Returns a score where higher = more likely to be an ad
|
||||
*/
|
||||
export function scoreBlock(
|
||||
block: Block,
|
||||
mainPattern: MainPattern,
|
||||
extraKeywords: string[] = [],
|
||||
isDuplicateSignature: boolean = false
|
||||
): number {
|
||||
let score = 0;
|
||||
|
||||
// If block has CUE tag or matches a duplicate signature, it's definitely an ad
|
||||
if (block.hasCueTag || isDuplicateSignature) {
|
||||
return 10; // Max score
|
||||
}
|
||||
|
||||
// Check path keywords (Built-in + Custom)
|
||||
// We filter out very short custom keywords to avoid false positives in scoring
|
||||
const safeExtraKeywords = extraKeywords.filter(k => k.length > 2);
|
||||
const allKeywords = [...AD_PATH_KEYWORDS, ...safeExtraKeywords];
|
||||
|
||||
for (const segment of block.segments) {
|
||||
const urlLower = segment.url.toLowerCase();
|
||||
for (const keyword of allKeywords) {
|
||||
if (urlLower.includes(keyword.toLowerCase())) {
|
||||
score += 2.5;
|
||||
break; // Only count once per segment
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check filename pattern mismatch
|
||||
if (mainPattern.filenameRegex) {
|
||||
const mismatchCount = block.segments.filter(s => {
|
||||
if (!mainPattern.filenameRegex) return false; // No pattern to compare against
|
||||
const filename = extractFilename(s.url);
|
||||
return !mainPattern.filenameRegex.test(filename);
|
||||
}).length;
|
||||
|
||||
if (mismatchCount === block.segments.length && block.segments.length > 0) {
|
||||
score += 1.5; // All filenames differ from main pattern
|
||||
}
|
||||
}
|
||||
|
||||
// **KEY FEATURE**: Check path prefix mismatch (e.g., different date/folder/bitrate)
|
||||
// This is the most reliable indicator for ads that come from different CDN paths
|
||||
if (mainPattern.pathPrefix !== undefined && block.segments.length > 0) {
|
||||
const pathMismatchCount = block.segments.filter(s => {
|
||||
const segmentPathPrefix = extractPathPrefix(s.url);
|
||||
return segmentPathPrefix !== mainPattern.pathPrefix;
|
||||
}).length;
|
||||
|
||||
if (pathMismatchCount === block.segments.length) {
|
||||
// ALL segments have different path prefix - strong ad indicator
|
||||
score += 5.0;
|
||||
}
|
||||
}
|
||||
|
||||
// **ENHANCEMENT**: Duration anomaly heuristic for mid-roll inserted ad blocks
|
||||
// If main content has a consistent segment duration (e.g., 6.0s or 10.0s),
|
||||
// but this small block has a vastly different duration signature (e.g., 2.0s),
|
||||
// add additional score when combined with filename or path mismatch.
|
||||
if (mainPattern.avgDuration > 0 && block.segments.length > 0 && block.segments.length <= 6) {
|
||||
const blockAvgDuration = block.segments.reduce((sum, s) => sum + s.duration, 0) / block.segments.length;
|
||||
const durationRatio = blockAvgDuration / mainPattern.avgDuration;
|
||||
if (durationRatio < 0.6 || durationRatio > 1.8) {
|
||||
score += 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
// **KEY FEATURE**: Framerate fraction grid anomaly detection (30fps vs 25fps inserted ad detection)
|
||||
// Spliced NTSC 30fps/60fps ads inserted into 25fps/50fps streams create distinct fractional ms (.333, .667, .867, .133, .733).
|
||||
// Note: Only triggered if the main content itself is NOT a global NTSC 30fps stream.
|
||||
if (!mainPattern.isGlobalNTSC && block.segments.length > 0 && block.segments.length <= 10) {
|
||||
const ntscFractions = new Set([33, 67, 133, 167, 233, 267, 333, 367, 433, 467, 533, 567, 633, 667, 733, 767, 833, 867, 933, 967]);
|
||||
let ntscMatches = 0;
|
||||
block.segments.forEach(s => {
|
||||
const msFraction = Math.round((s.duration % 1) * 1000);
|
||||
if (ntscFractions.has(msFraction)) ntscMatches++;
|
||||
});
|
||||
if (ntscMatches === block.segments.length) {
|
||||
score += 5.0; // Definite framerate anomaly for inserted ad block
|
||||
}
|
||||
}
|
||||
|
||||
// **KEY FEATURE**: Framerate fraction grid anomaly detection (25fps/integer ads inserted into 24fps/23.976fps streams)
|
||||
// 24fps movie streams have fractional ms (.004, .002, .008). 25fps/PAL inserted ads use exact integer milliseconds (.000).
|
||||
if (mainPattern.isGlobal24fps && block.segments.length > 0 && block.segments.length <= 10) {
|
||||
let int0Count = 0;
|
||||
block.segments.forEach(s => {
|
||||
const msFraction = Math.round((s.duration % 1) * 1000);
|
||||
if (msFraction === 0) int0Count++;
|
||||
});
|
||||
if (int0Count >= block.segments.length - 1 && int0Count >= 2) {
|
||||
score += 5.0; // Definite 25fps integer ad inserted into 24fps movie stream
|
||||
}
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Threshold configuration
|
||||
*/
|
||||
export const THRESHOLDS = {
|
||||
HIGH: 5.0, // Definitely an ad
|
||||
LOW: 3.0 // Possibly an ad (for future "fuzzy" mode)
|
||||
};
|
||||
|
||||
/**
|
||||
* Determine if a block should be filtered based on its score
|
||||
*/
|
||||
export function shouldFilterBlock(score: number, threshold: number = THRESHOLDS.HIGH): boolean {
|
||||
return score >= threshold;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import {
|
||||
hasDominantFilm24LikeGrid,
|
||||
hasDominantNtscLikeGrid,
|
||||
} from './m3u8-duration-grid';
|
||||
import type { Block, MainPattern, SegmentLocation } from './m3u8-ad-types';
|
||||
|
||||
function unwrapProxyUrl(url: string): string {
|
||||
if (!url.includes('/api/proxy?url=')) return url;
|
||||
|
||||
try {
|
||||
const match = url.match(/[?&]url=([^&]+)/);
|
||||
return match?.[1] ? decodeURIComponent(match[1]) : url;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractFilename(url: string): string {
|
||||
try {
|
||||
const unwrappedUrl = unwrapProxyUrl(url);
|
||||
const path = unwrappedUrl.includes('://')
|
||||
? new URL(unwrappedUrl).pathname
|
||||
: unwrappedUrl;
|
||||
return path.split('/').pop() || '';
|
||||
} catch {
|
||||
return url.split('/').pop() || '';
|
||||
}
|
||||
}
|
||||
|
||||
function findCommonPrefix(strings: string[]): string {
|
||||
if (strings.length < 2) return '';
|
||||
|
||||
let prefix = '';
|
||||
for (let index = 0; index < strings[0].length; index += 1) {
|
||||
const character = strings[0][index];
|
||||
if (!strings.every(value => value[index] === character)) break;
|
||||
prefix += character;
|
||||
}
|
||||
return prefix;
|
||||
}
|
||||
|
||||
export function extractSegmentLocation(url: string): SegmentLocation {
|
||||
try {
|
||||
const unwrappedUrl = unwrapProxyUrl(url);
|
||||
const parsedUrl = unwrappedUrl.includes('://') ? new URL(unwrappedUrl) : null;
|
||||
const path = parsedUrl?.pathname || unwrappedUrl;
|
||||
const lastSlash = path.lastIndexOf('/');
|
||||
return {
|
||||
origin: parsedUrl?.origin || '',
|
||||
pathPrefix: lastSlash >= 0 ? path.substring(0, lastSlash + 1) : '',
|
||||
};
|
||||
} catch {
|
||||
const lastSlash = url.lastIndexOf('/');
|
||||
return {
|
||||
origin: '',
|
||||
pathPrefix: lastSlash >= 0 ? url.substring(0, lastSlash + 1) : '',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function learnMainPattern(blocks: Block[]): MainPattern {
|
||||
const mainBlock = blocks.length > 0
|
||||
? blocks.reduce((largest, block) => (
|
||||
block.segments.length > largest.segments.length ? block : largest
|
||||
))
|
||||
: null;
|
||||
|
||||
if (!mainBlock || mainBlock.segments.length === 0) {
|
||||
return {
|
||||
filenameRegex: null,
|
||||
avgDuration: 0,
|
||||
commonPrefix: '',
|
||||
pathPrefix: '',
|
||||
origin: '',
|
||||
usesNtscLikeGrid: false,
|
||||
usesFilm24LikeGrid: false,
|
||||
};
|
||||
}
|
||||
|
||||
const filenames = mainBlock.segments.map(segment => extractFilename(segment.url));
|
||||
const commonPrefix = findCommonPrefix(filenames);
|
||||
const totalDuration = mainBlock.segments.reduce((sum, segment) => (
|
||||
sum + segment.duration
|
||||
), 0);
|
||||
const avgDuration = totalDuration / mainBlock.segments.length;
|
||||
const escapedPrefix = commonPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const filenameRegex = commonPrefix.length >= 2
|
||||
? new RegExp(`^${escapedPrefix}`)
|
||||
: null;
|
||||
const location = extractSegmentLocation(mainBlock.segments[0].url);
|
||||
|
||||
return {
|
||||
filenameRegex,
|
||||
avgDuration,
|
||||
commonPrefix,
|
||||
...location,
|
||||
usesNtscLikeGrid: hasDominantNtscLikeGrid(mainBlock.segments),
|
||||
usesFilm24LikeGrid: hasDominantFilm24LikeGrid(mainBlock.segments),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import {
|
||||
isSmallIntegerDurationBlock,
|
||||
isSmallNtscLikeBlock,
|
||||
} from './m3u8-duration-grid';
|
||||
import { extractFilename, extractSegmentLocation } from './m3u8-ad-pattern';
|
||||
import type { Block, MainPattern } from './m3u8-ad-types';
|
||||
|
||||
export const AD_PATH_KEYWORDS = [
|
||||
'advert', 'preroll', 'midroll', 'postroll',
|
||||
'dai', 'vast', 'ima', 'adjump', 'commercial', 'sponsor',
|
||||
];
|
||||
|
||||
function scoreKeywords(block: Block, extraKeywords: string[]): number {
|
||||
const keywords = [
|
||||
...AD_PATH_KEYWORDS,
|
||||
...extraKeywords.filter(keyword => keyword.length > 2),
|
||||
];
|
||||
|
||||
return block.segments.reduce((score, segment) => {
|
||||
const url = segment.url.toLowerCase();
|
||||
return keywords.some(keyword => url.includes(keyword.toLowerCase()))
|
||||
? score + 2.5
|
||||
: score;
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function scoreFilenameMismatch(block: Block, mainPattern: MainPattern): number {
|
||||
if (!mainPattern.filenameRegex || block.segments.length === 0) return 0;
|
||||
|
||||
const allMismatch = block.segments.every(segment => (
|
||||
!mainPattern.filenameRegex?.test(extractFilename(segment.url))
|
||||
));
|
||||
return allMismatch ? 1.5 : 0;
|
||||
}
|
||||
|
||||
function scoreLocationMismatch(block: Block, mainPattern: MainPattern): number {
|
||||
if ((!mainPattern.pathPrefix && !mainPattern.origin) || block.segments.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const locations = block.segments.map(segment => extractSegmentLocation(segment.url));
|
||||
const allPathsMismatch = locations.every(location => (
|
||||
location.pathPrefix !== mainPattern.pathPrefix
|
||||
));
|
||||
if (allPathsMismatch) return 5;
|
||||
|
||||
const allOriginsMismatch = locations.every(location => (
|
||||
Boolean(mainPattern.origin && location.origin) && location.origin !== mainPattern.origin
|
||||
));
|
||||
return allOriginsMismatch ? 3.5 : 0;
|
||||
}
|
||||
|
||||
function scoreDurationMismatch(block: Block, mainPattern: MainPattern): number {
|
||||
if (mainPattern.avgDuration <= 0 || block.segments.length === 0 || block.segments.length > 6) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const blockAverage = block.segments.reduce((sum, segment) => (
|
||||
sum + segment.duration
|
||||
), 0) / block.segments.length;
|
||||
const durationRatio = blockAverage / mainPattern.avgDuration;
|
||||
return durationRatio < 0.6 || durationRatio > 1.8 ? 1.5 : 0;
|
||||
}
|
||||
|
||||
function scoreDurationGrid(block: Block, mainPattern: MainPattern): number {
|
||||
if (!mainPattern.usesNtscLikeGrid && isSmallNtscLikeBlock(block.segments)) {
|
||||
return 3.5;
|
||||
}
|
||||
if (mainPattern.usesFilm24LikeGrid && isSmallIntegerDurationBlock(block.segments)) {
|
||||
return 3.5;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function scoreBlock(
|
||||
block: Block,
|
||||
mainPattern: MainPattern,
|
||||
extraKeywords: string[] = [],
|
||||
isDuplicateSignature: boolean = false,
|
||||
): number {
|
||||
if (block.hasCueTag || isDuplicateSignature) return 10;
|
||||
|
||||
return scoreKeywords(block, extraKeywords) +
|
||||
scoreFilenameMismatch(block, mainPattern) +
|
||||
scoreLocationMismatch(block, mainPattern) +
|
||||
scoreDurationMismatch(block, mainPattern) +
|
||||
scoreDurationGrid(block, mainPattern);
|
||||
}
|
||||
|
||||
export const THRESHOLDS = {
|
||||
HIGH: 5,
|
||||
LOW: 3,
|
||||
};
|
||||
|
||||
export function shouldFilterBlock(
|
||||
score: number,
|
||||
threshold: number = THRESHOLDS.HIGH,
|
||||
): boolean {
|
||||
return score >= threshold;
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { Block } from './m3u8-ad-types';
|
||||
|
||||
export function findDuplicateSignatureBlockIndices(blocks: Block[]): Set<number> {
|
||||
const duplicateIndices = new Set<number>();
|
||||
if (blocks.length < 2) return duplicateIndices;
|
||||
|
||||
let mainBlockIndex = -1;
|
||||
let maxSegments = 0;
|
||||
blocks.forEach((block, index) => {
|
||||
if (block.segments.length > maxSegments) {
|
||||
maxSegments = block.segments.length;
|
||||
mainBlockIndex = index;
|
||||
}
|
||||
});
|
||||
|
||||
const mainSegments = mainBlockIndex >= 0 ? blocks[mainBlockIndex].segments : [];
|
||||
const mainAvgDuration = mainSegments.length > 0
|
||||
? mainSegments.reduce((sum, segment) => sum + segment.duration, 0) / mainSegments.length
|
||||
: 0;
|
||||
const signatureMap = new Map<string, number[]>();
|
||||
|
||||
blocks.forEach((block, index) => {
|
||||
if (block.segments.length < 3) return;
|
||||
|
||||
const firstDuration = block.segments[0].duration;
|
||||
const isUniform = block.segments.every(segment => (
|
||||
Math.abs(segment.duration - firstDuration) < 0.005
|
||||
));
|
||||
const matchesMainDuration = mainAvgDuration > 0 &&
|
||||
Math.abs(firstDuration - mainAvgDuration) < 0.05;
|
||||
if (isUniform && matchesMainDuration) return;
|
||||
|
||||
const signature = block.segments
|
||||
.map(segment => segment.duration.toFixed(3))
|
||||
.join(',');
|
||||
const matchingBlocks = signatureMap.get(signature) || [];
|
||||
matchingBlocks.push(index);
|
||||
signatureMap.set(signature, matchingBlocks);
|
||||
});
|
||||
|
||||
const maxOccurrences = Math.max(2, Math.floor(blocks.length * 0.3));
|
||||
signatureMap.forEach((indices) => {
|
||||
if (indices.length < 2 || indices.length > maxOccurrences) return;
|
||||
|
||||
indices.forEach((index) => {
|
||||
const isClearlySmallerThanMain = blocks[index].segments.length < maxSegments * 0.8;
|
||||
if (index !== mainBlockIndex && isClearlySmallerThanMain) {
|
||||
duplicateIndices.add(index);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return duplicateIndices;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
export interface Segment {
|
||||
url: string;
|
||||
duration: number;
|
||||
lineIndex: number;
|
||||
}
|
||||
|
||||
export interface Block {
|
||||
segments: Segment[];
|
||||
startLineIndex: number;
|
||||
endLineIndex: number;
|
||||
hasCueTag: boolean;
|
||||
}
|
||||
|
||||
export interface MainPattern {
|
||||
filenameRegex: RegExp | null;
|
||||
avgDuration: number;
|
||||
commonPrefix: string;
|
||||
pathPrefix: string;
|
||||
origin: string;
|
||||
usesNtscLikeGrid: boolean;
|
||||
usesFilm24LikeGrid: boolean;
|
||||
}
|
||||
|
||||
export interface SegmentLocation {
|
||||
origin: string;
|
||||
pathPrefix: string;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
interface DurationSample {
|
||||
duration: number;
|
||||
}
|
||||
|
||||
const NTSC_LIKE_MILLISECOND_FRACTIONS = new Set([
|
||||
33, 67, 133, 167, 233, 267, 333, 367, 433, 467,
|
||||
533, 567, 633, 667, 733, 767, 833, 867, 933, 967,
|
||||
]);
|
||||
|
||||
const FILM_24_LIKE_MILLISECOND_FRACTIONS = new Set([
|
||||
2, 4, 6, 8, 12, 16, 20, 24,
|
||||
]);
|
||||
|
||||
function millisecondFraction(duration: number): number {
|
||||
const fractionalSeconds = Math.abs(duration - Math.trunc(duration));
|
||||
return Math.round(fractionalSeconds * 1000) % 1000;
|
||||
}
|
||||
|
||||
function matchingRatio(samples: DurationSample[], fractions: Set<number>): number {
|
||||
if (samples.length === 0) return 0;
|
||||
|
||||
const matches = samples.reduce((count, sample) => (
|
||||
fractions.has(millisecondFraction(sample.duration)) ? count + 1 : count
|
||||
), 0);
|
||||
|
||||
return matches / samples.length;
|
||||
}
|
||||
|
||||
export function hasDominantNtscLikeGrid(samples: DurationSample[]): boolean {
|
||||
return samples.length >= 4 &&
|
||||
matchingRatio(samples, NTSC_LIKE_MILLISECOND_FRACTIONS) > 0.3;
|
||||
}
|
||||
|
||||
export function hasDominantFilm24LikeGrid(samples: DurationSample[]): boolean {
|
||||
return samples.length >= 4 &&
|
||||
matchingRatio(samples, FILM_24_LIKE_MILLISECOND_FRACTIONS) > 0.35;
|
||||
}
|
||||
|
||||
export function isSmallNtscLikeBlock(samples: DurationSample[]): boolean {
|
||||
return samples.length >= 2 &&
|
||||
samples.length <= 10 &&
|
||||
matchingRatio(samples, NTSC_LIKE_MILLISECOND_FRACTIONS) >= 0.8;
|
||||
}
|
||||
|
||||
export function isSmallIntegerDurationBlock(samples: DurationSample[]): boolean {
|
||||
if (samples.length < 2 || samples.length > 10) return false;
|
||||
|
||||
const integerDurations = samples.reduce((count, sample) => (
|
||||
millisecondFraction(sample.duration) === 0 ? count + 1 : count
|
||||
), 0);
|
||||
|
||||
return integerDurations >= 2 && integerDurations / samples.length >= 0.75;
|
||||
}
|
||||
+4
-45
@@ -81,58 +81,17 @@ function isAuxiliaryAdMetadataLine(trimmedLine: string, normalizedKeywords: stri
|
||||
* @param baseUrl The base URL of the M3U8 file (to resolve relative paths)
|
||||
* @returns The filtered M3U8 content
|
||||
*/
|
||||
/**
|
||||
* Strips TypeScript type annotations from dynamic ad filter JavaScript/TypeScript code
|
||||
*/
|
||||
export function removeTypeAnnotations(code: string): string {
|
||||
return code
|
||||
.replace(/(\w+)\s*:\s*(string|number|boolean|any|void|never|unknown|object)\s*([,)])/g, '$1$3')
|
||||
.replace(/\)\s*:\s*(string|number|boolean|any|void|never|unknown|object)\s*\{/g, ') {')
|
||||
.replace(/(const|let|var)\s+(\w+)\s*:\s*(string|number|boolean|any|void|never|unknown|object)\s*=/g, '$1 $2 =');
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely executes dynamic custom ad filter code using Function constructor sandbox.
|
||||
* Returns filtered content string or null if execution fails/throws.
|
||||
*/
|
||||
export function executeCustomAdFilter(customCode: string, content: string, baseUrl: string): string | null {
|
||||
if (!customCode || !customCode.trim()) return null;
|
||||
try {
|
||||
const jsCode = removeTypeAnnotations(customCode);
|
||||
const customFunction = new Function(
|
||||
'content',
|
||||
'baseUrl',
|
||||
jsCode + '\nif (typeof filterAdsFromM3U8 === "function") { return filterAdsFromM3U8(content, baseUrl); } else if (typeof filterAds === "function") { return filterAds(content, baseUrl); } return content;'
|
||||
);
|
||||
const result = customFunction(content, baseUrl);
|
||||
return typeof result === 'string' ? result : null;
|
||||
} catch (err) {
|
||||
console.warn('[AdFilter] Custom script execution failed, falling back to built-in rules:', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export type AdFilterMode = 'off' | 'keyword' | 'heuristic' | 'aggressive';
|
||||
|
||||
export function filterM3u8Ad(
|
||||
content: string,
|
||||
baseUrl: string,
|
||||
mode: AdFilterMode = 'heuristic',
|
||||
customKeywords: string[] = [],
|
||||
customCode?: string
|
||||
customKeywords: string[] = []
|
||||
): string {
|
||||
if (!content) return '';
|
||||
if (mode === 'off') return content;
|
||||
|
||||
// 1. Try executing custom dynamic filter script first (if provided)
|
||||
if (customCode && customCode.trim()) {
|
||||
const customResult = executeCustomAdFilter(customCode, content, baseUrl);
|
||||
if (customResult !== null) {
|
||||
return customResult;
|
||||
}
|
||||
// If custom script execution fails/throws, gracefully fall through to built-in heuristics
|
||||
}
|
||||
|
||||
// Use keywords passed from AdKeywordsWrapper (already loaded from env/file)
|
||||
const normalizedKeywords = normalizeKeywords(customKeywords);
|
||||
|
||||
@@ -144,14 +103,14 @@ export function filterM3u8Ad(
|
||||
if (urlMatch && urlMatch[1]) {
|
||||
effectiveBaseUrl = decodeURIComponent(urlMatch[1]);
|
||||
}
|
||||
} catch (e) { /* ignore */ }
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
const basePath = effectiveBaseUrl.substring(0, effectiveBaseUrl.lastIndexOf('/') + 1);
|
||||
let origin = '';
|
||||
try {
|
||||
origin = new URL(effectiveBaseUrl).origin;
|
||||
} catch (e) { /* ignore */ }
|
||||
} catch { /* ignore */ }
|
||||
|
||||
// 2. Global Scan: Check if any ad keywords exist in the content
|
||||
const hasKeywordMatchInPlaylist = hasKeywordMatch(content, normalizedKeywords);
|
||||
@@ -159,7 +118,7 @@ export function filterM3u8Ad(
|
||||
|
||||
// 3. Heuristic Analysis: If no explicit ad signals, use block-based detection
|
||||
const lines = content.split(/\r?\n/);
|
||||
let adLineIndices = new Set<number>();
|
||||
const adLineIndices = new Set<number>();
|
||||
|
||||
if (!hasCueTag && (mode === 'heuristic' || mode === 'aggressive')) {
|
||||
// No obvious ad signals - run heuristic analysis
|
||||
|
||||
Reference in New Issue
Block a user