新增M3U8启发式广告检测算法并支持动态关键词配置

lib/store/settings-store.ts

@@ -18,6 +18,7 @@ export type SortOption =
   | 'name-desc';
 
 export type SearchDisplayMode = 'normal' | 'grouped';
+export type AdFilterMode = 'off' | 'keyword' | 'heuristic' | 'aggressive';
 
 export interface AppSettings {
   sources: VideoSource[];
@@ -35,6 +36,9 @@ export interface AppSettings {
   autoSkipOutro: boolean;
   skipOutroSeconds: number;
   showModeIndicator: boolean; // Show '直连模式'/'代理模式' badge on player
+  adFilter: boolean; // Filter ad tags from m3u8 (legacy, kept for compatibility)
+  adFilterMode: AdFilterMode; // 'off' | 'keyword' | 'heuristic' | 'aggressive'
+  adKeywords: string[]; // Dynamically loaded ad keywords
   // Search & Display settings
   realtimeLatency: boolean; // Enable real-time latency ping updates
   searchDisplayMode: SearchDisplayMode; // 'normal' = individual cards, 'grouped' = group same-name videos
@@ -87,51 +91,43 @@ function getEnvSubscriptions(customValue?: string): SourceSubscription[] {
 // Debugging helper
 // console.log("Environment Subscriptions:", getEnvSubscriptions());
 
+// Shared default settings factory to avoid code duplication
+function getDefaultAppSettings(): AppSettings {
+  return {
+    sources: getDefaultSources(),
+    premiumSources: getDefaultPremiumSources(),
+    subscriptions: getEnvSubscriptions(),
+    sortBy: 'default',
+    searchHistory: true,
+    watchHistory: true,
+    passwordAccess: false,
+    accessPasswords: [],
+    autoNextEpisode: true,
+    autoSkipIntro: false,
+    skipIntroSeconds: 0,
+    autoSkipOutro: false,
+    skipOutroSeconds: 0,
+    showModeIndicator: false,
+    adFilter: false,
+    adFilterMode: 'heuristic',
+    adKeywords: [],
+    realtimeLatency: false,
+    searchDisplayMode: 'normal',
+    episodeReverseOrder: false,
+  };
+}
+
 export const settingsStore = {
   getSettings(): AppSettings {
+    // SSR: Return defaults
     if (typeof window === 'undefined') {
-      return {
-        sources: getDefaultSources(),
-        premiumSources: getDefaultPremiumSources(),
-        subscriptions: getEnvSubscriptions(),
-        sortBy: 'default',
-        searchHistory: true,
-        watchHistory: true,
-        passwordAccess: false,
-        accessPasswords: [],
-        autoNextEpisode: true,
-        autoSkipIntro: false,
-        skipIntroSeconds: 0,
-        autoSkipOutro: false,
-        skipOutroSeconds: 0,
-        showModeIndicator: false,
-        realtimeLatency: false,
-        searchDisplayMode: 'normal',
-        episodeReverseOrder: false,
-      };
+      return getDefaultAppSettings();
     }
 
+    // Client: No stored settings, return defaults
     const stored = localStorage.getItem(SETTINGS_KEY);
     if (!stored) {
-      return {
-        sources: getDefaultSources(),
-        premiumSources: getDefaultPremiumSources(),
-        subscriptions: getEnvSubscriptions(),
-        sortBy: 'default',
-        searchHistory: true,
-        watchHistory: true,
-        passwordAccess: false,
-        accessPasswords: [],
-        autoNextEpisode: true,
-        autoSkipIntro: false,
-        skipIntroSeconds: 0,
-        autoSkipOutro: false,
-        skipOutroSeconds: 0,
-        showModeIndicator: false,
-        realtimeLatency: false,
-        searchDisplayMode: 'normal',
-        episodeReverseOrder: false,
-      };
+      return getDefaultAppSettings();
     }
 
     try {
@@ -187,33 +183,16 @@ export const settingsStore = {
         autoSkipOutro: parsed.autoSkipOutro !== undefined ? parsed.autoSkipOutro : false,
         skipOutroSeconds: typeof parsed.skipOutroSeconds === 'number' ? parsed.skipOutroSeconds : 0,
         showModeIndicator: parsed.showModeIndicator !== undefined ? parsed.showModeIndicator : false,
+        adFilter: parsed.adFilter !== undefined ? parsed.adFilter : false,
+        adFilterMode: parsed.adFilterMode || 'heuristic',
+        adKeywords: Array.isArray(parsed.adKeywords) ? parsed.adKeywords : [],
         realtimeLatency: parsed.realtimeLatency !== undefined ? parsed.realtimeLatency : false,
         searchDisplayMode: parsed.searchDisplayMode === 'grouped' ? 'grouped' : 'normal',
         episodeReverseOrder: parsed.episodeReverseOrder !== undefined ? parsed.episodeReverseOrder : false,
       };
     } catch {
       // Even if localStorage fails, we should return defaults + ENV subscriptions
-      const envSubscriptions = getEnvSubscriptions();
-
-      return {
-        sources: getDefaultSources(),
-        premiumSources: getDefaultPremiumSources(),
-        subscriptions: envSubscriptions,
-        sortBy: 'default',
-        searchHistory: true,
-        watchHistory: true,
-        passwordAccess: false,
-        accessPasswords: [],
-        autoNextEpisode: true,
-        autoSkipIntro: false,
-        skipIntroSeconds: 0,
-        autoSkipOutro: false,
-        skipOutroSeconds: 0,
-        showModeIndicator: false,
-        realtimeLatency: false,
-        searchDisplayMode: 'normal',
-        episodeReverseOrder: false,
-      };
+      return getDefaultAppSettings();
     }
   },
 

lib/utils/m3u8-ad-detector.ts

@@ -0,0 +1,265 @@
+/**
+ * Heuristic Ad Detection Module
+ * 
+ * Provides block-based analysis for detecting ads in M3U8 playlists
+ * using filename pattern matching and other heuristics.
+ */
+
+// Ad-related path keywords for scoring
+export const AD_PATH_KEYWORDS = [
+    'advert', 'preroll', 'midroll', 'postroll',
+    'dai', 'vast', 'ima', 'adjump', 'commercial', 'sponsor'
+];
+
+/**
+ * Represents a segment in the playlist
+ */
+interface Segment {
+    url: string;
+    duration: number;
+    lineIndex: number;
+}
+
+/**
+ * Represents a block of segments between DISCONTINUITY markers
+ */
+interface Block {
+    segments: Segment[];
+    startLineIndex: number;
+    endLineIndex: number;
+    hasCueTag: boolean;
+}
+
+/**
+ * Pattern extracted from main content for comparison
+ */
+interface MainPattern {
+    filenameRegex: RegExp | null;
+    avgDuration: number;
+    commonPrefix: string;
+    pathPrefix: string; // Directory path prefix (e.g., "/20230907/73PWifvT/1392kb/hls/")
+}
+
+/**
+ * Parse M3U8 content into blocks separated by DISCONTINUITY markers
+ */
+export function parseBlocks(lines: string[]): Block[] {
+    const blocks: Block[] = [];
+    let currentBlock: Block = {
+        segments: [],
+        startLineIndex: 0,
+        endLineIndex: 0,
+        hasCueTag: false
+    };
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+
+        // Check for CUE tags
+        if (line.startsWith('#EXT-X-CUE-OUT') || line.startsWith('#EXT-X-CUE-IN')) {
+            currentBlock.hasCueTag = true;
+        }
+
+        // DISCONTINUITY marks block boundary
+        if (line === '#EXT-X-DISCONTINUITY') {
+            if (currentBlock.segments.length > 0) {
+                currentBlock.endLineIndex = i - 1;
+                blocks.push(currentBlock);
+            }
+            currentBlock = {
+                segments: [],
+                startLineIndex: i + 1,
+                endLineIndex: 0,
+                hasCueTag: false
+            };
+            continue;
+        }
+
+        // Parse EXTINF and the following URL
+        if (line.startsWith('#EXTINF:')) {
+            const durationMatch = line.match(/#EXTINF:([\d.]+)/);
+            const duration = durationMatch ? parseFloat(durationMatch[1]) : 0;
+
+            // Next line should be the URL
+            if (i + 1 < lines.length) {
+                const url = lines[i + 1].trim();
+                if (url && !url.startsWith('#')) {
+                    currentBlock.segments.push({
+                        url,
+                        duration,
+                        lineIndex: i + 1
+                    });
+                }
+            }
+        }
+    }
+
+    // Don't forget the last block
+    if (currentBlock.segments.length > 0) {
+        currentBlock.endLineIndex = lines.length - 1;
+        blocks.push(currentBlock);
+    }
+
+    return blocks;
+}
+
+/**
+ * Extract filename from URL (handles both relative and absolute URLs)
+ */
+function extractFilename(url: string): string {
+    try {
+        const path = url.includes('://') ? new URL(url).pathname : url;
+        const parts = path.split('/');
+        return parts[parts.length - 1] || '';
+    } catch {
+        return url.split('/').pop() || '';
+    }
+}
+
+/**
+ * Find common prefix among an array of strings
+ */
+function findCommonPrefix(strings: string[]): string {
+    if (!strings || strings.length < 2) return '';
+
+    let prefix = '';
+    const first = strings[0];
+
+    for (let i = 0; i < first.length; i++) {
+        const char = first[i];
+        if (strings.every(s => s[i] === char)) {
+            prefix += char;
+        } else {
+            break;
+        }
+    }
+
+    return prefix;
+}
+
+/**
+ * Extract path prefix (directory) from URL
+ * e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
+ */
+function extractPathPrefix(url: string): string {
+    try {
+        const path = url.includes('://') ? new URL(url).pathname : url;
+        const lastSlash = path.lastIndexOf('/');
+        return lastSlash >= 0 ? path.substring(0, lastSlash + 1) : '';
+    } catch {
+        const lastSlash = url.lastIndexOf('/');
+        return lastSlash >= 0 ? url.substring(0, lastSlash + 1) : '';
+    }
+}
+
+/**
+ * Learn pattern from the largest block (assumed to be main content)
+ */
+export function learnMainPattern(blocks: Block[]): MainPattern {
+    // Find the largest block by segment count (likely main content)
+    const mainBlock = blocks.length > 0 ? blocks.reduce((largest, block) =>
+        block.segments.length > largest.segments.length ? block : largest
+    ) : null;
+
+    if (!mainBlock || mainBlock.segments.length === 0) {
+        return { filenameRegex: null, avgDuration: 0, commonPrefix: '', pathPrefix: '' };
+    }
+
+    // Extract filenames
+    const filenames = mainBlock.segments.map(s => extractFilename(s.url));
+
+    // Find common prefix
+    const commonPrefix = findCommonPrefix(filenames);
+
+    // Calculate average duration
+    const totalDuration = mainBlock.segments.reduce((sum, s) => sum + s.duration, 0);
+    const avgDuration = totalDuration / mainBlock.segments.length;
+
+    // Try to build a regex pattern from the filenames
+    // Common patterns: "0000001.ts", "seg-1.ts", "segment_001.ts"
+    let filenameRegex: RegExp | null = null;
+    if (commonPrefix.length >= 2) {
+        // Escape special regex characters in prefix
+        const escapedPrefix = commonPrefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        filenameRegex = new RegExp(`^${escapedPrefix}`);
+    }
+
+    // Extract path prefix (directory path without filename)
+    // e.g., "/20230907/73PWifvT/1392kb/hls/" from "/20230907/73PWifvT/1392kb/hls/gFE6lwIk.ts"
+    const firstUrl = mainBlock.segments[0].url;
+    const pathPrefix = extractPathPrefix(firstUrl);
+
+    return { filenameRegex, avgDuration, commonPrefix, pathPrefix };
+}
+
+/**
+ * Score a block for ad likelihood based on heuristics
+ * Returns a score where higher = more likely to be an ad
+ */
+export function scoreBlock(block: Block, mainPattern: MainPattern, extraKeywords: string[] = []): number {
+    let score = 0;
+
+    // If block has CUE tag, it's definitely an ad
+    if (block.hasCueTag) {
+        return 10; // Max score
+    }
+
+    // Check path keywords (Built-in + Custom)
+    // We filter out very short custom keywords to avoid false positives in scoring
+    const safeExtraKeywords = extraKeywords.filter(k => k.length > 2);
+    const allKeywords = [...AD_PATH_KEYWORDS, ...safeExtraKeywords];
+
+    for (const segment of block.segments) {
+        const urlLower = segment.url.toLowerCase();
+        for (const keyword of allKeywords) {
+            if (urlLower.includes(keyword.toLowerCase())) {
+                score += 2.5;
+                break; // Only count once per segment
+            }
+        }
+    }
+
+    // Check filename pattern mismatch
+    if (mainPattern.filenameRegex) {
+        const mismatchCount = block.segments.filter(s => {
+            if (!mainPattern.filenameRegex) return false; // No pattern to compare against
+            const filename = extractFilename(s.url);
+            return !mainPattern.filenameRegex.test(filename);
+        }).length;
+
+        if (mismatchCount === block.segments.length && block.segments.length > 0) {
+            score += 1.5; // All filenames differ from main pattern
+        }
+    }
+
+    // **KEY FEATURE**: Check path prefix mismatch (e.g., different date/folder/bitrate)
+    // This is the most reliable indicator for ads that come from different CDN paths
+    if (mainPattern.pathPrefix && block.segments.length > 0) {
+        const pathMismatchCount = block.segments.filter(s => {
+            const segmentPathPrefix = extractPathPrefix(s.url);
+            return segmentPathPrefix !== mainPattern.pathPrefix;
+        }).length;
+
+        if (pathMismatchCount === block.segments.length) {
+            // ALL segments have different path prefix - strong ad indicator
+            score += 5.0;
+        }
+    }
+
+    return score;
+}
+
+/**
+ * Threshold configuration
+ */
+export const THRESHOLDS = {
+    HIGH: 5.0,   // Definitely an ad
+    LOW: 3.0    // Possibly an ad (for future "fuzzy" mode)
+};
+
+/**
+ * Determine if a block should be filtered based on its score
+ */
+export function shouldFilterBlock(score: number, threshold: number = THRESHOLDS.HIGH): boolean {
+    return score >= threshold;
+}

lib/utils/m3u8-utils.ts

@@ -0,0 +1,162 @@
+
+/**
+ * Utility functions for M3U8 playlist manipulation
+ */
+
+import { parseBlocks, learnMainPattern, scoreBlock, shouldFilterBlock } from './m3u8-ad-detector';
+
+/**
+ * Filters ads from specific M3U8 content using multiple detection strategies:
+ * 1. Keyword matching (configurable via env)
+ * 2. CUE-OUT/CUE-IN standard tags
+ * 3. Heuristic block analysis (filename patterns, ad path keywords)
+ * 
+ * Also converts relative URLs to absolute URLs for Blob playback.
+ * 
+ * @param content The raw M3U8 content string
+ * @param baseUrl The base URL of the M3U8 file (to resolve relative paths)
+ * @returns The filtered M3U8 content
+ */
+export type AdFilterMode = 'off' | 'keyword' | 'heuristic' | 'aggressive';
+
+export function filterM3u8Ad(content: string, baseUrl: string, mode: AdFilterMode = 'heuristic', customKeywords: string[] = []): string {
+    if (!content) return '';
+
+    // 1. Performance: Parse Keywords & Base URL ONLY ONCE
+    // Merge env keywords (build time) with custom keywords (runtime injected)
+    const envKeywordsStr = process.env.NEXT_PUBLIC_AD_KEYWORDS || '';
+    const envKeywords = envKeywordsStr.split(/[\n,]/).map(k => k.trim()).filter(k => k);
+
+    // Combine and deduplicate
+    const keywords = Array.from(new Set([...envKeywords, ...customKeywords]));
+
+    const basePath = baseUrl.substring(0, baseUrl.lastIndexOf('/') + 1);
+    let origin = '';
+    try {
+        origin = new URL(baseUrl).origin;
+    } catch (e) { /* ignore */ }
+
+    // 2. Global Scan: Check if any ad keywords exist in the content
+    const hasKeywordMatch = mode !== 'off' && keywords.some(k => content.includes(k));
+    const hasCueTag = mode !== 'off' && (content.includes('#EXT-X-CUE-OUT') || content.includes('#EXT-X-CUE-IN'));
+
+    // 3. Heuristic Analysis: If no explicit ad signals, use block-based detection
+    const lines = content.split(/\r?\n/);
+    let adLineIndices = new Set<number>();
+
+    if (!hasCueTag && (mode === 'heuristic' || mode === 'aggressive')) {
+        // No obvious ad signals - run heuristic analysis
+        const blocks = parseBlocks(lines);
+        if (blocks.length > 1) {
+            const mainPattern = learnMainPattern(blocks);
+            for (const block of blocks) {
+                // Pass all keywords (including custom ones) to heuristic scorer
+                const score = scoreBlock(block, mainPattern, keywords);
+                const threshold = mode === 'aggressive' ? 3.0 : 5.0;
+                if (shouldFilterBlock(score, threshold)) {
+                    // Mark all lines in this block for removal
+                    for (const segment of block.segments) {
+                        adLineIndices.add(segment.lineIndex);
+                        adLineIndices.add(segment.lineIndex - 1); // EXTINF line
+                    }
+                }
+            }
+        }
+    }
+
+    const processedLines: string[] = [];
+
+    // State machine for CUE-OUT/CUE-IN tracking
+    let insideCueAdBlock = false;
+
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmedLine = line.trim();
+
+        // Skip lines marked by heuristic analysis
+        if (adLineIndices.has(i)) {
+            continue;
+        }
+
+        // 3. CUE Tag Detection (SCTE-35 Standard)
+        // EXT-X-CUE-OUT marks start of ad, EXT-X-CUE-IN marks end
+        if (mode !== 'off' && trimmedLine.startsWith('#EXT-X-CUE-OUT')) {
+            insideCueAdBlock = true;
+            // Remove preceding DISCONTINUITY if present
+            if (processedLines.length > 0 && processedLines[processedLines.length - 1].trim() === '#EXT-X-DISCONTINUITY') {
+                processedLines.pop();
+            }
+            continue; // Skip the CUE-OUT tag itself
+        }
+
+        if (trimmedLine.startsWith('#EXT-X-CUE-IN')) {
+            insideCueAdBlock = false;
+            // Also skip the next line if it's a DISCONTINUITY (ad block ending marker)
+            if (i + 1 < lines.length && lines[i + 1].trim() === '#EXT-X-DISCONTINUITY') {
+                i++; // Skip the following DISCONTINUITY
+            }
+            continue; // Skip the CUE-IN tag itself
+        }
+
+        // Skip all content inside CUE ad block
+        if (insideCueAdBlock) {
+            continue;
+        }
+
+        // 4. Keyword-based Ad Detection & Backtrack (skip if no keywords configured)
+        if (keywords.length > 0 && hasKeywordMatch && keywords.some(keyword => trimmedLine.includes(keyword))) {
+            // Found Ad: Remove it and backtrack to remove associated metadata
+            while (processedLines.length > 0) {
+                const lastIndex = processedLines.length - 1;
+                const lastLine = processedLines[lastIndex].trim();
+
+                if (lastLine.startsWith('#EXTINF:') || lastLine === '#EXT-X-DISCONTINUITY') {
+                    processedLines.pop();
+                } else {
+                    break;
+                }
+            }
+            continue; // Skip the ad line itself
+        }
+
+        // 5. Discontinuity Handling (Conservative Mode)
+        // Keep all Discontinuity tags by default.
+        // They will ONLY be removed via backtracking when a confirmed ad segment is found.
+        // This prevents false positives on legitimate concatenated streams.
+        if (trimmedLine === '#EXT-X-DISCONTINUITY') {
+            processedLines.push(line);
+            continue;
+        }
+
+        // 6. General Cleanup & URL Normalization
+        if (!trimmedLine || trimmedLine.startsWith('http') || trimmedLine.startsWith('blob:')) {
+            processedLines.push(line);
+            continue;
+        }
+
+        if (trimmedLine.startsWith('#')) {
+            // Handle URI="..." in attributes (e.g. #EXT-X-KEY)
+            if (trimmedLine.includes('URI="')) {
+                processedLines.push(line.replace(/URI="([^"]+)"/g, (match, uri) => {
+                    if (uri.startsWith('http')) return match; // Already absolute
+                    if (uri.startsWith('/')) {
+                        return `URI="${origin}${uri}"`; // Root-relative
+                    }
+                    return `URI="${basePath}${uri}"`; // Path-relative
+                }));
+            } else {
+                processedLines.push(line);
+            }
+            continue;
+        }
+
+        // 7. Resolve Relative URLs (for Blob support)
+        if (trimmedLine.startsWith('/')) {
+            processedLines.push(origin ? `${origin}${trimmedLine}` : trimmedLine);
+        } else {
+            processedLines.push(`${basePath}${trimmedLine}`);
+        }
+    }
+
+    return processedLines.join('\n');
+}