Implement streaming search API and enhance video availability checks with loading animation

2026-08-22 12:13:43 +08:00 · 2025-11-16 19:03:19 +08:00
parent 610cc522c7
commit d0790d1307
6 changed files with 664 additions and 149 deletions
@@ -0,0 +1,172 @@
+/**
+ * Streaming Search API Route
+ * Returns results progressively as they become available
+ */
+
+import { NextRequest } from 'next/server';
+import { searchVideos } from '@/lib/api/client';
+import { getSourceById } from '@/lib/api/video-sources';
+import { checkVideoAvailability } from '@/lib/utils/source-checker';
+
+export async function POST(request: NextRequest) {
+  const encoder = new TextEncoder();
+  
+  const stream = new ReadableStream({
+    async start(controller) {
+      try {
+        const body = await request.json();
+        const { query, sources: sourceIds, page = 1 } = body;
+
+        // Validate input
+        if (!query || typeof query !== 'string' || query.trim().length === 0) {
+          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: 'Invalid query' })}\n\n`));
+          controller.close();
+          return;
+        }
+
+        // Get source configurations
+        const sources = sourceIds
+          .map((id: string) => getSourceById(id))
+          .filter((source: any): source is NonNullable<typeof source> => source !== undefined);
+
+        if (sources.length === 0) {
+          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: 'No valid sources' })}\n\n`));
+          controller.close();
+          return;
+        }
+
+        // Send progress: searching sources
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+          type: 'progress', 
+          stage: 'searching',
+          checkedSources: 0,
+          totalSources: sourceIds.length
+        })}\n\n`));
+
+        // Perform search with progress tracking for each source
+        let checkedSourcesCount = 0;
+        const searchResults = await Promise.all(
+          sources.map(async (source: any) => {
+            try {
+              const result = await searchVideos(query.trim(), [source], page);
+              checkedSourcesCount++;
+              
+              // Send progress update after each source completes
+              controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+                type: 'progress', 
+                stage: 'searching',
+                checkedSources: checkedSourcesCount,
+                totalSources: sourceIds.length
+              })}\n\n`));
+              
+              return result[0];
+            } catch (error) {
+              checkedSourcesCount++;
+              
+              // Still send progress even on error
+              controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+                type: 'progress', 
+                stage: 'searching',
+                checkedSources: checkedSourcesCount,
+                totalSources: sourceIds.length
+              })}\n\n`));
+              
+              return {
+                results: [],
+                source: source.id,
+                error: error instanceof Error ? error.message : 'Unknown error',
+              };
+            }
+          })
+        );
+
+        // Get all videos from all sources
+        const allVideos = searchResults.flatMap(r => r.results);
+        
+        if (allVideos.length === 0) {
+          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+            type: 'complete',
+            totalResults: 0
+          })}\n\n`));
+          controller.close();
+          return;
+        }
+
+        // Send progress: start checking videos
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+          type: 'progress', 
+          stage: 'checking',
+          checkedVideos: 0,
+          totalVideos: allVideos.length
+        })}\n\n`));
+
+        const availableVideos: any[] = [];
+        let checkedCount = 0;
+        const concurrency = 5; // Smaller batches for faster response
+
+        // Process videos in smaller batches for immediate feedback
+        for (let i = 0; i < allVideos.length; i += concurrency) {
+          const batch = allVideos.slice(i, i + concurrency);
+          
+          const results = await Promise.all(
+            batch.map(async (video) => {
+              const isAvailable = await checkVideoAvailability(video);
+              return isAvailable ? video : null;
+            })
+          );
+
+          // Add available videos
+          const newAvailableVideos = results.filter(v => v !== null);
+          availableVideos.push(...newAvailableVideos);
+          
+          checkedCount += batch.length;
+
+          // ALWAYS send update after each batch (even if no new videos)
+          if (newAvailableVideos.length > 0) {
+            // Send new videos immediately
+            controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+              type: 'videos',
+              videos: newAvailableVideos,
+              checkedVideos: checkedCount,
+              totalVideos: allVideos.length,
+              availableCount: availableVideos.length
+            })}\n\n`));
+          }
+          
+          // Always send progress update
+          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+            type: 'progress',
+            stage: 'checking',
+            checkedVideos: checkedCount,
+            totalVideos: allVideos.length,
+            availableCount: availableVideos.length
+          })}\n\n`));
+        }
+
+        // Send completion
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+          type: 'complete',
+          totalResults: availableVideos.length,
+          checkedVideos: allVideos.length,
+          totalVideos: allVideos.length
+        })}\n\n`));
+
+        controller.close();
+      } catch (error) {
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify({ 
+          type: 'error',
+          error: error instanceof Error ? error.message : 'Unknown error'
+        })}\n\n`));
+        controller.close();
+      }
+    },
+  });
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+    },
+  });
+}
@@ -7,7 +7,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { searchVideos } from '@/lib/api/client';
 import { getEnabledSources, getSourceById } from '@/lib/api/video-sources';
-import { checkMultipleSources, filterByAvailableSources } from '@/lib/utils/source-checker';
+import { checkMultipleVideos } from '@/lib/utils/source-checker';
 import type { SearchRequest, SearchResult } from '@/lib/types';

 export async function POST(request: NextRequest) {
@@ -68,53 +68,46 @@ export async function POST(request: NextRequest) {
      return sourceNames[sourceId] || sourceId;
    };

-    // Check source availability by testing sample videos
-    console.log(`🔍 Checking availability of ${searchResults.length} sources...`);
-    const sourcesWithVideos = searchResults
-      .filter(result => result.results.length > 0)
-      .map(result => ({
-        sourceId: result.source,
-        sourceName: getSourceName(result.source),
-        videos: result.results.slice(0, 3), // Use first 3 videos as samples
-      }));
-
-    const availabilityResults = await checkMultipleSources(sourcesWithVideos);
-    
-    const availableCount = availabilityResults.filter(r => r.isAvailable).length;
-    console.log(`✅ ${availableCount} out of ${availabilityResults.length} sources are available`);
-
-    // Filter results to only include videos from available sources
+    // Get all videos from all sources
    const allVideos = searchResults.flatMap(r => r.results);
-    const availableVideos = filterByAvailableSources(allVideos, availabilityResults);

-    // Group available videos back by source
-    const availableSources = availabilityResults
-      .filter(r => r.isAvailable)
-      .map(r => {
-        const sourceVideos = availableVideos.filter(v => v.source === r.sourceId);
-        return {
-          source: r.sourceId,
-          results: sourceVideos,
-          responseTime: searchResults.find(sr => sr.source === r.sourceId)?.responseTime,
-        };
-      });
+    // Check each video individually
+    const availableVideos = await checkMultipleVideos(allVideos, 10);

-    // Format response
-    const response: SearchResult[] = availableSources.map(result => ({
-      results: result.results,
-      source: result.source,
-      responseTime: result.responseTime,
+    // Group available videos by source
+    const videosBySource = new Map<string, any[]>();
+    for (const video of availableVideos) {
+      const sourceId = video.source;
+      if (!videosBySource.has(sourceId)) {
+        videosBySource.set(sourceId, []);
+      }
+      videosBySource.get(sourceId)!.push(video);
+    }
+
+    // Build response with actual video counts per source
+    const response: SearchResult[] = Array.from(videosBySource.entries()).map(([sourceId, videos]) => ({
+      results: videos,
+      source: sourceId,
+      responseTime: searchResults.find(sr => sr.source === sourceId)?.responseTime,
    }));

+    // Calculate source statistics
+    const sourceStats = sourceIds.map(sourceId => {
+      const count = videosBySource.get(sourceId)?.length || 0;
+      return {
+        sourceId,
+        sourceName: getSourceName(sourceId),
+        count,
+      };
+    });
+
    return NextResponse.json({
      success: true,
      query: query.trim(),
      page,
      sources: response,
      totalResults: availableVideos.length,
-      availableSources: availableCount,
-      totalSources: availabilityResults.length,
-      sourceAvailability: availabilityResults,
+      sourceStats, // Include real counts per source
    });
  } catch (error) {
    console.error('Search API error:', error);
@@ -187,53 +180,46 @@ export async function GET(request: NextRequest) {
      return sourceNames[sourceId] || sourceId;
    };

-    // Check source availability by testing sample videos
-    console.log(`🔍 [GET] Checking availability of ${searchResults.length} sources...`);
-    const sourcesWithVideos = searchResults
-      .filter(result => result.results.length > 0)
-      .map(result => ({
-        sourceId: result.source,
-        sourceName: getSourceName(result.source),
-        videos: result.results.slice(0, 3), // Use first 3 videos as samples
-      }));
-
-    const availabilityResults = await checkMultipleSources(sourcesWithVideos);
-    
-    const availableCount = availabilityResults.filter(r => r.isAvailable).length;
-    console.log(`✅ [GET] ${availableCount} out of ${availabilityResults.length} sources are available`);
-
-    // Filter results to only include videos from available sources
+    // Get all videos from all sources
    const allVideos = searchResults.flatMap(r => r.results);
-    const availableVideos = filterByAvailableSources(allVideos, availabilityResults);

-    // Group available videos back by source
-    const availableSources = availabilityResults
-      .filter(r => r.isAvailable)
-      .map(r => {
-        const sourceVideos = availableVideos.filter(v => v.source === r.sourceId);
-        return {
-          source: r.sourceId,
-          results: sourceVideos,
-          responseTime: searchResults.find(sr => sr.source === r.sourceId)?.responseTime,
-        };
-      });
+    // Check each video individually
+    const availableVideos = await checkMultipleVideos(allVideos, 10);

-    // Format response
-    const response: SearchResult[] = availableSources.map(result => ({
-      results: result.results,
-      source: result.source,
-      responseTime: result.responseTime,
+    // Group available videos by source
+    const videosBySource = new Map<string, any[]>();
+    for (const video of availableVideos) {
+      const sourceId = video.source;
+      if (!videosBySource.has(sourceId)) {
+        videosBySource.set(sourceId, []);
+      }
+      videosBySource.get(sourceId)!.push(video);
+    }
+
+    // Build response with actual video counts per source
+    const response: SearchResult[] = Array.from(videosBySource.entries()).map(([sourceId, videos]) => ({
+      results: videos,
+      source: sourceId,
+      responseTime: searchResults.find(sr => sr.source === sourceId)?.responseTime,
    }));

+    // Calculate source statistics
+    const sourceStats = sourceIds.map(sourceId => {
+      const count = videosBySource.get(sourceId)?.length || 0;
+      return {
+        sourceId,
+        sourceName: getSourceName(sourceId),
+        count,
+      };
+    });
+
    return NextResponse.json({
      success: true,
      query: query.trim(),
      page,
      sources: response,
      totalResults: availableVideos.length,
-      availableSources: availableCount,
-      totalSources: availabilityResults.length,
-      sourceAvailability: availabilityResults,
+      sourceStats, // Include real counts per source
    });
  } catch (error) {
    console.error('Search API error:', error);