fix issue 210 slow source search reliability

lib/utils/latency.ts

@@ -10,6 +10,46 @@ interface LatencyInfo {
   level: 'excellent' | 'good' | 'fair' | 'slow';
 }
 
+export interface LatencyProbeTarget {
+  id: string;
+  baseUrl: string;
+}
+
+export interface LatencyProbeResult {
+  id: string;
+  latency: number | null;
+}
+
+export async function probeLatencyTargets(
+  targets: LatencyProbeTarget[],
+  probe: (target: LatencyProbeTarget) => Promise<number | null>,
+  concurrency: number = 4,
+): Promise<LatencyProbeResult[]> {
+  if (targets.length === 0) return [];
+
+  const results = new Array<LatencyProbeResult>(targets.length);
+  const workerCount = Math.min(
+    targets.length,
+    Math.max(1, Math.floor(concurrency)),
+  );
+  let nextIndex = 0;
+
+  const workers = Array.from({ length: workerCount }, async () => {
+    while (nextIndex < targets.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      const target = targets[index];
+      results[index] = {
+        id: target.id,
+        latency: await probe(target),
+      };
+    }
+  });
+
+  await Promise.all(workers);
+  return results;
+}
+
 /**
  * Get latency information with color coding
  * @param latency - Response time in milliseconds

lib/utils/search-stream.ts

@@ -35,80 +35,65 @@ export async function processSearchStream({
 }: StreamHandlerParams) {
     const decoder = new TextDecoder();
     let buffer = '';
-    let timeoutId: NodeJS.Timeout | null = null;
     let isCompleted = false;
 
-    // Auto-complete if no progress for 3 seconds
-    const resetTimeout = () => {
-        if (timeoutId) clearTimeout(timeoutId);
+    const blockedCategories = settingsStore.getSettings().blockedCategories;
 
-        timeoutId = setTimeout(() => {
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
             if (!isCompleted) {
-
                 isCompleted = true;
                 onComplete();
             }
-        }, 3000);
-    };
+            break;
+        }
 
-    try {
-        resetTimeout(); // Start initial timeout
-        const blockedCategories = settingsStore.getSettings().blockedCategories;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
 
-        while (true) {
-            const { done, value } = await reader.read();
-            if (done) break;
+        for (const line of lines) {
+            if (!line.startsWith('data: ')) continue;
 
-            buffer += decoder.decode(value, { stream: true });
-            const lines = buffer.split('\n');
-            buffer = lines.pop() || '';
+            try {
+                const data = JSON.parse(line.slice(6));
 
-            for (const line of lines) {
-                if (!line.startsWith('data: ')) continue;
-
-                try {
-                    const data = JSON.parse(line.slice(6));
-
-                    if (data.type === 'start') {
-                        onStart(data.totalSources);
-                        resetTimeout();
-                    } else if (data.type === 'videos') {
-                        const newVideos: Video[] = data.videos
-                            .filter((video: any) => hasMinimumMatch(video.vod_name, currentQuery))
-                            .filter((video: any) => !isCategoryBlocked(video, blockedCategories))
-                            .map((video: any) => ({
-                                ...video,
-                                sourceName: video.sourceDisplayName || getSourceName(video.source),
-                                isNew: true,
-                                relevanceScore: calculateRelevanceScore(video, currentQuery),
-                            }));
-                        onVideos(newVideos, data.source);
-                        if (data.pagecount && onPageInfo) {
-                            onPageInfo(data.pagecount);
-                        }
-                        resetTimeout();
-                    } else if (data.type === 'progress') {
-                        onProgress(data.completedSources, data.totalVideosFound);
-                        resetTimeout();
-                    } else if (data.type === 'complete') {
-                        if (timeoutId) clearTimeout(timeoutId);
+                if (data.type === 'start') {
+                    onStart(data.totalSources);
+                } else if (data.type === 'videos') {
+                    const newVideos: Video[] = data.videos
+                        .filter((video: any) => hasMinimumMatch(video.vod_name, currentQuery))
+                        .filter((video: any) => !isCategoryBlocked(video, blockedCategories))
+                        .map((video: any) => ({
+                            ...video,
+                            sourceName: video.sourceDisplayName || getSourceName(video.source),
+                            isNew: true,
+                            relevanceScore: calculateRelevanceScore(video, currentQuery),
+                        }));
+                    onVideos(newVideos, data.source);
+                    if (data.pagecount && onPageInfo) {
+                        onPageInfo(data.pagecount);
+                    }
+                } else if (data.type === 'progress') {
+                    onProgress(data.completedSources, data.totalVideosFound);
+                } else if (data.type === 'complete') {
+                    if (!isCompleted) {
                         isCompleted = true;
                         if (data.maxPageCount && onPageInfo) {
                             onPageInfo(data.maxPageCount);
                         }
                         onComplete();
-                    } else if (data.type === 'error') {
+                    }
+                } else if (data.type === 'error') {
+                    if (!isCompleted) {
+                        isCompleted = true;
                         onError(data.message);
                     }
-                } catch (error) {
-                    console.error('Error parsing stream data:', error);
                 }
+            } catch (error) {
+                console.error('Error parsing stream data:', error);
             }
         }
-    } catch (error) {
-        if (timeoutId) clearTimeout(timeoutId);
-        throw error;
-    } finally {
-        if (timeoutId) clearTimeout(timeoutId);
     }
 }