feat: Implement parallel multi-page fetching from each search source and introduce client-side load more pagination for aggregated results.

app/api/search-parallel/route.ts

@@ -55,41 +55,40 @@ export async function POST(request: NextRequest) {
         // Track progress
         let completedSources = 0;
         let totalVideosFound = 0;
+        let maxPageCount = 1;
 
         // Search all sources in PARALLEL - don't wait for all to finish
         const searchPromises = sources.map(async (source: any) => {
           const startTime = performance.now(); // Track start time
           try {
 
-
-            // Search this source
-            const result = await searchVideos(query.trim(), [source], page);
+            // Search page 1 for this source
+            const result = await searchVideos(query.trim(), [source], 1);
             const endTime = performance.now(); // Track end time
             const latency = Math.round(endTime - startTime); // Calculate latency in ms
             const videos = result[0]?.results || [];
+            const pagecount = result[0]?.pagecount ?? 1;
 
             completedSources++;
             totalVideosFound += videos.length;
 
-
-
-            // Stream videos immediately as they arrive WITH latency data
+            // Stream page 1 videos immediately
             if (videos.length > 0) {
               controller.enqueue(encoder.encode(`data: ${JSON.stringify({
                 type: 'videos',
                 videos: videos.map((video: any) => ({
                   ...video,
                   sourceDisplayName: getSourceName(source.id),
-                  latency, // Add latency to each video
+                  latency,
                 })),
                 source: source.id,
                 completedSources,
                 totalSources: sources.length,
-                latency, // Also include at source level
+                latency,
               })}\n\n`));
             }
 
-            // Send progress update
+            // Send progress update for page 1
             controller.enqueue(encoder.encode(`data: ${JSON.stringify({
               type: 'progress',
               completedSources,
@@ -97,6 +96,47 @@ export async function POST(request: NextRequest) {
               totalVideosFound
             })}\n\n`));
 
+            // Auto-fetch remaining pages if pagecount > 1
+            if (pagecount > 1) {
+              const remainingPages = Array.from({ length: pagecount - 1 }, (_, i) => i + 2);
+              const pagePromises = remainingPages.map(async (pg) => {
+                try {
+                  const pageResult = await searchVideos(query.trim(), [source], pg);
+                  const pageVideos = pageResult[0]?.results || [];
+
+                  totalVideosFound += pageVideos.length;
+
+                  if (pageVideos.length > 0) {
+                    controller.enqueue(encoder.encode(`data: ${JSON.stringify({
+                      type: 'videos',
+                      videos: pageVideos.map((video: any) => ({
+                        ...video,
+                        sourceDisplayName: getSourceName(source.id),
+                        latency,
+                      })),
+                      source: source.id,
+                      completedSources,
+                      totalSources: sources.length,
+                      latency,
+                    })}\n\n`));
+                  }
+
+                  // Progress update for each additional page
+                  controller.enqueue(encoder.encode(`data: ${JSON.stringify({
+                    type: 'progress',
+                    completedSources,
+                    totalSources: sources.length,
+                    totalVideosFound
+                  })}\n\n`));
+
+                } catch (pageError) {
+                  console.error(`[Search Parallel] Source ${source.id} page ${pg} failed:`, pageError);
+                }
+              });
+
+              await Promise.all(pagePromises);
+            }
+
           } catch (error) {
             const endTime = performance.now();
             const latency = Math.round(endTime - startTime);
@@ -122,7 +162,8 @@ export async function POST(request: NextRequest) {
         controller.enqueue(encoder.encode(`data: ${JSON.stringify({
           type: 'complete',
           totalVideosFound,
-          totalSources: sources.length
+          totalSources: sources.length,
+          maxPageCount
         })}\n\n`));
 
         controller.close();

components/home/SearchResults.tsx

@@ -20,7 +20,7 @@ export function SearchResults({
     availableSources,
     loading,
     isPremium = false,
-    latencies = {}
+    latencies = {},
 }: SearchResultsProps) {
     // Source badges hook - filters by video source
     const {
@@ -77,3 +77,5 @@ export function SearchResults({
         </div>
     );
 }
+
+

lib/api/search-api.ts

@@ -11,7 +11,7 @@ async function searchVideosBySource(
     query: string,
     source: VideoSource,
     page: number = 1
-): Promise<{ results: VideoItem[]; source: string; responseTime: number }> {
+): Promise<{ results: VideoItem[]; source: string; responseTime: number; pagecount: number }> {
     const startTime = Date.now();
 
     const url = new URL(`${source.baseUrl}${source.searchPath}`);
@@ -51,6 +51,7 @@ async function searchVideosBySource(
             results,
             source: source.id,
             responseTime: Date.now() - startTime,
+            pagecount: data.pagecount ?? 1,
         };
     } catch (error) {
         console.error(`Search failed for source ${source.name}:`, error);
@@ -71,7 +72,7 @@ export async function searchVideos(
     query: string,
     sources: VideoSource[],
     page: number = 1
-): Promise<Array<{ results: VideoItem[]; source: string; responseTime?: number; error?: string }>> {
+): Promise<Array<{ results: VideoItem[]; source: string; responseTime?: number; pagecount?: number; error?: string }>> {
     const searchPromises = sources.map(async source => {
         try {
             return await searchVideosBySource(query, source, page);

lib/hooks/useHomePage.ts

@@ -33,6 +33,9 @@ export function useHomePage() {
         resetSearch,
         loadCachedResults,
         applySorting,
+        loadMore,
+        hasMore,
+        loadingMore,
     } = useParallelSearch(
         saveToCache,
         onUrlUpdate
@@ -152,5 +155,8 @@ export function useHomePage() {
         totalSources,
         handleSearch,
         handleReset,
+        loadMore,
+        hasMore,
+        loadingMore,
     };
 }

lib/hooks/useParallelSearch.ts

@@ -18,6 +18,9 @@ interface ParallelSearchResult {
   resetSearch: () => void;
   loadCachedResults: (results: Video[], sources: any[]) => void;
   applySorting: (sortBy: SortOption) => void;
+  loadMore: () => Promise<void>;
+  hasMore: boolean;
+  loadingMore: boolean;
 }
 
 export function useParallelSearch(
@@ -32,18 +35,23 @@ export function useParallelSearch(
     completedSources,
     totalSources,
     totalVideosFound,
+    currentPage,
+    maxPageCount,
+    loadingMore,
     setResults,
     setAvailableSources,
     setTotalVideosFound,
     resetState,
   } = state;
 
-  const { performSearch, cancelSearch } = useSearchAction({
+  const { performSearch, loadMore: loadMoreAction, cancelSearch } = useSearchAction({
     state,
     onCacheUpdate,
     onUrlUpdate,
   });
 
+  const hasMore = currentPage < maxPageCount;
+
   /**
    * Reset search state
    */
@@ -79,6 +87,9 @@ export function useParallelSearch(
     resetSearch,
     loadCachedResults,
     applySorting,
+    loadMore: loadMoreAction,
+    hasMore,
+    loadingMore,
   };
 }
 

lib/hooks/usePremiumHomePage.ts

@@ -36,6 +36,9 @@ export function usePremiumHomePage() {
         resetSearch,
         loadCachedResults,
         applySorting,
+        loadMore,
+        hasMore,
+        loadingMore,
     } = useParallelSearch(
         saveToCache,
         onUrlUpdate
@@ -140,5 +143,8 @@ export function usePremiumHomePage() {
         totalSources,
         handleSearch,
         handleReset,
+        loadMore,
+        hasMore,
+        loadingMore,
     };
 }

lib/hooks/useSearchAction.ts

@@ -24,10 +24,17 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
         setCompletedSources,
         setTotalSources,
         setTotalVideosFound,
+        setCurrentPage,
+        setMaxPageCount,
+        setLoadingMore,
+        currentPage,
+        maxPageCount,
         startSearch,
     } = state;
 
     const abortControllerRef = useRef<AbortController | null>(null);
+    // Keep track of the last search params so loadMore can re-use them
+    const lastSearchParamsRef = useRef<{ query: string; sources: any[]; sortBy: SortOption } | null>(null);
 
     const performSearch = useCallback(async (searchQuery: string, sources: any[] = [], sortBy: SortOption = 'default') => {
         if (!searchQuery.trim()) return;
@@ -39,9 +46,6 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
             targetSources = [
                 ...settings.sources,
                 ...settings.subscriptions.filter(s => (s as any).enabled !== false), // Include valid subscriptions
-                // Maybe check premium settings? For main search, we usually include all enabled.
-                // But typically search implies general search. Premium might be separate?
-                // The prompt for "search" includes all.
             ].filter(s => (s as any).enabled !== false);
         }
 
@@ -54,6 +58,9 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
         // Reset state
         startSearch(searchQuery.trim());
 
+        // Save search params for loadMore
+        lastSearchParamsRef.current = { query: searchQuery.trim(), sources: targetSources, sortBy };
+
         // Update URL
         onUrlUpdate(searchQuery);
 
@@ -61,7 +68,7 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
             const response = await fetch('/api/search-parallel', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({ query: searchQuery, sources: targetSources }),
+                body: JSON.stringify({ query: searchQuery, sources: targetSources, page: 1 }),
                 signal: abortControllerRef.current.signal,
             });
 
@@ -92,6 +99,9 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
                     setCompletedSources(completed);
                     setTotalVideosFound(found);
                 },
+                onPageInfo: (pageCount) => {
+                    setMaxPageCount((prev) => Math.max(prev, pageCount));
+                },
                 onComplete: () => {
                     setLoading(false);
 
@@ -131,7 +141,68 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
             }
             setLoading(false);
         }
-    }, [startSearch, onUrlUpdate, onCacheUpdate, setTotalSources, setResults, setCompletedSources, setTotalVideosFound, setLoading, setAvailableSources]);
+    }, [startSearch, onUrlUpdate, onCacheUpdate, setTotalSources, setResults, setCompletedSources, setTotalVideosFound, setLoading, setAvailableSources, setMaxPageCount]);
+
+    const loadMore = useCallback(async () => {
+        const params = lastSearchParamsRef.current;
+        if (!params) return;
+
+        const nextPage = currentPage + 1;
+        if (nextPage > maxPageCount) return;
+
+        // Abort any ongoing load-more (but not the main search)
+        if (abortControllerRef.current) {
+            abortControllerRef.current.abort();
+        }
+        abortControllerRef.current = new AbortController();
+
+        setLoadingMore(true);
+
+        try {
+            const response = await fetch('/api/search-parallel', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ query: params.query, sources: params.sources, page: nextPage }),
+                signal: abortControllerRef.current.signal,
+            });
+
+            if (!response.ok) throw new Error('Load more failed');
+
+            const reader = response.body?.getReader();
+            if (!reader) throw new Error('No response stream');
+
+            await processSearchStream({
+                reader,
+                currentQuery: params.query,
+                onStart: () => { },
+                onVideos: (newVideos) => {
+                    // Append new videos to existing results
+                    setResults((prev) => binaryInsertVideos(prev, newVideos));
+                },
+                onProgress: (_, found) => {
+                    setTotalVideosFound((prev) => prev + found);
+                },
+                onPageInfo: (pageCount) => {
+                    setMaxPageCount((prev) => Math.max(prev, pageCount));
+                },
+                onComplete: () => {
+                    setCurrentPage(nextPage);
+                    setLoadingMore(false);
+                },
+                onError: (message) => {
+                    console.error('Load more error:', message);
+                    setLoadingMore(false);
+                },
+            });
+
+        } catch (error) {
+            if (error instanceof Error && error.name === 'AbortError') {
+                return;
+            }
+            console.error('Load more error:', error);
+            setLoadingMore(false);
+        }
+    }, [currentPage, maxPageCount, setLoadingMore, setResults, setTotalVideosFound, setCurrentPage, setMaxPageCount]);
 
     const cancelSearch = useCallback(() => {
         if (abortControllerRef.current) {
@@ -139,5 +210,5 @@ export function useSearchAction({ state, onCacheUpdate, onUrlUpdate }: UseSearch
         }
     }, []);
 
-    return { performSearch, cancelSearch };
+    return { performSearch, loadMore, cancelSearch };
 }

lib/hooks/useSearchState.ts

@@ -8,6 +8,9 @@ export function useSearchState() {
     const [completedSources, setCompletedSources] = useState(0);
     const [totalSources, setTotalSources] = useState(0);
     const [totalVideosFound, setTotalVideosFound] = useState(0);
+    const [currentPage, setCurrentPage] = useState(1);
+    const [maxPageCount, setMaxPageCount] = useState(1);
+    const [loadingMore, setLoadingMore] = useState(false);
     const currentQueryRef = useRef<string>('');
 
     const resetState = useCallback(() => {
@@ -17,6 +20,9 @@ export function useSearchState() {
         setCompletedSources(0);
         setTotalSources(0);
         setTotalVideosFound(0);
+        setCurrentPage(1);
+        setMaxPageCount(1);
+        setLoadingMore(false);
         currentQueryRef.current = '';
     }, []);
 
@@ -27,6 +33,9 @@ export function useSearchState() {
         setCompletedSources(0);
         setTotalSources(0);
         setTotalVideosFound(0);
+        setCurrentPage(1);
+        setMaxPageCount(1);
+        setLoadingMore(false);
         currentQueryRef.current = query;
     }, []);
 
@@ -43,6 +52,12 @@ export function useSearchState() {
         setTotalSources,
         totalVideosFound,
         setTotalVideosFound,
+        currentPage,
+        setCurrentPage,
+        maxPageCount,
+        setMaxPageCount,
+        loadingMore,
+        setLoadingMore,
         currentQueryRef,
         resetState,
         startSearch,

lib/utils/search-stream.ts

@@ -9,6 +9,7 @@ interface StreamHandlerParams {
     onProgress: (completedSources: number, totalVideosFound: number) => void;
     onComplete: () => void;
     onError: (message: string) => void;
+    onPageInfo?: (maxPageCount: number) => void;
     currentQuery: string;
 }
 
@@ -19,6 +20,7 @@ export async function processSearchStream({
     onProgress,
     onComplete,
     onError,
+    onPageInfo,
     currentQuery,
 }: StreamHandlerParams) {
     const decoder = new TextDecoder();
@@ -69,6 +71,9 @@ export async function processSearchStream({
                                 relevanceScore: calculateRelevanceScore(video, currentQuery),
                             }));
                         onVideos(newVideos, data.source);
+                        if (data.pagecount && onPageInfo) {
+                            onPageInfo(data.pagecount);
+                        }
                         resetTimeout();
                     } else if (data.type === 'progress') {
                         onProgress(data.completedSources, data.totalVideosFound);
@@ -76,6 +81,9 @@ export async function processSearchStream({
                     } else if (data.type === 'complete') {
                         if (timeoutId) clearTimeout(timeoutId);
                         isCompleted = true;
+                        if (data.maxPageCount && onPageInfo) {
+                            onPageInfo(data.maxPageCount);
+                        }
                         onComplete();
                     } else if (data.type === 'error') {
                         onError(data.message);

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "kvideo",
-  "version": "4.1.0",
+  "version": "4.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "kvideo",
-      "version": "4.1.0",
+      "version": "4.1.1",
       "dependencies": {
         "@dnd-kit/core": "^6.3.1",
         "@dnd-kit/sortable": "^10.0.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "kvideo",
-  "version": "4.1.0",
+  "version": "4.1.1",
   "private": true,
   "scripts": {
     "dev": "next dev",