From c1bcb5141aa4fbe1cf167e829b85ba04f6e22b21 Mon Sep 17 00:00:00 2001 From: kuekhaoyang Date: Thu, 16 Apr 2026 17:06:37 +0800 Subject: [PATCH] Fix Android APK PiP capture --- .../main/java/com/kvideo/tv/MainActivity.kt | 66 ++++- .../player/hooks/desktop/android-pip-utils.ts | 60 ++++ .../hooks/desktop/useFullscreenControls.ts | 274 +++++++++++++----- tests/android-pip-utils.test.ts | 76 +++++ 4 files changed, 407 insertions(+), 69 deletions(-) create mode 100644 components/player/hooks/desktop/android-pip-utils.ts create mode 100644 tests/android-pip-utils.test.ts diff --git a/android-tv/app/src/main/java/com/kvideo/tv/MainActivity.kt b/android-tv/app/src/main/java/com/kvideo/tv/MainActivity.kt index a948866..d7cbd23 100644 --- a/android-tv/app/src/main/java/com/kvideo/tv/MainActivity.kt +++ b/android-tv/app/src/main/java/com/kvideo/tv/MainActivity.kt @@ -4,6 +4,7 @@ import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager import android.content.res.Configuration +import android.graphics.Rect import android.net.Uri import android.os.Build import android.os.Bundle @@ -25,6 +26,10 @@ import android.widget.EditText import android.widget.FrameLayout import android.widget.TextView import androidx.activity.ComponentActivity +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.math.roundToInt class MainActivity : ComponentActivity() { @@ -210,6 +215,7 @@ class MainActivity : ComponentActivity() { newConfig: Configuration ) { super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig) + dispatchPictureInPictureChange(isInPictureInPictureMode) if (!isInPictureInPictureMode) { applyImmersiveMode() } @@ -338,30 +344,82 @@ class MainActivity : ComponentActivity() { return packageManager.hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE) } + private fun dispatchPictureInPictureChange(isInPictureInPictureMode: Boolean) { + val js = """ + window.dispatchEvent(new CustomEvent('kvideo-android-pip-change', { + detail: { inPictureInPicture: ${if (isInPictureInPictureMode) "true" else "false"} } + })); + """.trimIndent() + + webView.post { + try { + webView.evaluateJavascript(js, null) + } catch (error: Throwable) { + Log.w(TAG, "Failed to dispatch Picture-in-Picture change event", error) + } + } + } + + private fun createSourceRectHint(left: Int, top: Int, right: Int, bottom: Int): Rect? { + if (right <= left || bottom <= top) { + return null + } + + val density = resources.displayMetrics.density + return Rect( + (left * density).roundToInt(), + (top * density).roundToInt(), + (right * density).roundToInt(), + (bottom * density).roundToInt() + ) + } + private inner class AndroidPlayerBridge { @JavascriptInterface fun isPictureInPictureSupported(): Boolean = this@MainActivity.isPictureInPictureSupported() @JavascriptInterface - fun enterPictureInPicture(width: Int, height: Int): Boolean { + fun enterPictureInPicture( + width: Int, + height: Int, + left: Int, + top: Int, + right: Int, + bottom: Int + ): Boolean { if (!this@MainActivity.isPictureInPictureSupported()) { return false } + val didEnterPiP = AtomicBoolean(false) + val latch = CountDownLatch(1) + runOnUiThread { try { - exitCustomFullscreen() val builder = android.app.PictureInPictureParams.Builder() if (width > 0 && height > 0) { builder.setAspectRatio(Rational(width, height)) } - enterPictureInPictureMode(builder.build()) + createSourceRectHint(left, top, right, bottom)?.let(builder::setSourceRectHint) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + builder.setSeamlessResizeEnabled(true) + } + didEnterPiP.set(enterPictureInPictureMode(builder.build())) } catch (error: IllegalStateException) { Log.w(TAG, "Failed to enter Picture-in-Picture mode", error) + } finally { + latch.countDown() } } - return true + return try { + latch.await(1500, TimeUnit.MILLISECONDS) + didEnterPiP.get() + } catch (error: InterruptedException) { + Thread.currentThread().interrupt() + Log.w(TAG, "Interrupted while waiting for Picture-in-Picture result", error) + false + } } } } diff --git a/components/player/hooks/desktop/android-pip-utils.ts b/components/player/hooks/desktop/android-pip-utils.ts new file mode 100644 index 0000000..1655902 --- /dev/null +++ b/components/player/hooks/desktop/android-pip-utils.ts @@ -0,0 +1,60 @@ +import type { FullscreenMode } from '../useDesktopPlayerState'; + +export interface AndroidPiPTransitionPlan { + enterTemporaryWindowFullscreen: boolean; + restoreInlineOnExit: boolean; +} + +export interface AndroidPiPSessionState { + enteredTemporaryWindowFullscreen: boolean; + restoreInlineOnExit: boolean; +} + +export interface AndroidPiPSourceRect { + left: number; + top: number; + right: number; + bottom: number; +} + +export function createAndroidPiPTransitionPlan(fullscreenMode: FullscreenMode): AndroidPiPTransitionPlan { + const enterTemporaryWindowFullscreen = fullscreenMode !== 'window'; + + return { + enterTemporaryWindowFullscreen, + restoreInlineOnExit: enterTemporaryWindowFullscreen, + }; +} + +export function shouldRestoreInlineAfterAndroidPiP( + session: AndroidPiPSessionState | null, + inPictureInPicture: boolean +): boolean { + if (!session || inPictureInPicture) { + return false; + } + + return session.enteredTemporaryWindowFullscreen && session.restoreInlineOnExit; +} + +export function shouldRollbackTemporaryWindowFullscreen(session: AndroidPiPSessionState | null): boolean { + return Boolean(session?.enteredTemporaryWindowFullscreen); +} + +export function getAndroidPiPSourceRect(container: HTMLElement | null): AndroidPiPSourceRect | null { + if (!container) { + return null; + } + + const rect = container.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) { + return null; + } + + return { + left: Math.max(0, Math.round(rect.left)), + top: Math.max(0, Math.round(rect.top)), + right: Math.max(0, Math.round(rect.right)), + bottom: Math.max(0, Math.round(rect.bottom)), + }; +} diff --git a/components/player/hooks/desktop/useFullscreenControls.ts b/components/player/hooks/desktop/useFullscreenControls.ts index 2ca92b1..c4c317d 100644 --- a/components/player/hooks/desktop/useFullscreenControls.ts +++ b/components/player/hooks/desktop/useFullscreenControls.ts @@ -1,9 +1,75 @@ -import { useCallback, useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useRef } from 'react'; import type { FullscreenMode } from '../useDesktopPlayerState'; +import { + createAndroidPiPTransitionPlan, + getAndroidPiPSourceRect, + shouldRestoreInlineAfterAndroidPiP, + shouldRollbackTemporaryWindowFullscreen, + type AndroidPiPSessionState, +} from './android-pip-utils'; interface AndroidPiPBridge { isPictureInPictureSupported?: () => boolean; - enterPictureInPicture?: (width: number, height: number) => boolean; + enterPictureInPicture?: ( + width: number, + height: number, + left: number, + top: number, + right: number, + bottom: number + ) => boolean; +} + +interface AndroidPiPChangeDetail { + inPictureInPicture?: boolean; +} + +type OrientationCapableScreen = Screen & { + orientation?: ScreenOrientation & { + lock?: (orientation: string) => Promise; + unlock?: () => void; + }; +}; + +type FullscreenCapableDocument = Document & { + webkitFullscreenElement?: Element | null; + mozFullScreenElement?: Element | null; + msFullscreenElement?: Element | null; + webkitExitFullscreen?: () => Promise; + mozCancelFullScreen?: () => Promise; + msExitFullscreen?: () => Promise; + pictureInPictureEnabled?: boolean; + pictureInPictureElement?: Element | null; + exitPictureInPicture?: () => Promise; +}; + +type FullscreenCapableElement = HTMLDivElement & { + webkitRequestFullscreen?: () => Promise; + mozRequestFullScreen?: () => Promise; + msRequestFullscreen?: () => Promise; +}; + +type PiPCapableVideoElement = HTMLVideoElement & { + webkitEnterFullscreen?: () => void; + webkitSupportsPresentationMode?: (mode: 'picture-in-picture') => boolean; + webkitPresentationMode?: 'inline' | 'picture-in-picture' | string; + webkitSetPresentationMode?: (mode: 'inline' | 'picture-in-picture') => void; + webkitShowPlaybackTargetPicker?: () => void; +}; + +function getFullscreenDocument(): FullscreenCapableDocument { + return document as FullscreenCapableDocument; +} + +function waitForAnimationFrame(): Promise { + return new Promise((resolve) => { + window.requestAnimationFrame(() => resolve()); + }); +} + +async function waitForWindowFullscreenLayout(): Promise { + await waitForAnimationFrame(); + await waitForAnimationFrame(); } interface UseFullscreenControlsProps { @@ -31,10 +97,13 @@ export function useFullscreenControls({ setIsAirPlaySupported, fullscreenType = 'native' }: UseFullscreenControlsProps) { + const androidPiPSessionRef = useRef(null); + const lockLandscape = useCallback(async () => { - if (window.screen && (window.screen as any).orientation && (window.screen as any).orientation.lock) { + const orientation = (window.screen as OrientationCapableScreen).orientation; + if (orientation?.lock) { try { - await (window.screen as any).orientation.lock('landscape'); + await orientation.lock('landscape'); } catch (error) { console.warn('Orientation lock failed:', error); } @@ -42,21 +111,25 @@ export function useFullscreenControls({ }, []); const unlockOrientation = useCallback(() => { - if (window.screen && (window.screen as any).orientation && (window.screen as any).orientation.unlock) { + const orientation = (window.screen as OrientationCapableScreen).orientation; + if (orientation?.unlock) { try { - (window.screen as any).orientation.unlock(); + orientation.unlock(); } catch { // Ignore unlock errors from unsupported browsers. } } }, []); - const getNativeFullscreenElement = useCallback(() => ( - document.fullscreenElement || - (document as any).webkitFullscreenElement || - (document as any).mozFullScreenElement || - (document as any).msFullscreenElement - ), []); + const getNativeFullscreenElement = useCallback(() => { + const fullscreenDocument = getFullscreenDocument(); + return ( + fullscreenDocument.fullscreenElement || + fullscreenDocument.webkitFullscreenElement || + fullscreenDocument.mozFullScreenElement || + fullscreenDocument.msFullscreenElement + ); + }, []); const getAndroidPiPBridge = useCallback((): AndroidPiPBridge | null => { if (typeof window === 'undefined') return null; @@ -67,31 +140,15 @@ export function useFullscreenControls({ return bridge; }, []); - const requestAndroidPictureInPicture = useCallback(() => { - const bridge = getAndroidPiPBridge(); - const video = videoRef.current; - if (!bridge || !video || typeof bridge.enterPictureInPicture !== 'function') { - return false; - } - - const width = video.videoWidth || containerRef.current?.clientWidth || 16; - const height = video.videoHeight || containerRef.current?.clientHeight || 9; - - try { - return bridge.enterPictureInPicture(width, height) !== false; - } catch (error) { - console.error('Android Picture-in-Picture bridge failed:', error); - return false; - } - }, [containerRef, getAndroidPiPBridge, videoRef]); - useEffect(() => { if (typeof document !== 'undefined') { - const hasNativePiP = Boolean((document as any).pictureInPictureEnabled); - const hasWebkitPiP = videoRef.current && ( - 'webkitSupportsPresentationMode' in (videoRef.current as any) || - 'webkitPresentationMode' in (videoRef.current as any) - ); + const fullscreenDocument = getFullscreenDocument(); + const video = videoRef.current as PiPCapableVideoElement | null; + const hasNativePiP = Boolean(fullscreenDocument.pictureInPictureEnabled); + const hasWebkitPiP = Boolean(video && ( + typeof video.webkitSupportsPresentationMode === 'function' || + typeof video.webkitPresentationMode === 'string' + )); const androidBridge = getAndroidPiPBridge(); const hasAndroidPiPBridge = Boolean(androidBridge?.isPictureInPictureSupported?.()); setIsPiPSupported(hasNativePiP || !!hasWebkitPiP || hasAndroidPiPBridge); @@ -102,15 +159,17 @@ export function useFullscreenControls({ }, [getAndroidPiPBridge, setIsPiPSupported, setIsAirPlaySupported, videoRef]); const exitNativeFullscreen = useCallback(async () => { + const fullscreenDocument = getFullscreenDocument(); + try { - if (document.exitFullscreen) { - await document.exitFullscreen(); - } else if ((document as any).webkitExitFullscreen) { - await (document as any).webkitExitFullscreen(); - } else if ((document as any).mozCancelFullScreen) { - await (document as any).mozCancelFullScreen(); - } else if ((document as any).msExitFullscreen) { - await (document as any).msExitFullscreen(); + if (fullscreenDocument.exitFullscreen) { + await fullscreenDocument.exitFullscreen(); + } else if (fullscreenDocument.webkitExitFullscreen) { + await fullscreenDocument.webkitExitFullscreen(); + } else if (fullscreenDocument.mozCancelFullScreen) { + await fullscreenDocument.mozCancelFullScreen(); + } else if (fullscreenDocument.msExitFullscreen) { + await fullscreenDocument.msExitFullscreen(); } } catch (error) { console.error('Failed to exit fullscreen:', error); @@ -138,23 +197,25 @@ export function useFullscreenControls({ }, [exitNativeFullscreen, fullscreenMode, lockLandscape, setFullscreenMode, setIsFullscreen]); const enterNativeFullscreen = useCallback(async () => { - if (!containerRef.current) return; + const container = containerRef.current as FullscreenCapableElement | null; + const video = videoRef.current as PiPCapableVideoElement | null; + if (!container) return; if (fullscreenMode === 'window') { exitWindowFullscreen(); } try { - if (containerRef.current.requestFullscreen) { - await containerRef.current.requestFullscreen(); - } else if ((containerRef.current as any).webkitRequestFullscreen) { - await (containerRef.current as any).webkitRequestFullscreen(); - } else if ((containerRef.current as any).mozRequestFullScreen) { - await (containerRef.current as any).mozRequestFullScreen(); - } else if ((containerRef.current as any).msRequestFullscreen) { - await (containerRef.current as any).msRequestFullscreen(); - } else if (videoRef.current && (videoRef.current as any).webkitEnterFullscreen) { - (videoRef.current as any).webkitEnterFullscreen(); + if (container.requestFullscreen) { + await container.requestFullscreen(); + } else if (container.webkitRequestFullscreen) { + await container.webkitRequestFullscreen(); + } else if (container.mozRequestFullScreen) { + await container.mozRequestFullScreen(); + } else if (container.msRequestFullscreen) { + await container.msRequestFullscreen(); + } else if (video?.webkitEnterFullscreen) { + video.webkitEnterFullscreen(); } setFullscreenMode('native'); @@ -162,9 +223,9 @@ export function useFullscreenControls({ await lockLandscape(); } catch (error) { console.warn('Fullscreen request failed, trying fallback:', error); - if (videoRef.current && (videoRef.current as any).webkitEnterFullscreen) { + if (video?.webkitEnterFullscreen) { try { - (videoRef.current as any).webkitEnterFullscreen(); + video.webkitEnterFullscreen(); setFullscreenMode('native'); setIsFullscreen(true); } catch (fallbackError) { @@ -182,6 +243,65 @@ export function useFullscreenControls({ videoRef, ]); + const requestAndroidPictureInPicture = useCallback(async () => { + const bridge = getAndroidPiPBridge(); + const video = videoRef.current; + if (!bridge || !video || typeof bridge.enterPictureInPicture !== 'function') { + return false; + } + + const plan = createAndroidPiPTransitionPlan(fullscreenMode); + const session: AndroidPiPSessionState = { + enteredTemporaryWindowFullscreen: plan.enterTemporaryWindowFullscreen, + restoreInlineOnExit: plan.restoreInlineOnExit, + }; + + try { + if (plan.enterTemporaryWindowFullscreen) { + await enterWindowFullscreen(); + await waitForWindowFullscreenLayout(); + } + + const width = video.videoWidth || containerRef.current?.clientWidth || 16; + const height = video.videoHeight || containerRef.current?.clientHeight || 9; + const sourceRect = getAndroidPiPSourceRect(containerRef.current); + + androidPiPSessionRef.current = session; + + const didEnterPiP = bridge.enterPictureInPicture( + width, + height, + sourceRect?.left ?? 0, + sourceRect?.top ?? 0, + sourceRect?.right ?? 0, + sourceRect?.bottom ?? 0 + ) !== false; + + if (!didEnterPiP) { + if (shouldRollbackTemporaryWindowFullscreen(session)) { + exitWindowFullscreen(); + } + androidPiPSessionRef.current = null; + } + + return didEnterPiP; + } catch (error) { + if (shouldRollbackTemporaryWindowFullscreen(session)) { + exitWindowFullscreen(); + } + androidPiPSessionRef.current = null; + console.error('Android Picture-in-Picture bridge failed:', error); + return false; + } + }, [ + containerRef, + enterWindowFullscreen, + exitWindowFullscreen, + fullscreenMode, + getAndroidPiPBridge, + videoRef, + ]); + const toggleWindowFullscreen = useCallback(async () => { if (fullscreenMode === 'window') { exitWindowFullscreen(); @@ -285,23 +405,47 @@ export function useFullscreenControls({ return () => window.removeEventListener('keydown', handleEsc); }, [exitWindowFullscreen, fullscreenMode]); + useEffect(() => { + if (typeof window === 'undefined') return; + + const handleAndroidPiPChange = (event: Event) => { + const detail = (event as CustomEvent).detail; + const inPictureInPicture = Boolean(detail?.inPictureInPicture); + + if (shouldRestoreInlineAfterAndroidPiP(androidPiPSessionRef.current, inPictureInPicture)) { + exitWindowFullscreen(); + } + + if (!inPictureInPicture) { + androidPiPSessionRef.current = null; + } + }; + + window.addEventListener('kvideo-android-pip-change', handleAndroidPiPChange); + return () => window.removeEventListener('kvideo-android-pip-change', handleAndroidPiPChange); + }, [exitWindowFullscreen]); + const togglePictureInPicture = useCallback(async () => { if (!videoRef.current || !isPiPSupported) return; - const video = videoRef.current as any; + const video = videoRef.current as PiPCapableVideoElement; + const fullscreenDocument = getFullscreenDocument(); try { - if (document.pictureInPictureElement) { - await document.exitPictureInPicture(); + if (fullscreenDocument.pictureInPictureElement && fullscreenDocument.exitPictureInPicture) { + await fullscreenDocument.exitPictureInPicture(); } else if (video.webkitPresentationMode === 'picture-in-picture') { - video.webkitSetPresentationMode('inline'); - } else if (video.requestPictureInPicture && (document as any).pictureInPictureEnabled) { + video.webkitSetPresentationMode?.('inline'); + } else if (video.requestPictureInPicture && fullscreenDocument.pictureInPictureEnabled) { await video.requestPictureInPicture(); - } else if (requestAndroidPictureInPicture()) { + } else if (await requestAndroidPictureInPicture()) { return; - } else if (video.webkitSupportsPresentationMode && video.webkitSupportsPresentationMode('picture-in-picture')) { + } else if ( + video.webkitSupportsPresentationMode?.('picture-in-picture') && + video.webkitSetPresentationMode + ) { video.webkitSetPresentationMode('picture-in-picture'); } } catch (error) { - if (requestAndroidPictureInPicture()) { + if (await requestAndroidPictureInPicture()) { return; } console.error('Failed to toggle Picture-in-Picture:', error); @@ -310,7 +454,7 @@ export function useFullscreenControls({ const showAirPlayMenu = useCallback(() => { if (!videoRef.current || !isAirPlaySupported) return; - const video = videoRef.current as any; + const video = videoRef.current as PiPCapableVideoElement; if (video.webkitShowPlaybackTargetPicker) { video.webkitShowPlaybackTargetPicker(); } diff --git a/tests/android-pip-utils.test.ts b/tests/android-pip-utils.test.ts new file mode 100644 index 0000000..5d62578 --- /dev/null +++ b/tests/android-pip-utils.test.ts @@ -0,0 +1,76 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + createAndroidPiPTransitionPlan, + shouldRestoreInlineAfterAndroidPiP, + shouldRollbackTemporaryWindowFullscreen, +} from '@/components/player/hooks/desktop/android-pip-utils'; + +test('inline playback enters temporary window fullscreen and restores inline after PiP closes', () => { + const plan = createAndroidPiPTransitionPlan('none'); + + assert.deepEqual(plan, { + enterTemporaryWindowFullscreen: true, + restoreInlineOnExit: true, + }); + + assert.equal( + shouldRestoreInlineAfterAndroidPiP({ + enteredTemporaryWindowFullscreen: plan.enterTemporaryWindowFullscreen, + restoreInlineOnExit: plan.restoreInlineOnExit, + }, false), + true + ); +}); + +test('existing page fullscreen stays in page fullscreen after PiP closes', () => { + const plan = createAndroidPiPTransitionPlan('window'); + + assert.deepEqual(plan, { + enterTemporaryWindowFullscreen: false, + restoreInlineOnExit: false, + }); + + assert.equal( + shouldRestoreInlineAfterAndroidPiP({ + enteredTemporaryWindowFullscreen: plan.enterTemporaryWindowFullscreen, + restoreInlineOnExit: plan.restoreInlineOnExit, + }, false), + false + ); +}); + +test('system fullscreen also restores inline after PiP closes', () => { + const plan = createAndroidPiPTransitionPlan('native'); + + assert.deepEqual(plan, { + enterTemporaryWindowFullscreen: true, + restoreInlineOnExit: true, + }); + + assert.equal( + shouldRestoreInlineAfterAndroidPiP({ + enteredTemporaryWindowFullscreen: plan.enterTemporaryWindowFullscreen, + restoreInlineOnExit: plan.restoreInlineOnExit, + }, false), + true + ); +}); + +test('temporary page fullscreen rolls back when Android PiP entry fails', () => { + assert.equal( + shouldRollbackTemporaryWindowFullscreen({ + enteredTemporaryWindowFullscreen: true, + restoreInlineOnExit: true, + }), + true + ); + + assert.equal( + shouldRollbackTemporaryWindowFullscreen({ + enteredTemporaryWindowFullscreen: false, + restoreInlineOnExit: false, + }), + false + ); +});