diff --git a/README.md b/README.md index d1f3bb14..a812d8d9 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Install the app on an Android device and open the displayed address in your car' ## Features +- **Multiple-source framework**: A single active projection session routes video, touch and media controls independently of CarPlay. Three diagnostic sources exercise H.264, Surface and framebuffer output; Android Auto and ICCOA adapters are follow-up work. See the [implementation and device test guide](docs/PROJECTION_IMPLEMENTATION.md). - **Browser display and touch control**: CarPlay streaming, touch input, and fullscreen display. - **Selectable streaming modes**: Choose WebRTC video passthrough (H.264 / HEVC) or image compatibility mode in settings, with JPEG / WebP format and image quality from 1 to 100. Image streaming retains JPEG compatibility fallback and overlaps up to two frames while audio and touch keep their existing routes. - **Optional browser audio**: After pairing, the browser can play CarPlay audio. The browser microphone can provide CarPlay voice input through a trusted HTTPS page; Android handles audio and microphone input by default. @@ -82,7 +83,7 @@ Use Android NDK 28.2.13676358 and CMake 3.22.1. The first build fetches pinned v ./gradlew :mobile:assembleDebug ``` -Source-only builds omit CarPlay authentication resources. Web and CarPlay remain disabled until valid resources are configured; successful import automatically starts the Web service. APKs with valid bundled resources work immediately and still allow replacement, with imported resources taking priority. Open **Settings → CarPlay authentication resources** to select `identity.pk8` and `certificate.p7b`, or follow the DiPlay website download link and import the downloaded APK. You can also share that APK with WheelPlay from a file manager; DiPlay does not need to be installed. Imports run locally and replace existing resources only after key/certificate validation. Resources stay in private, backup-excluded app storage and are never uploaded. See the [authentication import guide](docs/AUTHENTICATION_IMPORT.md). +Source-only builds omit CarPlay authentication resources. The Web service and diagnostic projection sources work without these resources; only CarPlay connections require a valid identity. APKs with valid bundled resources work immediately and still allow replacement, with imported resources taking priority. Open **Settings → CarPlay authentication resources** to select `identity.pk8` and `certificate.p7b`, or follow the DiPlay website download link and import the downloaded APK. You can also share that APK with WheelPlay from a file manager; DiPlay does not need to be installed. Imports run locally and replace existing resources only after key/certificate validation. Resources stay in private, backup-excluded app storage and are never uploaded. See the [authentication import guide](docs/AUTHENTICATION_IMPORT.md). Developers may still set `WHEELPLAY_AUTH_ASSETS_DIR` (or `DIPLAY_AUTH_ASSETS_DIR`) to a local asset directory, or use `.local/auth-assets/`, containing `offline-mfi/identity.pk8` and `offline-mfi/certificate.p7b`. These files stay out of Git. Run `./gradlew :mobile:assembleProvisionedDebug` to build and verify both packaged assets against the local input. Public source-only APKs should use the ordinary build without configured authentication assets, letting users import their own resources after installation. diff --git a/common/src/main/assets/projection/demo.h264clip b/common/src/main/assets/projection/demo.h264clip new file mode 100644 index 00000000..586b7ba9 Binary files /dev/null and b/common/src/main/assets/projection/demo.h264clip differ diff --git a/common/src/main/assets/web/app.js b/common/src/main/assets/web/app.js index 102d377e..059d41ba 100644 --- a/common/src/main/assets/web/app.js +++ b/common/src/main/assets/web/app.js @@ -931,6 +931,7 @@ if (typeof document !== 'undefined') { } $('refresh-qr').addEventListener('click', startQr); const pointers = new Map(); + let maxTouchContacts = 2; const send = (data, kind) => { if (data.type === 'touch' && presentation.current.videoEpoch !== null) { const { sessionId, videoEpoch } = presentation.current; @@ -1172,6 +1173,8 @@ if (typeof document !== 'undefined') { performanceStats.transport = data.performance?.transport ?? null; performanceStats.image = data.performance?.image ?? null; lastStatus = Date.now(); + const nextTouchLimit = Number.isInteger(data.maxTouchContacts) ? Math.max(0, Math.min(10, data.maxTouchContacts)) : 2; + if (nextTouchLimit !== maxTouchContacts) { release(); maxTouchContacts = nextTouchLimit; } const previousRevision = presentation.current.revision; if (!presentation.status(data)) return; if (previousRevision !== presentation.current.revision) { @@ -1306,11 +1309,12 @@ if (typeof document !== 'undefined') { function bindSurface(surface) { surface.addEventListener('contextmenu', event => event.preventDefault()); surface.addEventListener('pointerdown', event => { - if (!live || pointers.size >= 2 || (event.pointerType === 'mouse' && event.button !== 0)) return; + if (!live || pointers.size >= maxTouchContacts || (event.pointerType === 'mouse' && event.button !== 0)) return; const point = mapPoint(event.clientX, event.clientY, surface.getBoundingClientRect(), width, height); if (!point) return; event.preventDefault(); - const id = [...pointers.values()].some(p => p.id === 0) ? 1 : 0; + const used = new Set([...pointers.values()].map(p => p.id)); + let id = 0; while (used.has(id)) id++; pointers.set(event.pointerId, { id, ...point }); surface.setPointerCapture(event.pointerId); cancelMove(); diff --git a/common/src/main/java/com/shilapi/xcertplay/CarPlayHostActivity.kt b/common/src/main/java/com/shilapi/xcertplay/CarPlayHostActivity.kt index 46a4c2e8..e8060655 100644 --- a/common/src/main/java/com/shilapi/xcertplay/CarPlayHostActivity.kt +++ b/common/src/main/java/com/shilapi/xcertplay/CarPlayHostActivity.kt @@ -1,5 +1,9 @@ package com.shilapi.xcertplay +import com.shilapi.xcertplay.projection.CarPlaySourceSession +import com.shilapi.xcertplay.projection.ProjectionResource +import com.shilapi.xcertplay.projection.projectionSessions + import com.shilapi.xcertplay.shared.R as UiR import com.shilapi.xcertplay.i18n.* @@ -415,6 +419,10 @@ class CarPlayHostActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) showSystemBars() + if (projectionSessions.hasSession() && + projectionSessions.snapshot()?.descriptor?.id != "carplay") { + showDiPlayHome("home"); finish(); return + } if (intent.action == "android.hardware.usb.action.USB_DEVICE_ATTACHED") { AirPlayPersistence.saveWirelessEnabled(this, false) } @@ -2796,8 +2804,14 @@ class CarPlayHostActivity : ComponentActivity() { return AndroidMediaSink( surface = output.surface, encodedVideo = source, + frameDecoderFactory = { key -> + try { + com.shilapi.xcertplay.web.NativeVideoDecoder(key, platform = true) + .also { source.setProcessorName(it.name) } + } catch (_: LinkageError) { null } + }, remoteAudio = WebSession.audio.createRoute(), - onClosed = { output.close(); WebSession.endVideo(videoGeneration) }, + onClosed = { output.closeAndAwait(); WebSession.endVideo(videoGeneration) }, videoWidth = videoWidth, videoHeight = videoHeight, preferSoftwareHevcDecoder = hevcSoftwareDecoderEnabled, @@ -2837,6 +2851,7 @@ class CarPlayHostActivity : ComponentActivity() { override fun onSessionEnded(session: AirPlaySession) { runOnUiThread { + if (controllerGeneration != restartGeneration) return@runOnUiThread if (activeAirPlaySession === session) activeAirPlaySession = null CarPlayBackgroundSession.active = false WebSession.videoActive = false @@ -2903,10 +2918,6 @@ class CarPlayHostActivity : ComponentActivity() { private fun adoptBackgroundSession(): Boolean { val snapshot = CarPlayBackgroundSession.snapshot() ?: return false - if (snapshot.controller.isClosed()) { - CarPlayBackgroundSession.clear(snapshot.controller) - return false - } displayDiagnosticAttempt = DisplayDiagnosticSnapshot.currentAttempt(this) controller = snapshot.controller sink = snapshot.sink @@ -2919,6 +2930,10 @@ class CarPlayHostActivity : ComponentActivity() { if (snapshot.width > 0 && snapshot.height > 0) { activeDisplaySize = DisplaySize(snapshot.width, snapshot.height) } + if (snapshot.controller.isClosed()) { + restartCarPlay("Replacing closed background CarPlay controller") + return true + } val generation = restartGeneration snapshot.controller.attachUi( createSessionListener(generation), @@ -2960,6 +2975,13 @@ class CarPlayHostActivity : ComponentActivity() { setLocalizedConnectionStage(UiText(UiR.string.ui_authentication_resources_are_not_ready_import_files_or_a)) return } + if (!CarPlayBackgroundSession.reserve(this, { WebSession.stageDetail(this) }) { completion -> + runOnUiThread { shutdown(false, "WheelPlay disconnect", completion); finish() } + }) { + showDiPlayHome("home") + finish() + return + } val controllerGeneration = restartGeneration // The settings page can change branding while this activity remains in the back stack. simulatedManufacturerName = AirPlayPersistence.loadSimulatedManufacturerName(this) @@ -2993,6 +3015,8 @@ class CarPlayHostActivity : ComponentActivity() { ) } catch (error: Exception) { Log.e(TAG, "Offscreen video initialization failed", error) + if (error is com.shilapi.xcertplay.projection.ProjectionCleanupException) + CarPlayBackgroundSession.teardownFinished(false) setLocalizedConnectionStage(UiText(UiR.string.host_this_device_cannot_create_an_offscreen_decoder_surface_check)) return } @@ -3179,9 +3203,16 @@ class CarPlayHostActivity : ComponentActivity() { controller = null sink = null teardownExecutor.execute { - oldController?.close() - oldController?.awaitClosed(CONTROLLER_CLOSE_TIMEOUT_MILLIS) - oldSink?.close() + val clean = runCatching { + oldController?.close() + val transportStopped = oldController?.awaitClosed(CONTROLLER_CLOSE_TIMEOUT_MILLIS) ?: true + if (transportStopped) oldSink?.close(awaitWorkers = true) + transportStopped && CarPlayBackgroundSession.teardownClean + }.getOrDefault(false) + if (!clean) { + CarPlayBackgroundSession.teardownFinished(false) + return@execute + } runOnUiThread { if (!shuttingDown.get() && generation == restartGeneration) { handshakeResetInProgress = false @@ -3252,16 +3283,22 @@ class CarPlayHostActivity : ComponentActivity() { sink = null Log.i(TAG, "shutdown reason=$reason terminateProcess=$terminateProcess") teardownExecutor.execute { - oldController?.close() - val clean = oldController?.awaitClosed(CONTROLLER_CLOSE_TIMEOUT_MILLIS) ?: true - oldSink?.close() + val clean = runCatching { + oldController?.close() + val transportStopped = oldController?.awaitClosed(CONTROLLER_CLOSE_TIMEOUT_MILLIS) ?: true + if (transportStopped) oldSink?.close(awaitWorkers = true) + transportStopped && CarPlayBackgroundSession.teardownClean + }.getOrDefault(false) if (terminateProcess) { applicationContext.stopService(Intent(applicationContext, CarPlayVpnService::class.java)) } Log.i(TAG, "shutdown complete clean=$clean") - applicationContext.stopService(Intent(applicationContext, DiPlaySessionService::class.java)) + if (terminateProcess) applicationContext.stopService(Intent(applicationContext, DiPlaySessionService::class.java)) teardownExecutor.shutdown() - mainHandler.post { completion() } + mainHandler.post { + completion() + CarPlayBackgroundSession.teardownFinished(clean) + } if (terminateProcess) Process.killProcess(Process.myPid()) } } @@ -3548,6 +3585,31 @@ internal object CarPlayBackgroundSession { val height: Int, ) + @Volatile var teardownClean = true + private set + + @Synchronized fun reserve(owner: Any, status: () -> String?, stop: (() -> Unit) -> Unit): Boolean { + if (projection != null) return this.owner === owner + val adapter = CarPlaySourceSession(status) + if (!projectionSessions.adopt(adapter, + ProjectionResource.entries.toSet())) return false + projection = adapter + this.owner = owner + stopAction = stop + teardownClean = true + return true + } + + fun teardownFinished(clean: Boolean) { + val adapter = synchronized(this) { + teardownClean = clean + projection.also { if (clean) projection = null } + } + adapter?.completed(clean) + if (clean) adapter?.let { projectionSessions.ended(it) } + } + + private var projection: CarPlaySourceSession? = null private var controller: CarPlayController? = null private var sink: AndroidMediaSink? = null private var width = 0 @@ -3573,9 +3635,11 @@ internal object CarPlayBackgroundSession { this.owner = owner this.controller = controller this.sink = sink + projection?.bind(controller, sink) this.width = width this.height = height } + projection?.let { projectionSessions.changed(it) } listeners.forEach { it(snapshot()) } } @@ -3584,6 +3648,7 @@ internal object CarPlayBackgroundSession { if (expected != null && controller !== expected) return controller = null sink = null + projection?.bind(null, null) if (!keepOwner) { stopAction = null; owner = null } active = false width = 0 diff --git a/common/src/main/java/com/shilapi/xcertplay/DiPlayActivity.kt b/common/src/main/java/com/shilapi/xcertplay/DiPlayActivity.kt index 1f2b3f3b..4444156d 100644 --- a/common/src/main/java/com/shilapi/xcertplay/DiPlayActivity.kt +++ b/common/src/main/java/com/shilapi/xcertplay/DiPlayActivity.kt @@ -2,6 +2,9 @@ // UI copy and visual language adapted from DiAuto. See docs/THIRD_PARTY_NOTICES.md. package com.shilapi.xcertplay +import com.shilapi.xcertplay.projection.projectionSessions +import com.shilapi.xcertplay.projection.registerProjectionSources + import com.shilapi.xcertplay.shared.R as UiR import com.shilapi.xcertplay.i18n.* @@ -272,7 +275,7 @@ class DiPlayActivity : AppCompatActivity() { initialLaunch = false if (setupError == null && !DiPlayBootstrap.importing && selectedPrivateKey == null && intent.action != Intent.ACTION_SEND && intent.action != Intent.ACTION_VIEW && - !CarPlayBackgroundSession.hasSession() && + !projectionSessions.hasSession() && DiPlayPreferences.autoConnect(this) && intent.getStringExtra("page") == null) { handler.post { connect(AirPlayPersistence.loadWirelessEnabled(this)) } } @@ -477,8 +480,7 @@ class DiPlayActivity : AppCompatActivity() { private fun refreshService() { val addresses = if (WebSession.running && WebSession.error == null) WebSession.addresses(this) else emptyList() val primary = addresses.firstOrNull() - val addressText = if (!WebSession.running && setupError != null) getString(UiR.string.ui_import_authentication_resources_to_activate_the_web_service) - else WebSession.error ?: if (WebSession.running) primary?.url + val addressText = WebSession.error ?: if (WebSession.running) primary?.url ?: getString(UiR.string.ui_no_local_network_address_found_connect_to_wi_fi) else getString(UiR.string.ui_starting_service) if (webAddress?.text?.toString() != addressText) webAddress?.text = addressText webInterface?.text = primary?.let { it.description(this) + if (it.hostname.isNotEmpty()) getString(UiR.string.address_direct_ip, it.ipUrl) else "" }.orEmpty() @@ -498,9 +500,8 @@ class DiPlayActivity : AppCompatActivity() { quickBrowserButton?.isEnabled = WebSession.running && BrowserExperienceLink.local(WebSession.code, WebSession.httpPort) != null val viewerConnected = WebSession.hasViewer webPairingControls?.visibility = if (viewerConnected) View.GONE else View.VISIBLE - webViewer?.text = if (!WebSession.running && setupError != null) getString(UiR.string.ui_import_authentication_resources_to_pair_a_browser) - else if (viewerConnected) getString(UiR.string.ui_head_unit_browser_connected) else getString(UiR.string.ui_waiting_for_head_unit_browser) - webStage?.text = setupError ?: if (WebSession.videoActive) getString(UiR.string.ui_streaming_carplay_video) else WebSession.stage(this) + webViewer?.text = if (viewerConnected) getString(UiR.string.ui_head_unit_browser_connected) else getString(UiR.string.ui_waiting_for_head_unit_browser) + webStage?.text = WebSession.stage(this) } private fun openLocalBrowserExperience() { @@ -521,7 +522,17 @@ class DiPlayActivity : AppCompatActivity() { } private fun home(content: LinearLayout) { - content.addView(label(getString(UiR.string.ui_connect_iphone), 30, TEXT, true)) + content.addView(label(getString(UiR.string.projection_sources), 30, TEXT, true)) + ui.card(content, getString(UiR.string.projection_sources), R.drawable.ic_server_phone) { body -> + body.addView(label(getString(UiR.string.projection_demo_hint), 15, MUTED)) + body.addView(button(getString(UiR.string.projection_choose_demo), false) { chooseProjectionSource() }, matchButton(12)) + body.addView(button(getString(UiR.string.projection_stop), false) { + projectionSessions.stop().whenComplete { _, failure -> runOnUiThread { + if (failure != null) toast(getString(UiR.string.projection_failed, failure.cause?.message ?: failure.message.orEmpty())) + refreshStatus(); refreshService() + } } + }, ui.secondaryButtonLayout(12)) + } content.addView(label(getString(UiR.string.ui_this_device_receives_carplay_the_head_unit_browser_provides), 16, MUTED).apply { setPadding(0, dp(8), 0, dp(24)) }) status = label(getString(UiR.string.ui_waiting_for_connection), 20, ACCENT, true) ui.card(content, getString(UiR.string.ui_connection_status), R.drawable.ic_server_phone) { it.addView(status) } @@ -551,6 +562,19 @@ class DiPlayActivity : AppCompatActivity() { setupError?.let { content.addView(label(it, 16, WARNING)) } } + private fun chooseProjectionSource() { + registerProjectionSources(this) + val sources = projectionSessions.sources() + android.app.AlertDialog.Builder(this).setTitle(getString(UiR.string.projection_choose_demo)) + .setItems(sources.map { it.descriptor.displayName }.toTypedArray()) { _, index -> + startWebServiceIfReady() + projectionSessions.switchTo(sources[index].descriptor.id).whenComplete { _, failure -> runOnUiThread { + if (failure != null) toast(getString(UiR.string.projection_failed, failure.cause?.message ?: failure.message.orEmpty())) + refreshStatus(); refreshService() + } } + }.setNegativeButton(android.R.string.cancel, null).show() + } + private fun refreshAuthentication() { setupError = if (DiPlayBootstrap.importing) getString(UiR.string.ui_importing_authentication_resources_please_wait) else runCatching { DiPlayBootstrap.ensure(this) }.exceptionOrNull()?.let { @@ -559,9 +583,7 @@ class DiPlayActivity : AppCompatActivity() { } private fun startWebServiceIfReady() { - if (setupError == null && !DiPlayBootstrap.importing) { - startForegroundService(Intent(this, DiPlaySessionService::class.java)) - } + startForegroundService(Intent(this, DiPlaySessionService::class.java)) } private fun canImportAuthentication(): Boolean { @@ -719,8 +741,10 @@ class DiPlayActivity : AppCompatActivity() { AirPlayPersistence.saveWirelessEnabled(this, wireless) openProjection() } - if (CarPlayBackgroundSession.hasSession()) CarPlayBackgroundSession.stop { runOnUiThread { open() } } - else open() + if (projectionSessions.hasSession()) projectionSessions.stop().whenComplete { _, failure -> runOnUiThread { + if (failure == null) open() + else toast(getString(UiR.string.projection_failed, failure.cause?.message ?: failure.message.orEmpty())) + } } else open() } private fun openProjection() { navigate("service") @@ -818,6 +842,8 @@ class DiPlayActivity : AppCompatActivity() { private fun refreshStatus() { val running = CarPlayBackgroundSession.hasSession() status?.text = when { + projectionSessions.snapshot()?.descriptor?.id?.let { it != "carplay" } == true -> + projectionSessions.snapshot()?.descriptor?.displayName setupError != null -> getString(UiR.string.ui_authentication_setup_needs_attention) CarPlayBackgroundSession.active -> getString(UiR.string.ui_carplay_connected) running -> getString(UiR.string.ui_connecting_to_iphone) diff --git a/common/src/main/java/com/shilapi/xcertplay/DiPlaySessionService.kt b/common/src/main/java/com/shilapi/xcertplay/DiPlaySessionService.kt index cd61653b..7b73eda5 100644 --- a/common/src/main/java/com/shilapi/xcertplay/DiPlaySessionService.kt +++ b/common/src/main/java/com/shilapi/xcertplay/DiPlaySessionService.kt @@ -29,6 +29,7 @@ import com.shilapi.xcertplay.shared.R as UiR import com.shilapi.xcertplay.web.RootAccess import com.shilapi.xcertplay.web.WebListenSettings import com.shilapi.xcertplay.web.WebSession +import com.shilapi.xcertplay.projection.projectionSessions import java.util.concurrent.CompletableFuture /** Keeps the connection and its media session alive while no dashboard is visible. */ @@ -38,13 +39,13 @@ class DiPlaySessionService : Service() { private var sessionSubscription: AutoCloseable? = null private val main = Handler(Looper.getMainLooper()) private var destroyed = false - private var activationBlocked = false private var currentState = NowPlayingState() private var currentSession: MediaSession? = null private var currentArtwork: Bitmap? = null override fun onCreate() { super.onCreate() + com.shilapi.xcertplay.projection.registerProjectionSources(this) if ( com.shilapi.xcertplay.web.TeslaHttpCompatibility.config(this).let { it.enabled && it.mode == com.shilapi.xcertplay.network.TeslaHttpMode.ROOT @@ -68,11 +69,12 @@ class DiPlaySessionService : Service() { } browserMediaBridge = media sessionSubscription = - CarPlayBackgroundSession.subscribe { _ -> + projectionSessions.subscribe { _ -> main.post { if (!destroyed) { - val current = CarPlayBackgroundSession.snapshot() - media.bind(current?.controller, current?.sink) + val current = projectionSessions.snapshot() + media.bind(current?.media, current?.mediaOutput) + updateNotification() } } } @@ -85,35 +87,19 @@ class DiPlaySessionService : Service() { if (intent?.action == ACTION_STOP) { media.close() runWebAction { WebSession.stop() } - CarPlayBackgroundSession.stop() + projectionSessions.stop() stopSelf() return START_NOT_STICKY } if ( intent?.action == ACTION_MEDIA && !currentState.available && - !CarPlayBackgroundSession.hasSession() + !projectionSessions.hasSession() ) { stopSelf() return START_NOT_STICKY } - activationBlocked = false updateNotification() - // Foreground startup must be acknowledged even when an external start is rejected. - // Never open listeners or acquire a wake lock without a validated identity. - if ( - !(DiPlayBootstrap.importing && WebSession.running) && - runCatching { DiPlayBootstrap.ensure(this) }.isFailure - ) { - activationBlocked = true - WebSession.stop() - CarPlayBackgroundSession.stop() - wakeLock?.let { if (it.isHeld) it.release() } - wakeLock = null - stopForeground(STOP_FOREGROUND_REMOVE) - stopSelf() - return START_NOT_STICKY - } if (intent?.action == ACTION_MEDIA) { if (intent.getLongExtra(EXTRA_GENERATION, -1) == media.commandGeneration) { intent.getStringExtra(EXTRA_COMMAND)?.let { name -> @@ -167,7 +153,7 @@ class DiPlaySessionService : Service() { } private fun updateNotification() { - if (destroyed || activationBlocked) return + if (destroyed) return val open = PendingIntent.getActivity( this, @@ -257,6 +243,8 @@ class DiPlaySessionService : Service() { types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK if ( Build.VERSION.SDK_INT >= 30 && + projectionSessions.snapshot()?.capabilities?.microphone == true && + AirPlayPersistence.loadMicrophoneSource(this) != CarPlayMicrophoneSource.DISABLED && checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED ) { @@ -274,7 +262,7 @@ class DiPlaySessionService : Service() { runWebAction { WebSession.stop() } wakeLock?.let { if (it.isHeld) it.release() } wakeLock = null - CarPlayBackgroundSession.stop() + projectionSessions.stop() super.onDestroy() } diff --git a/common/src/main/java/com/shilapi/xcertplay/media/CarPlayMediaSessionBridge.kt b/common/src/main/java/com/shilapi/xcertplay/media/CarPlayMediaSessionBridge.kt index 777f758e..6ea96382 100644 --- a/common/src/main/java/com/shilapi/xcertplay/media/CarPlayMediaSessionBridge.kt +++ b/common/src/main/java/com/shilapi/xcertplay/media/CarPlayMediaSessionBridge.kt @@ -24,14 +24,14 @@ internal class CarPlayMediaSessionBridge( ) : AutoCloseable { private val main = Handler(Looper.getMainLooper()) private val audio = context.getSystemService(AudioManager::class.java) - internal val session = MediaSession(context, "WheelPlay iPhone") + internal val session = MediaSession(context, "WheelPlay projection") private val artworkWorker = Executors.newSingleThreadExecutor { Thread(it, "carplay-artwork-preview").apply { isDaemon = true } } private val defaultArtwork = Bitmap.createBitmap(128, 128, Bitmap.Config.ARGB_8888).also { bitmap -> context.applicationInfo.loadIcon(context.packageManager).apply { setBounds(0, 0, 128, 128); draw(Canvas(bitmap)) } } - private var source: CarPlayMediaSource? = null + private var source: MediaControlSource? = null private var output: MediaOutputControl? = null private var subscription: AutoCloseable? = null private var routeSubscription: AutoCloseable? = null @@ -91,7 +91,7 @@ internal class CarPlayMediaSessionBridge( }, main) } - fun bind(next: CarPlayMediaSource?, nextOutput: MediaOutputControl?) { + fun bind(next: MediaControlSource?, nextOutput: MediaOutputControl?) { if (closed || source === next && output === nextOutput) return val generation = ++epoch ++commandGeneration diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/BrowserProjectionVideo.kt b/common/src/main/java/com/shilapi/xcertplay/projection/BrowserProjectionVideo.kt new file mode 100644 index 00000000..d094a735 --- /dev/null +++ b/common/src/main/java/com/shilapi/xcertplay/projection/BrowserProjectionVideo.kt @@ -0,0 +1,117 @@ +package com.shilapi.xcertplay.projection + +import android.content.Context +import android.graphics.Bitmap +import android.view.Surface +import com.shilapi.xcertplay.AirPlayPersistence +import com.shilapi.xcertplay.media.AndroidMediaSink +import com.shilapi.xcertplay.media.CompressedVideoFrame +import com.shilapi.xcertplay.media.VideoCodec +import com.shilapi.xcertplay.web.ImageFrameEncoder +import com.shilapi.xcertplay.web.OffscreenVideoOutput +import com.shilapi.xcertplay.web.StreamTechnology +import com.shilapi.xcertplay.web.WebSession +import com.shilapi.xcertplay.web.WebVideoSource +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference + +/** Android output endpoint shared by encoded, Surface and framebuffer sources. */ +internal class BrowserProjectionVideo( + context: Context, val width: Int, val height: Int, + private val kind: VideoIngressKind, +) : ProjectionVideoOutput { + private val app = context.applicationContext + private val closed = AtomicBoolean() + private var generation = -1L + private val encoded = WebVideoSource(StreamTechnology.WEBRTC) + private var offscreen: OffscreenVideoOutput? = null + private var decoder: AndroidMediaSink? = null + private val images = Executors.newSingleThreadScheduledExecutor { Thread(it, "projection-images").apply { isDaemon = true } } + private val pending = AtomicReference() + private var latest: ProjectionFramebuffer.Snapshot? = null + private val imageEncoder = ImageFrameEncoder() + private var hadViewer = false + private var configured = false + private var revision = 0L + + init { + require(width in 1..8192 && height in 1..8192 && width.toLong() * height <= 8_388_608) + try { + if (kind != VideoIngressKind.FRAMEBUFFER) offscreen = OffscreenVideoOutput(width, height, + wanted = { !closed.get() && WebSession.hasViewer && (kind != VideoIngressKind.ENCODED || encoded.jpegNeeded) }, + encoding = { AirPlayPersistence.imageEncoding(app, WebSession.imageFormat) }, + publish = { bytes, format, quality -> + if (!closed.get()) { + WebSession.setVideoActive(generation, true) + WebSession.publish(generation, bytes, format, quality) + } + }) + if (kind == VideoIngressKind.ENCODED) decoder = AndroidMediaSink( + surface = offscreen!!.surface, videoWidth = width, videoHeight = height, encodedVideo = encoded) + generation = WebSession.beginVideo(width, height) + if (kind == VideoIngressKind.ENCODED) WebSession.attachVideoSource(generation, encoded) + if (kind == VideoIngressKind.FRAMEBUFFER) images.scheduleWithFixedDelay({ + if (!closed.get()) { + val next = pending.getAndSet(null) + if (next != null) latest = next + val viewer = WebSession.hasViewer + if (viewer && (next != null || !hadViewer)) latest?.let { snapshot -> + val bitmap = Bitmap.createBitmap(snapshot.argb, width, height, Bitmap.Config.ARGB_8888) + try { + val options = AirPlayPersistence.imageEncoding(app, WebSession.imageFormat) + val (bytes, format) = imageEncoder.encode(options) { format, quality, output -> + bitmap.compress(format.compression, quality, output) + } + WebSession.publish(generation, bytes, format, options.quality) + } finally { bitmap.recycle() } + } + hadViewer = viewer + } + }, 0, 66, TimeUnit.MILLISECONDS) + } catch (failure: Throwable) { + try { close() } catch (cleanup: Throwable) { + cleanup.addSuppressed(failure) + throw ProjectionCleanupException("Video startup cleanup failed", cleanup) + } + throw failure + } + } + + override val surface: Surface get() { + check(kind == VideoIngressKind.SURFACE && !closed.get()) + return offscreen!!.surface + } + @Synchronized override fun configure(codec: VideoCodec, codecData: ByteArray, requestKeyframe: () -> Unit): Long { + check(kind == VideoIngressKind.ENCODED && !closed.get()) + val target = decoder!! + target.setVideoRecoveryHandler(0, requestKeyframe) + target.onVideoCodec(0, codec) + target.onVideoConfig(0, codecData.copyOf()) + configured = true + return ++revision + } + @Synchronized override fun frame(formatRevision: Long, frame: CompressedVideoFrame, ptsUs: Long): Boolean { + if (closed.get() || !configured || formatRevision != revision) return false + decoder!!.onVideoFrame(0, frame, ptsUs, System.nanoTime(), 0) + WebSession.setVideoActive(generation, true) + return true + } + override fun framebuffer(snapshot: ProjectionFramebuffer.Snapshot): Boolean { + if (closed.get() || kind != VideoIngressKind.FRAMEBUFFER) return false + require(snapshot.width == width && snapshot.height == height && snapshot.argb.size == width * height) + pending.set(snapshot.copy(argb = snapshot.argb.copyOf())) + WebSession.setVideoActive(generation, true) + return true + } + override fun close() { + if (!closed.compareAndSet(false, true)) return + images.shutdownNow() + check(images.awaitTermination(5, TimeUnit.SECONDS)) { "Image producer did not stop" } + pending.set(null) + decoder?.close(awaitWorkers = true) + offscreen?.closeAndAwait() + WebSession.endVideo(generation) + } +} diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt b/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt new file mode 100644 index 00000000..c09cdc9f --- /dev/null +++ b/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt @@ -0,0 +1,39 @@ +package com.shilapi.xcertplay.projection + +import com.shilapi.xcertplay.CarPlayBackgroundSession +import java.util.concurrent.CompletableFuture + +/** The only projection adapter that knows about the legacy CarPlay controller. */ +internal class CarPlaySourceSession(private val status: () -> String?) : ProjectionSession { + @Volatile private var controller: com.shilapi.xcertplay.orchestration.CarPlayController? = null + @Volatile private var sink: com.shilapi.xcertplay.media.AndroidMediaSink? = null + private val closed = CompletableFuture() + fun bind(controller: com.shilapi.xcertplay.orchestration.CarPlayController?, sink: com.shilapi.xcertplay.media.AndroidMediaSink?) { + this.controller = controller + this.sink = sink + } + fun completed(clean: Boolean) { + if (clean) closed.complete(Unit) + else closed.completeExceptionally(IllegalStateException("CarPlay transport did not stop")) + } + override val statusText get() = status() + override val descriptor = SourceDescriptor("carplay", "CarPlay") + override val capabilities = ProjectionCapabilities(setOf(VideoIngressKind.ENCODED), 2, + nightMode = true, audioPlayback = true, microphone = true) + override val inputReady get() = controller?.touchReady == true + override val media get() = controller + override val mediaOutput get() = sink + override fun touch(contacts: List): Boolean { + if (contacts.any { it.id !in 0..1 }) return false + // The CarPlay HID device has fixed slots, even when the first finger is released. + return controller?.sendTouch((0..1).map { id -> + contacts.find { it.id == id } ?: ProjectionContact(id, 0.0, 0.0, false) + }) ?: false + } + override fun setNightMode(night: Boolean) = + if (controller?.setNightMode(night) == true) ProjectionCommandResult.SENT else ProjectionCommandResult.UNAVAILABLE + override fun stop(): CompletableFuture { + CarPlayBackgroundSession.stop() + return closed + } +} diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionSessions.kt b/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionSessions.kt new file mode 100644 index 00000000..0857fc63 --- /dev/null +++ b/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionSessions.kt @@ -0,0 +1,14 @@ +package com.shilapi.xcertplay.projection + +/** Application-scoped coordinator; the foreground service owns its shutdown. */ +internal val projectionSessions = ProjectionRuntime() + +@Synchronized internal fun registerProjectionSources(context: android.content.Context) { + if (projectionSessions.sources().isNotEmpty()) return + val app = context.applicationContext + VideoIngressKind.entries.forEach { kind -> + projectionSessions.register(SyntheticSource(kind, + { ingress, width, height -> BrowserProjectionVideo(app, width, height, ingress) }, + { app.assets.open("projection/demo.h264clip") })) + } +} diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionVideoOutput.kt b/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionVideoOutput.kt new file mode 100644 index 00000000..1bd6bcd8 --- /dev/null +++ b/common/src/main/java/com/shilapi/xcertplay/projection/ProjectionVideoOutput.kt @@ -0,0 +1,18 @@ +package com.shilapi.xcertplay.projection + +import android.view.Surface +import com.shilapi.xcertplay.media.CompressedVideoFrame +import com.shilapi.xcertplay.media.VideoCodec + +/** Android host port. A source receives this port from the composition layer, never a WebSession. + * Only one ingress kind is enabled per output. Stop the producer before closing the output. + */ +internal interface ProjectionVideoOutput : AutoCloseable { + val surface: Surface + /** ISO-BMFF avcC/hvcC configuration; returns a revision required by every subsequent frame. */ + fun configure(codec: VideoCodec, codecData: ByteArray, requestKeyframe: () -> Unit): Long + /** Complete Annex-B access unit; borrowed for this call, retained by asynchronous consumers. PTS is microseconds. */ + fun frame(formatRevision: Long, frame: CompressedVideoFrame, ptsUs: Long): Boolean + /** Complete composed ARGB snapshot. The output copies it before returning and coalesces pending snapshots. */ + fun framebuffer(snapshot: ProjectionFramebuffer.Snapshot): Boolean +} diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/SyntheticSource.kt b/common/src/main/java/com/shilapi/xcertplay/projection/SyntheticSource.kt new file mode 100644 index 00000000..43b637f2 --- /dev/null +++ b/common/src/main/java/com/shilapi/xcertplay/projection/SyntheticSource.kt @@ -0,0 +1,122 @@ +package com.shilapi.xcertplay.projection + +import android.graphics.Color +import android.graphics.Paint +import com.shilapi.xcertplay.media.HeapVideoFrame +import com.shilapi.xcertplay.media.VideoCodec +import java.io.DataInputStream +import java.io.InputStream +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +/** Deterministic host diagnostic, with no network, authentication or microphone requirement. */ +internal class SyntheticSource( + private val kind: VideoIngressKind, + private val output: (VideoIngressKind, Int, Int) -> ProjectionVideoOutput, + private val encodedFixture: () -> InputStream, +) : ProjectionSourceProvider { + override val descriptor = SourceDescriptor("demo-${kind.name.lowercase()}", "Demo · ${kind.name.lowercase()}") + override val resources = setOf(ProjectionResource.DISPLAY) + override fun probe() = SourceAvailability.Ready + override fun open(): CompletableFuture = CompletableFuture.supplyAsync { + val clip = if (kind == VideoIngressKind.ENCODED) Clip.read(encodedFixture()) else null + val video = output(kind, WIDTH, HEIGHT) + try { Session(video, clip) } catch (failure: Throwable) { + try { video.close() } catch (cleanup: Throwable) { + cleanup.addSuppressed(failure) + throw ProjectionCleanupException("Demo startup cleanup failed", cleanup) + } + throw failure + } + } + + private inner class Session(private val video: ProjectionVideoOutput, private val clip: Clip?) : ProjectionSession { + override val descriptor = this@SyntheticSource.descriptor + override val capabilities = ProjectionCapabilities(setOf(kind), if (clip == null) 10 else 0, nightMode = clip == null) + private val stopped = AtomicBoolean() + private val keyframe = AtomicBoolean() + private val worker = Executors.newSingleThreadScheduledExecutor { Thread(it, "projection-demo").apply { isDaemon = true } } + private val completion = CompletableFuture() + @Volatile private var contacts = emptyList() + @Volatile private var night = false + @Volatile private var failureMessage: String? = null + override val statusText get() = failureMessage + override val inputReady get() = !stopped.get() && failureMessage == null && capabilities.maxTouchContacts > 0 + private val pixels = ProjectionFramebuffer(WIDTH, HEIGHT) + private var frame = 0L + private var clipPosition = 0 + private val revision = clip?.let { video.configure(VideoCodec.H264, it.config) { keyframe.set(true) } } + + init { + worker.scheduleWithFixedDelay({ + if (!stopped.get() && failureMessage == null) try { draw() } catch (failure: Exception) { failureMessage = failure.message ?: failure.javaClass.simpleName } + }, 0, 66, TimeUnit.MILLISECONDS) + } + private fun draw() { + if (clip != null) { + if (keyframe.getAndSet(false)) clipPosition = 0 + HeapVideoFrame(clip.frames[clipPosition]).use { video.frame(revision!!, it, frame * 66_667) } + clipPosition = (clipPosition + 1) % clip.frames.size + } else if (kind == VideoIngressKind.SURFACE) { + val surface = video.surface + val canvas = surface.lockCanvas(null) + try { + canvas.drawColor(if (night) Color.DKGRAY else Color.rgb(20, 60, 90)) + val paint = Paint().apply { color = Color.CYAN; textSize = 18f } + canvas.drawText("Surface · $frame", 12f, 28f, paint) + canvas.drawRect((frame % WIDTH).toFloat(), 48f, (frame % WIDTH + 18).toFloat(), 72f, paint) + contacts.filter { it.down }.forEach { canvas.drawCircle((it.x * (WIDTH - 1)).toFloat(), (it.y * (HEIGHT - 1)).toFloat(), 12f, paint) } + } finally { surface.unlockCanvasAndPost(canvas) } + } else { + // Apply ordered increments, including overlapping CopyRect, before publishing a full snapshot. + pixels.copy(1, 0, 0, 0, WIDTH - 1, HEIGHT) + val color = if (night) Color.DKGRAY else Color.rgb((frame * 3 % 255).toInt(), 90, 150) + pixels.update(WIDTH - 1, 0, 1, HEIGHT, IntArray(HEIGHT) { color }) + contacts.filter { it.down }.forEach { + val x = (it.x * (WIDTH - 9)).toInt(); val y = (it.y * (HEIGHT - 9)).toInt() + pixels.update(x, y, 9, 9, IntArray(81) { Color.WHITE }) + } + video.framebuffer(pixels.snapshot()) + } + frame++ + } + override fun touch(contacts: List): Boolean { + if (!inputReady) return false + this.contacts = contacts.toList() + return true + } + override fun setNightMode(night: Boolean): ProjectionCommandResult { + if (!capabilities.nightMode) return ProjectionCommandResult.UNSUPPORTED + if (stopped.get()) return ProjectionCommandResult.UNAVAILABLE + this.night = night + return ProjectionCommandResult.SENT + } + override fun stop(): CompletableFuture { + if (stopped.compareAndSet(false, true)) { + contacts = emptyList() + worker.shutdownNow() + CompletableFuture.runAsync { + try { + check(worker.awaitTermination(5, TimeUnit.SECONDS)) { "Demo producer did not stop" } + video.close() + completion.complete(Unit) + } catch (failure: Throwable) { completion.completeExceptionally(failure) } + } + } + return completion + } + } + private data class Clip(val config: ByteArray, val frames: List) { + companion object { + fun read(stream: InputStream) = DataInputStream(stream).use { input -> + fun bytes(): ByteArray = ByteArray(input.readInt().also { require(it in 1..1_048_576) }).also(input::readFully) + val config = bytes() + val count = input.readInt().also { require(it in 1..300) } + Clip(config, List(count) { bytes() }) + } + } + } + companion object { const val WIDTH = 320; const val HEIGHT = 180 } +} diff --git a/common/src/main/java/com/shilapi/xcertplay/web/BrowserMediaBridge.kt b/common/src/main/java/com/shilapi/xcertplay/web/BrowserMediaBridge.kt index 5e18da3d..ae0047a2 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/BrowserMediaBridge.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/BrowserMediaBridge.kt @@ -3,7 +3,7 @@ package com.shilapi.xcertplay.web import android.os.Handler import android.os.Looper import android.os.SystemClock -import com.shilapi.xcertplay.CarPlayBackgroundSession +import com.shilapi.xcertplay.projection.projectionSessions import com.shilapi.xcertplay.DiPlaySessionService import com.shilapi.xcertplay.media.CarPlayMediaSource import com.shilapi.xcertplay.media.MediaArtwork @@ -18,8 +18,8 @@ import org.json.JSONObject /** One paired viewer. All binding/state/command work is serialized on main, never on iAP2. */ internal class BrowserMediaBridge( private val send: (String) -> Unit, - private val currentSource: () -> CarPlayMediaSource? = { CarPlayBackgroundSession.snapshot()?.controller }, - private val observeSource: (() -> Unit) -> AutoCloseable = { listener -> CarPlayBackgroundSession.subscribe { listener() } }, + private val currentSource: () -> CarPlayMediaSource? = { projectionSessions.snapshot()?.media }, + private val observeSource: (() -> Unit) -> AutoCloseable = { listener -> projectionSessions.subscribe { listener() } }, private val dispatch: (CarPlayMediaSource, Long, MediaCommand) -> CompletableFuture = DiPlaySessionService::browserMediaCommand, private val dispatchSeek: (CarPlayMediaSource, Long, Long, Long) -> CompletableFuture = DiPlaySessionService::browserMediaSeek, private val main: Handler = Handler(Looper.getMainLooper()), diff --git a/common/src/main/java/com/shilapi/xcertplay/web/LanWebServer.kt b/common/src/main/java/com/shilapi/xcertplay/web/LanWebServer.kt index 90a512a2..35cce817 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/LanWebServer.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/LanWebServer.kt @@ -2,7 +2,7 @@ package com.shilapi.xcertplay.web import android.content.Context import android.os.SystemClock -import com.shilapi.xcertplay.airplay.AirPlayContact +import com.shilapi.xcertplay.projection.ProjectionContact import com.shilapi.xcertplay.shared.R as UiR import fi.iki.elonen.NanoHTTPD import fi.iki.elonen.NanoWSD @@ -696,6 +696,9 @@ internal class LanWebServer( .put("videoEpoch", media.videoEpoch) .put("sourceState", media.sourceState) .put("inputReady", media.inputReady) + .put("sourceId", media.sourceId) + .put("sourceName", media.sourceName) + .put("maxTouchContacts", media.maxTouchContacts) .put("stage", WebSession.stage(context)) .put( "streaming", @@ -783,11 +786,11 @@ internal class LanWebServer( } "touch" -> { val contacts = data.getJSONArray("contacts") - require(contacts.length() <= 2) + require(contacts.length() <= 10) val list = (0 until contacts.length()).map { val point = contacts.getJSONObject(it) - AirPlayContact( + ProjectionContact( point.getInt("id"), point.getDouble("x"), point.getDouble("y"), diff --git a/common/src/main/java/com/shilapi/xcertplay/web/NativeVideoDecoder.kt b/common/src/main/java/com/shilapi/xcertplay/web/NativeVideoDecoder.kt index 300bf320..da96a0f1 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/NativeVideoDecoder.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/NativeVideoDecoder.kt @@ -2,7 +2,7 @@ package com.shilapi.xcertplay.web import androidx.annotation.Keep import com.shilapi.xcertplay.airplay.AirPlayCrypto -import com.shilapi.xcertplay.airplay.VideoCodec +import com.shilapi.xcertplay.media.VideoCodec import com.shilapi.xcertplay.media.CompressedVideoFrame import com.shilapi.xcertplay.media.VideoFrameDecoder import java.util.concurrent.atomic.AtomicInteger diff --git a/common/src/main/java/com/shilapi/xcertplay/web/RtcVideoStream.kt b/common/src/main/java/com/shilapi/xcertplay/web/RtcVideoStream.kt index 7a597ad5..49b472e1 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/RtcVideoStream.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/RtcVideoStream.kt @@ -1,7 +1,7 @@ package com.shilapi.xcertplay.web import android.util.Log -import com.shilapi.xcertplay.airplay.VideoCodec +import com.shilapi.xcertplay.media.VideoCodec import com.shilapi.xcertplay.media.CompressedVideoFrame import java.io.Closeable import java.util.concurrent.Executors diff --git a/common/src/main/java/com/shilapi/xcertplay/web/TouchLease.kt b/common/src/main/java/com/shilapi/xcertplay/web/TouchLease.kt index 7466b454..88a79d36 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/TouchLease.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/TouchLease.kt @@ -1,9 +1,9 @@ package com.shilapi.xcertplay.web -import com.shilapi.xcertplay.airplay.AirPlayContact +import com.shilapi.xcertplay.projection.ProjectionContact /** Serializes browser ownership and releases fingers when a client disappears. */ -class TouchLease(private val send: (List) -> Boolean) { +class TouchLease(private val maxContacts: () -> Int = { 2 }, private val send: (List) -> Boolean) { private var owner: Any? = null private var lastTouch = 0L private var pressed = false @@ -14,13 +14,11 @@ class TouchLease(private val send: (List) -> Boolean) { return true } - @Synchronized fun touch(client: Any, contacts: List, now: Long): Boolean { - if (owner !== client || contacts.size > 2 || contacts.any { - !it.x.isFinite() || !it.y.isFinite() || it.x !in 0.0..1.0 || it.y !in 0.0..1.0 || it.id !in 0..1 + @Synchronized fun touch(client: Any, contacts: List, now: Long): Boolean { + if (owner !== client || contacts.size > maxContacts() || contacts.any { + !it.x.isFinite() || !it.y.isFinite() || it.x !in 0.0..1.0 || it.y !in 0.0..1.0 || it.id !in 0 until maxContacts() } || contacts.map { it.id }.distinct().size != contacts.size) return false - // HID slots must not shift when the first of two fingers lifts. - val slots = (0..1).map { id -> contacts.find { it.id == id } ?: AirPlayContact(id, 0.0, 0.0, false) } - val accepted = send(slots) + val accepted = send(contacts.sortedBy { it.id }) pressed = accepted && contacts.any { it.down } lastTouch = now return accepted @@ -35,5 +33,9 @@ class TouchLease(private val send: (List) -> Boolean) { releaseFingers(); owner = null } + @Synchronized fun reset() { releaseFingers(); owner = null } + + @Synchronized fun cancel() { releaseFingers() } + private fun releaseFingers() { send(emptyList()); pressed = false } } diff --git a/common/src/main/java/com/shilapi/xcertplay/web/WebSession.kt b/common/src/main/java/com/shilapi/xcertplay/web/WebSession.kt index 87e367f3..4993ee98 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/WebSession.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/WebSession.kt @@ -7,9 +7,10 @@ import android.os.SystemClock import com.shilapi.xcertplay.AirPlayPersistence import com.shilapi.xcertplay.BrowserOfflinePolicy import com.shilapi.xcertplay.CarPlayAppearance -import com.shilapi.xcertplay.CarPlayBackgroundSession +import com.shilapi.xcertplay.projection.projectionSessions +import com.shilapi.xcertplay.projection.ProjectionContact +import com.shilapi.xcertplay.projection.ProjectionSession import com.shilapi.xcertplay.CarPlayMicrophoneSource -import com.shilapi.xcertplay.DiPlayBootstrap import com.shilapi.xcertplay.host.R import com.shilapi.xcertplay.i18n.* import com.shilapi.xcertplay.isDarkMode @@ -29,6 +30,7 @@ internal object WebSession { data class MediaSnapshot( val sessionId: Long, val videoEpoch: Long, val width: Int, val height: Int, val sourceState: String, val inputReady: Boolean, val frame: Frame?, val source: WebVideoSource?, + val sourceId: String? = null, val sourceName: String? = null, val maxTouchContacts: Int = 0, ) private var mediaSessionId = SecureRandom().nextLong().ushr(11) @@ -43,8 +45,10 @@ internal object WebSession { frame == null -> "starting" else -> "recovering" }, - !mediaEnded && videoActive && CarPlayBackgroundSession.snapshot()?.controller?.touchReady == true, + !mediaEnded && videoActive && projectionSessions.inputReady(inputSession), frame, videoSource, + inputSession?.descriptor?.id, inputSession?.descriptor?.displayName, + inputSession?.capabilities?.maxTouchContacts ?: 0, ) @Synchronized fun mediaMatches(sessionId: Long, epoch: Long): Boolean = @@ -52,12 +56,9 @@ internal object WebSession { /** Keep version validation and delivery in the same configuration critical section. */ @Synchronized fun mediaTouch(client: Any, sessionId: Long, epoch: Long, - contacts: List, at: Long): Boolean { - if (contacts.isNotEmpty() && (!mediaMatches(sessionId, epoch) || - CarPlayBackgroundSession.snapshot()?.controller?.touchReady != true)) { - touch.touch(client, emptyList(), at) - return false - } + contacts: List, at: Long): Boolean { + if (!mediaMatches(sessionId, epoch) || + !projectionSessions.inputReady(inputSession)) return false return touch.touch(client, contacts, at) } @@ -74,9 +75,18 @@ internal object WebSession { private set @Volatile var videoActive = false - @Volatile private var stageText = UiText(UiR.string.ui_waiting_to_connect_to_iphone) + @Volatile private var stageText = UiText(UiR.string.projection_waiting) + + fun stageDetail(context: Context): String = stageText.resolve(context) - fun stage(context: Context): String = stageText.resolve(context) + fun stage(context: Context): String { + val session = projectionSessions.snapshot() + return when { + session == null -> stageText.resolve(context) + session.statusText != null -> session.statusText!! + else -> context.getString(UiR.string.projection_active, session.descriptor.displayName) + } + } fun setStage(value: UiText) { stageText = value @@ -125,8 +135,11 @@ internal object WebSession { get() = browserViewport @Volatile private var browserViewportListener: ((Int?, Int?) -> Unit)? = null - val touch = TouchLease { contacts -> - CarPlayBackgroundSession.snapshot()?.controller?.sendTouch(contacts) ?: false + @Volatile private var inputSession: ProjectionSession? = null + val maxTouchContacts get() = projectionSessions.snapshot()?.capabilities?.maxTouchContacts ?: 0 + val touch = TouchLease(maxContacts = { maxTouchContacts }) { contacts -> + val target = inputSession + target != null && projectionSessions.touch(target, contacts) } val running get() = server?.ready == true @@ -154,21 +167,20 @@ internal object WebSession { private var offlineSubscription: AutoCloseable? = null private val preferenceExecutor = java.util.concurrent.Executors.newSingleThreadExecutor { task -> - Thread(task, "carplay-preferences").apply { isDaemon = true } + Thread(task, "projection-preferences").apply { isDaemon = true } } private val offlineMonitor = BrowserOfflineMonitor( mainHandler, - { CarPlayBackgroundSession.snapshot()?.controller }, + { projectionSessions.snapshot() }, { hasViewer }, ) { expected -> if ( experienceContext != null && - CarPlayBackgroundSession.snapshot()?.controller === expected + projectionSessions.snapshot() === expected ) { - CarPlayBackgroundSession.stop { - setStage(UiText(UiR.string.ui_browser_offline_timeout_iphone_disconnected)) - } + projectionSessions.stop(expected as ProjectionSession) + } } @@ -196,24 +208,25 @@ internal object WebSession { } fun refreshAppearance(context: Context) { - val controller = CarPlayBackgroundSession.snapshot()?.controller ?: return + val controller = projectionSessions.snapshot() ?: return preferenceExecutor.execute { - if (CarPlayBackgroundSession.snapshot()?.controller === controller) { + if (projectionSessions.snapshot() === controller) { val night = AirPlayPersistence.loadAppearance(context) .night(isDarkMode(context.resources.configuration.uiMode)) - controller.setNightMode(night) + projectionSessions.setNightMode(controller, night) } } } private fun configureAudio(context: Context) { val source = AirPlayPersistence.loadMicrophoneSource(context) + val capabilities = projectionSessions.snapshot()?.capabilities audio.configure( - AirPlayPersistence.loadBrowserAudioPlayback(context), - source == CarPlayMicrophoneSource.BROWSER, + capabilities?.audioPlayback == true && AirPlayPersistence.loadBrowserAudioPlayback(context), + capabilities?.microphone == true && source == CarPlayMicrophoneSource.BROWSER, AirPlayPersistence.loadMediaBufferMillis(context), - source != CarPlayMicrophoneSource.DISABLED, + capabilities?.microphone == true && source != CarPlayMicrophoneSource.DISABLED, ) } @@ -263,9 +276,9 @@ internal object WebSession { val now = SystemClock.elapsedRealtime() val message = when { - CarPlayBackgroundSession.hasSession() -> + projectionSessions.hasSession() -> context.getString( - UiR.string.ui_an_iphone_session_has_started_wait_for_the_connection + UiR.string.projection_busy ) phoneConnectHandler == null -> context.getString( @@ -292,13 +305,6 @@ internal object WebSession { fun start(context: Context) { if (server?.ready == true) return if (server != null) stop() - if (runCatching { DiPlayBootstrap.ensure(context) }.isFailure) { - error = - context.getString( - UiR.string.ui_authentication_resources_are_not_ready_import_them_in_settings - ) - return - } adaptiveBrowserSize = AirPlayPersistence.loadAdaptiveBrowserSize(context) configureAudio(context) code = (100000 + SecureRandom().nextInt(900000)).toString() @@ -311,7 +317,31 @@ internal object WebSession { error = null mdns = WebMdns(context.applicationContext) experienceContext = context.applicationContext - offlineSubscription = CarPlayBackgroundSession.subscribe { viewerStateChanged() } + var previousSource = inputSession + offlineSubscription = projectionSessions.subscribe { + val changed = synchronized(this) { + // Read under the configuration lock: a delayed notification cannot restore an old route. + val current = projectionSessions.snapshot() + if (previousSource === current) false else { + touch.reset() + inputSession = current + if (current == null) { + stageText = UiText(UiR.string.projection_waiting) + endVideo(generation) + } + previousSource = current + true + } + } + if (changed) { + // Versionless clients cannot distinguish old touches after a source switch. + server?.disconnectViewer() + secureServer?.disconnectViewer() + configureAudio(context) + refreshAppearance(context) + } + viewerStateChanged() + } httpsPort = WebListenSettings.httpsPort(context) tlsError = null TeslaHttpCompatibility.start(context) @@ -687,6 +717,7 @@ internal object WebSession { @Synchronized fun beginVideo(w: Int, h: Int): Long { + touch.cancel() if (mediaEnded) mediaSessionId = SecureRandom().nextLong().ushr(11) mediaEnded = false videoSource?.detach() @@ -698,6 +729,11 @@ internal object WebSession { return ++generation } + @Synchronized + fun setVideoActive(owner: Long, active: Boolean) { + if (owner == generation && !mediaEnded) videoActive = active + } + @Synchronized fun attachVideoSource(owner: Long, source: WebVideoSource) { if (owner == generation) videoSource = source @@ -719,7 +755,8 @@ internal object WebSession { @Synchronized fun endVideo(owner: Long) { - if (owner == generation) { + if (owner == generation && !mediaEnded) { + touch.cancel() mediaEnded = true videoSource?.detach() videoSource = null @@ -737,6 +774,7 @@ internal object WebSession { experienceContext = null offlineSubscription?.close() offlineSubscription = null + inputSession = null if (Looper.myLooper() == Looper.getMainLooper()) offlineMonitor.close() else mainHandler.post { if (experienceContext == null) offlineMonitor.close() } serverGeneration++ diff --git a/common/src/main/java/com/shilapi/xcertplay/web/WebVideoSource.kt b/common/src/main/java/com/shilapi/xcertplay/web/WebVideoSource.kt index fb77f56d..333ed25c 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/WebVideoSource.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/WebVideoSource.kt @@ -1,16 +1,14 @@ package com.shilapi.xcertplay.web -import com.shilapi.xcertplay.airplay.VideoCodec -import android.util.Log +import com.shilapi.xcertplay.media.VideoCodec import com.shilapi.xcertplay.media.CompressedVideoFrame import com.shilapi.xcertplay.media.HeapVideoFrame -import com.shilapi.xcertplay.media.VideoFrameDecoder import com.shilapi.xcertplay.media.EncodedVideoSink import com.shilapi.xcertplay.media.MediaCodecSupport import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong -/** Owned by one AirPlay session, including the decoder handoff and keyframe recovery. */ +/** Owned by one projection video stream, including the decoder handoff and keyframe recovery. */ internal class WebVideoSource(val technology: StreamTechnology) : EncodedVideoSink { val performance = VideoPerformance() @Volatile private var processor = "JVM" @@ -29,11 +27,7 @@ internal class WebVideoSource(val technology: StreamTechnology) : EncodedVideoSi private var lastConfig: ByteArray? = null val jpegNeeded get() = !displayed - override fun createFrameDecoder(key: ByteArray): VideoFrameDecoder? = try { - NativeVideoDecoder(key, platform = true).also { processor = it.name } - } catch (e: LinkageError) { - Log.w("WheelPlayRtc", "Native video processing unavailable; using JVM", e); null - } + fun setProcessorName(name: String) { processor = name } override fun recovery(type: Int, request: () -> Unit) { recovery[type] = request } diff --git a/common/src/test/java/com/shilapi/xcertplay/AuthenticationActivationTest.kt b/common/src/test/java/com/shilapi/xcertplay/AuthenticationActivationTest.kt index 8a67f791..bc38a8fa 100644 --- a/common/src/test/java/com/shilapi/xcertplay/AuthenticationActivationTest.kt +++ b/common/src/test/java/com/shilapi/xcertplay/AuthenticationActivationTest.kt @@ -42,34 +42,31 @@ class AuthenticationActivationTest : ComposeSettingsTest() { else emptyList() @Test - fun missingResourcesGuideToSettingsWithoutStartingWebOrCarPlay() { + fun missingResourcesAllowHostButCarPlayRemainsUnavailable() { val controller = Robolectric.buildActivity(DiPlayActivity::class.java).setup() val activity = controller.get() val ui = views(activity.window.decorView) - assertEquals(3, ui.filterIsInstance().single().selectedItemId) - assertNull(shadowOf(activity).nextStartedService) - assertFalse(WebSession.running) + assertNotNull(shadowOf(activity).nextStartedService) assertFalse(CarPlayBackgroundSession.hasSession()) - compose.onNodeWithText("从 DiPlay APK 导入").assertExists() assertFalse(ui.filterIsInstance