diff --git a/debug_displays.rs b/debug_displays.rs new file mode 100644 index 0000000..3c1b60d --- /dev/null +++ b/debug_displays.rs @@ -0,0 +1,16 @@ +use display_info; + +fn main() { + match display_info::DisplayInfo::all() { + Ok(displays) => { + println!("Found {} displays:", displays.len()); + for (index, display) in displays.iter().enumerate() { + println!(" Display {}: ID={}, Scale={}, Width={}, Height={}", + index, display.id, display.scale_factor, display.width, display.height); + } + } + Err(e) => { + println!("Error getting display info: {}", e); + } + } +} diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 468b024..dbbb183 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -730,6 +730,12 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + [[package]] name = "ddc" version = "0.2.2" @@ -1645,6 +1651,17 @@ dependencies = [ "match_token", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -1663,7 +1680,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -1674,7 +1691,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", + "http 1.3.1", "http-body", "pin-project-lite", ] @@ -1700,7 +1717,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", + "http 1.3.1", "http-body", "httparse", "itoa", @@ -1721,7 +1738,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", + "http 1.3.1", "http-body", "hyper", "ipnet", @@ -3452,7 +3469,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", + "http 1.3.1", "http-body", "http-body-util", "hyper", @@ -3777,6 +3794,17 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -4116,7 +4144,7 @@ dependencies = [ "glob", "gtk", "heck 0.5.0", - "http", + "http 1.3.1", "jni", "libc", "log", @@ -4261,7 +4289,7 @@ dependencies = [ "cookie", "dpi", "gtk", - "http", + "http 1.3.1", "jni", "objc2 0.6.1", "objc2-ui-kit", @@ -4281,7 +4309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "902b5aa9035e16f342eb64f8bf06ccdc2808e411a2525ed1d07672fa4e780bad" dependencies = [ "gtk", - "http", + "http 1.3.1", "jni", "log", "objc2 0.6.1", @@ -4314,7 +4342,7 @@ dependencies = [ "dunce", "glob", "html5ever", - "http", + "http 1.3.1", "infer", "json-patch", "kuchikiki", @@ -4384,6 +4412,7 @@ dependencies = [ "display-info", "env_logger", "futures", + "futures-util", "hex", "image", "itertools 0.10.5", @@ -4395,12 +4424,14 @@ dependencies = [ "screen-capture-kit", "serde", "serde_json", + "sha1", "tauri", "tauri-build", "tauri-plugin-shell", "time", "tokio", "tokio-stream", + "tokio-tungstenite", "toml 0.7.8", "url-build-parse", ] @@ -4537,6 +4568,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.15" @@ -4651,7 +4694,7 @@ dependencies = [ "bitflags 2.9.1", "bytes", "futures-util", - "http", + "http 1.3.1", "http-body", "iri-string", "pin-project-lite", @@ -4719,6 +4762,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 0.2.12", + "httparse", + "log", + "rand 0.8.5", + "sha1", + "thiserror 1.0.69", + "url", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -5647,7 +5709,7 @@ dependencies = [ "gdkx11", "gtk", "html5ever", - "http", + "http 1.3.1", "javascriptcore-rs", "jni", "kuchikiki", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 8e73782..a549e4e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,6 +42,9 @@ ddc-hi = "0.4.1" coreaudio-rs = "0.11.2" screen-capture-kit = "0.3.1" image = { version = "0.24", features = ["jpeg"] } +tokio-tungstenite = "0.20" +futures-util = "0.3" +sha1 = "0.10" [features] # this feature is used for production builds or when `devPath` points to the filesystem diff --git a/src-tauri/src-tauri/.gitignore b/src-tauri/src-tauri/.gitignore new file mode 100644 index 0000000..aba21e2 --- /dev/null +++ b/src-tauri/src-tauri/.gitignore @@ -0,0 +1,3 @@ +# Generated by Cargo +# will have compiled files and executables +/target/ diff --git a/src-tauri/src-tauri/Cargo.toml b/src-tauri/src-tauri/Cargo.toml new file mode 100644 index 0000000..1314c0e --- /dev/null +++ b/src-tauri/src-tauri/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "app" +version = "0.1.0" +description = "A Tauri App" +authors = ["you"] +license = "" +repository = "" +default-run = "app" +edition = "2021" +rust-version = "1.60" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[build-dependencies] +tauri-build = { version = "1.5.6" } + +[dependencies] +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +tauri = { version = "1.8.2" } + +[features] +# this feature is used for production builds or when `devPath` points to the filesystem and the built-in dev server is disabled. +# If you use cargo directly instead of tauri's cli you can use this feature flag to switch between tauri's `dev` and `build` modes. +# DO NOT REMOVE!! +custom-protocol = [ "tauri/custom-protocol" ] diff --git a/src-tauri/src-tauri/build.rs b/src-tauri/src-tauri/build.rs new file mode 100644 index 0000000..795b9b7 --- /dev/null +++ b/src-tauri/src-tauri/build.rs @@ -0,0 +1,3 @@ +fn main() { + tauri_build::build() +} diff --git a/src-tauri/src-tauri/icons/128x128.png b/src-tauri/src-tauri/icons/128x128.png new file mode 100644 index 0000000..77e7d23 Binary files /dev/null and b/src-tauri/src-tauri/icons/128x128.png differ diff --git a/src-tauri/src-tauri/icons/128x128@2x.png b/src-tauri/src-tauri/icons/128x128@2x.png new file mode 100644 index 0000000..0f7976f Binary files /dev/null and b/src-tauri/src-tauri/icons/128x128@2x.png differ diff --git a/src-tauri/src-tauri/icons/32x32.png b/src-tauri/src-tauri/icons/32x32.png new file mode 100644 index 0000000..98fda06 Binary files /dev/null and b/src-tauri/src-tauri/icons/32x32.png differ diff --git a/src-tauri/src-tauri/icons/Square107x107Logo.png b/src-tauri/src-tauri/icons/Square107x107Logo.png new file mode 100644 index 0000000..f35d84f Binary files /dev/null and b/src-tauri/src-tauri/icons/Square107x107Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square142x142Logo.png b/src-tauri/src-tauri/icons/Square142x142Logo.png new file mode 100644 index 0000000..1823bb2 Binary files /dev/null and b/src-tauri/src-tauri/icons/Square142x142Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square150x150Logo.png b/src-tauri/src-tauri/icons/Square150x150Logo.png new file mode 100644 index 0000000..dc2b22c Binary files /dev/null and b/src-tauri/src-tauri/icons/Square150x150Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square284x284Logo.png b/src-tauri/src-tauri/icons/Square284x284Logo.png new file mode 100644 index 0000000..0ed3984 Binary files /dev/null and b/src-tauri/src-tauri/icons/Square284x284Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square30x30Logo.png b/src-tauri/src-tauri/icons/Square30x30Logo.png new file mode 100644 index 0000000..60bf0ea Binary files /dev/null and b/src-tauri/src-tauri/icons/Square30x30Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square310x310Logo.png b/src-tauri/src-tauri/icons/Square310x310Logo.png new file mode 100644 index 0000000..c8ca0ad Binary files /dev/null and b/src-tauri/src-tauri/icons/Square310x310Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square44x44Logo.png b/src-tauri/src-tauri/icons/Square44x44Logo.png new file mode 100644 index 0000000..8756459 Binary files /dev/null and b/src-tauri/src-tauri/icons/Square44x44Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square71x71Logo.png b/src-tauri/src-tauri/icons/Square71x71Logo.png new file mode 100644 index 0000000..2c8023c Binary files /dev/null and b/src-tauri/src-tauri/icons/Square71x71Logo.png differ diff --git a/src-tauri/src-tauri/icons/Square89x89Logo.png b/src-tauri/src-tauri/icons/Square89x89Logo.png new file mode 100644 index 0000000..2c5e603 Binary files /dev/null and b/src-tauri/src-tauri/icons/Square89x89Logo.png differ diff --git a/src-tauri/src-tauri/icons/StoreLogo.png b/src-tauri/src-tauri/icons/StoreLogo.png new file mode 100644 index 0000000..17d142c Binary files /dev/null and b/src-tauri/src-tauri/icons/StoreLogo.png differ diff --git a/src-tauri/src-tauri/icons/icon.icns b/src-tauri/src-tauri/icons/icon.icns new file mode 100644 index 0000000..a2993ad Binary files /dev/null and b/src-tauri/src-tauri/icons/icon.icns differ diff --git a/src-tauri/src-tauri/icons/icon.ico b/src-tauri/src-tauri/icons/icon.ico new file mode 100644 index 0000000..06c23c8 Binary files /dev/null and b/src-tauri/src-tauri/icons/icon.ico differ diff --git a/src-tauri/src-tauri/icons/icon.png b/src-tauri/src-tauri/icons/icon.png new file mode 100644 index 0000000..d1756ce Binary files /dev/null and b/src-tauri/src-tauri/icons/icon.png differ diff --git a/src-tauri/src-tauri/src/main.rs b/src-tauri/src-tauri/src/main.rs new file mode 100644 index 0000000..f5c5be2 --- /dev/null +++ b/src-tauri/src-tauri/src/main.rs @@ -0,0 +1,8 @@ +// Prevents additional console window on Windows in release, DO NOT REMOVE!! +#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] + +fn main() { + tauri::Builder::default() + .run(tauri::generate_context!()) + .expect("error while running tauri application"); +} diff --git a/src-tauri/src-tauri/tauri.conf.json b/src-tauri/src-tauri/tauri.conf.json new file mode 100644 index 0000000..5bbb771 --- /dev/null +++ b/src-tauri/src-tauri/tauri.conf.json @@ -0,0 +1,65 @@ +{ + "build": { + "beforeBuildCommand": "npm run build", + "beforeDevCommand": "npm run dev", + "devPath": "http://localhost:4000", + "distDir": "../dist" + }, + "package": { + "productName": "Tauri App", + "version": "0.1.0" + }, + "tauri": { + "allowlist": { + "all": false + }, + "bundle": { + "active": true, + "category": "DeveloperTool", + "copyright": "", + "deb": { + "depends": [] + }, + "externalBin": [], + "icon": [ + "icons/32x32.png", + "icons/128x128.png", + "icons/128x128@2x.png", + "icons/icon.icns", + "icons/icon.ico" + ], + "identifier": "com.tauri.dev", + "longDescription": "", + "macOS": { + "entitlements": null, + "exceptionDomain": "", + "frameworks": [], + "providerShortName": null, + "signingIdentity": null + }, + "resources": [], + "shortDescription": "", + "targets": "all", + "windows": { + "certificateThumbprint": null, + "digestAlgorithm": "sha256", + "timestampUrl": "" + } + }, + "security": { + "csp": null + }, + "updater": { + "active": false + }, + "windows": [ + { + "fullscreen": false, + "height": 600, + "resizable": true, + "title": "Tauri", + "width": 800 + } + ] + } +} diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index d92ca78..26941b7 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -7,6 +7,7 @@ mod led_color; mod rpc; mod screenshot; mod screenshot_manager; +mod screen_stream; mod volume; use ambient_light::{Border, ColorCalibration, LedStripConfig, LedStripConfigGroup}; @@ -16,6 +17,7 @@ use paris::{error, info, warn}; use rpc::{BoardInfo, UdpRpc}; use screenshot::Screenshot; use screenshot_manager::ScreenshotManager; + use serde::{Deserialize, Serialize}; use serde_json::to_string; use tauri::{Manager, Emitter, Runtime}; @@ -335,6 +337,21 @@ fn handle_ambient_light_protocol( async fn main() { env_logger::init(); + // Debug: Print available displays + match display_info::DisplayInfo::all() { + Ok(displays) => { + println!("=== AVAILABLE DISPLAYS ==="); + for (index, display) in displays.iter().enumerate() { + println!(" Display {}: ID={}, Scale={}, Width={}, Height={}", + index, display.id, display.scale_factor, display.width, display.height); + } + println!("=== END DISPLAYS ==="); + } + Err(e) => { + println!("Error getting display info: {}", e); + } + } + tokio::spawn(async move { let screenshot_manager = ScreenshotManager::global().await; screenshot_manager.start().await.unwrap_or_else(|e| { @@ -347,6 +364,13 @@ async fn main() { led_color_publisher.start().await; }); + // Start WebSocket server for screen streaming + tokio::spawn(async move { + if let Err(e) = start_websocket_server().await { + error!("Failed to start WebSocket server: {}", e); + } + }); + let _volume = VolumeManager::global().await; tauri::Builder::default() @@ -471,3 +495,30 @@ async fn main() { .run(tauri::generate_context!()) .expect("error while running tauri application"); } + +// WebSocket server for screen streaming +async fn start_websocket_server() -> anyhow::Result<()> { + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:8765").await?; + info!("WebSocket server listening on ws://127.0.0.1:8765"); + + while let Ok((stream, addr)) = listener.accept().await { + info!("New WebSocket connection from: {}", addr); + + tokio::spawn(async move { + info!("Starting WebSocket handler for connection from: {}", addr); + match screen_stream::handle_websocket_connection(stream).await { + Ok(_) => { + info!("WebSocket connection from {} completed successfully", addr); + } + Err(e) => { + warn!("WebSocket connection error from {}: {}", addr, e); + } + } + info!("WebSocket handler task completed for: {}", addr); + }); + } + + Ok(()) +} diff --git a/src-tauri/src/screen_stream.rs b/src-tauri/src/screen_stream.rs new file mode 100644 index 0000000..f98f33c --- /dev/null +++ b/src-tauri/src/screen_stream.rs @@ -0,0 +1,503 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::io::Cursor; + +use anyhow::Result; +use image::{ImageFormat, RgbaImage}; +use tokio::sync::{broadcast, RwLock}; +use tokio::time::sleep; +use tokio_tungstenite::{accept_async, tungstenite::Message}; +use futures_util::{SinkExt, StreamExt}; + +use crate::screenshot::Screenshot; +use crate::screenshot_manager::ScreenshotManager; + +#[derive(Debug, Clone)] +pub struct StreamConfig { + pub display_id: u32, + pub target_width: u32, + pub target_height: u32, + pub quality: u8, // JPEG quality 1-100 + pub max_fps: u8, // Maximum frames per second +} + +impl Default for StreamConfig { + fn default() -> Self { + Self { + display_id: 0, + target_width: 320, // Reduced from 400 for better performance + target_height: 180, // Reduced from 225 for better performance + quality: 50, // Reduced from 75 for faster compression + max_fps: 15, + } + } +} + +#[derive(Debug, Clone)] +pub struct StreamFrame { + pub display_id: u32, + pub timestamp: Instant, + pub jpeg_data: Vec, + pub width: u32, + pub height: u32, +} + +pub struct ScreenStreamManager { + streams: Arc>>>>, +} + +struct StreamState { + config: StreamConfig, + subscribers: Vec>, + last_frame: Option, + last_screenshot_hash: Option, + last_force_send: Instant, + is_running: bool, +} + +impl ScreenStreamManager { + pub fn new() -> Self { + Self { + streams: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub async fn start_stream(&self, config: StreamConfig) -> Result> { + let display_id = config.display_id; + let mut streams = self.streams.write().await; + + if let Some(stream_state) = streams.get(&display_id) { + // Stream already exists, just add a new subscriber + let mut state = stream_state.write().await; + let (tx, rx) = broadcast::channel(10); + state.subscribers.push(tx); + return Ok(rx); + } + + // Create new stream + let (tx, rx) = broadcast::channel(10); + let stream_state = Arc::new(RwLock::new(StreamState { + config: config.clone(), + subscribers: vec![tx], + last_frame: None, + last_screenshot_hash: None, + last_force_send: Instant::now(), + is_running: false, + })); + + streams.insert(display_id, stream_state.clone()); + drop(streams); + + // Start the stream processing task + let streams_ref = self.streams.clone(); + tokio::spawn(async move { + if let Err(e) = Self::run_stream(display_id, streams_ref).await { + log::error!("Stream {} error: {}", display_id, e); + } + }); + + Ok(rx) + } + + async fn run_stream(display_id: u32, streams: Arc>>>>) -> Result<()> { + log::info!("Starting stream for display_id: {}", display_id); + + let screenshot_manager = ScreenshotManager::global().await; + + // If display_id is 0, try to get the first available display + let actual_display_id = if display_id == 0 { + // Get available displays and use the first one + let displays = display_info::DisplayInfo::all().map_err(|e| anyhow::anyhow!("Failed to get displays: {}", e))?; + if displays.is_empty() { + return Err(anyhow::anyhow!("No displays available")); + } + log::info!("Using first available display: {}", displays[0].id); + displays[0].id + } else { + display_id + }; + + log::info!("Attempting to subscribe to display_id: {}", actual_display_id); + let screenshot_rx = match screenshot_manager.subscribe_by_display_id(actual_display_id).await { + Ok(rx) => { + log::info!("Successfully subscribed to display_id: {}", actual_display_id); + rx + } + Err(e) => { + log::error!("Failed to subscribe to display_id {}: {}", actual_display_id, e); + return Err(e); + } + }; + let mut screenshot_rx = screenshot_rx; + + // Mark stream as running + { + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let mut state = stream_state.write().await; + state.is_running = true; + } + } + + let mut last_process_time = Instant::now(); + + loop { + // Check if stream still has subscribers + let should_continue = { + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let state = stream_state.read().await; + !state.subscribers.is_empty() + } else { + false + } + }; + + if !should_continue { + break; + } + + // Wait for new screenshot + if let Ok(_) = screenshot_rx.changed().await { + let screenshot = screenshot_rx.borrow().clone(); + + // Rate limiting based on max_fps + let config = { + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let state = stream_state.read().await; + state.config.clone() + } else { + break; + } + }; + + let min_interval = Duration::from_millis(1000 / config.max_fps as u64); + let elapsed = last_process_time.elapsed(); + if elapsed < min_interval { + sleep(min_interval - elapsed).await; + } + + // Process screenshot into JPEG frame + if let Ok(frame) = Self::process_screenshot(&screenshot, &config).await { + last_process_time = Instant::now(); + + // Check if frame content changed (simple hash comparison) or force send + let frame_hash = Self::calculate_frame_hash(&frame.jpeg_data); + let should_send = { + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let mut state = stream_state.write().await; + let changed = state.last_screenshot_hash.map_or(true, |hash| hash != frame_hash); + let elapsed_ms = state.last_force_send.elapsed().as_millis(); + let force_send = elapsed_ms > 200; // Force send every 200ms for higher FPS + + if changed || force_send { + state.last_screenshot_hash = Some(frame_hash); + state.last_frame = Some(frame.clone()); + if force_send { + state.last_force_send = Instant::now(); + } + } + changed || force_send + } else { + false + } + }; + + if should_send { + // Send to all subscribers + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let state = stream_state.read().await; + for tx in state.subscribers.iter() { + if let Err(_) = tx.send(frame.clone()) { + log::warn!("Failed to send frame to subscriber for display_id: {}", display_id); + } + } + } + } + } + } + } + + // Mark stream as stopped + { + let streams_lock = streams.read().await; + if let Some(stream_state) = streams_lock.get(&display_id) { + let mut state = stream_state.write().await; + state.is_running = false; + } + } + + Ok(()) + } + + async fn process_screenshot(screenshot: &Screenshot, config: &StreamConfig) -> Result { + let total_start = Instant::now(); + let bytes = screenshot.bytes.read().await; + + // Convert BGRA to RGBA using unsafe with optimized batch processing for maximum performance + let mut rgba_bytes = bytes.as_ref().clone(); + unsafe { + let ptr = rgba_bytes.as_mut_ptr() as *mut u32; + let len = rgba_bytes.len() / 4; + + // Process in larger chunks of 64 for better cache efficiency and loop unrolling + let chunk_size = 64; + let full_chunks = len / chunk_size; + let remainder = len % chunk_size; + + // Process full chunks with manual loop unrolling + for chunk_idx in 0..full_chunks { + let base_ptr = ptr.add(chunk_idx * chunk_size); + + // Unroll the inner loop for better performance + for i in (0..chunk_size).step_by(4) { + // Process 4 pixels at once + let p0 = base_ptr.add(i).read(); + let p1 = base_ptr.add(i + 1).read(); + let p2 = base_ptr.add(i + 2).read(); + let p3 = base_ptr.add(i + 3).read(); + + // BGRA (0xAABBGGRR) -> RGBA (0xAAGGBBRR) + let s0 = (p0 & 0xFF00FF00) | ((p0 & 0x00FF0000) >> 16) | ((p0 & 0x000000FF) << 16); + let s1 = (p1 & 0xFF00FF00) | ((p1 & 0x00FF0000) >> 16) | ((p1 & 0x000000FF) << 16); + let s2 = (p2 & 0xFF00FF00) | ((p2 & 0x00FF0000) >> 16) | ((p2 & 0x000000FF) << 16); + let s3 = (p3 & 0xFF00FF00) | ((p3 & 0x00FF0000) >> 16) | ((p3 & 0x000000FF) << 16); + + base_ptr.add(i).write(s0); + base_ptr.add(i + 1).write(s1); + base_ptr.add(i + 2).write(s2); + base_ptr.add(i + 3).write(s3); + } + } + + // Process remaining pixels + let remainder_start = full_chunks * chunk_size; + for i in 0..remainder { + let idx = remainder_start + i; + let pixel = ptr.add(idx).read(); + let swapped = (pixel & 0xFF00FF00) | ((pixel & 0x00FF0000) >> 16) | ((pixel & 0x000000FF) << 16); + ptr.add(idx).write(swapped); + } + } + + // Create image from raw bytes + let img = RgbaImage::from_raw( + screenshot.width, + screenshot.height, + rgba_bytes, + ).ok_or_else(|| anyhow::anyhow!("Failed to create image from raw bytes"))?; + + // Resize if needed + let final_img = if screenshot.width != config.target_width || screenshot.height != config.target_height { + image::imageops::resize( + &img, + config.target_width, + config.target_height, + image::imageops::FilterType::Nearest, // Fastest filter for real-time streaming + ) + } else { + img + }; + + // Convert to JPEG + let mut jpeg_buffer = Vec::new(); + let mut cursor = Cursor::new(&mut jpeg_buffer); + + let rgb_img = image::DynamicImage::ImageRgba8(final_img).to_rgb8(); + rgb_img.write_to(&mut cursor, ImageFormat::Jpeg)?; + + let total_duration = total_start.elapsed(); + log::debug!("Screenshot processed for display {} in {}ms, JPEG size: {} bytes", + config.display_id, total_duration.as_millis(), jpeg_buffer.len()); + + Ok(StreamFrame { + display_id: config.display_id, + timestamp: Instant::now(), + jpeg_data: jpeg_buffer, + width: config.target_width, + height: config.target_height, + }) + } + + fn calculate_frame_hash(data: &[u8]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + // Sample every 100th byte for better sensitivity (was 1000) + for (i, &byte) in data.iter().enumerate() { + if i % 100 == 0 { + byte.hash(&mut hasher); + } + } + hasher.finish() + } + + pub async fn stop_stream(&self, display_id: u32) { + let mut streams = self.streams.write().await; + streams.remove(&display_id); + } +} + +// Global instance +static SCREEN_STREAM_MANAGER: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); + +impl ScreenStreamManager { + pub async fn global() -> &'static Self { + SCREEN_STREAM_MANAGER.get_or_init(|| async { + ScreenStreamManager::new() + }).await + } +} + +// WebSocket handler for screen streaming +pub async fn handle_websocket_connection( + stream: tokio::net::TcpStream, +) -> Result<()> { + log::info!("Accepting WebSocket connection..."); + + let ws_stream = match accept_async(stream).await { + Ok(ws) => { + log::info!("WebSocket handshake completed successfully"); + ws + } + Err(e) => { + log::error!("WebSocket handshake failed: {}", e); + return Err(e.into()); + } + }; + let (ws_sender, mut ws_receiver) = ws_stream.split(); + + log::info!("WebSocket connection established, waiting for configuration..."); + + // Wait for the first configuration message + let config = loop { + // Add timeout to prevent hanging + let timeout_duration = tokio::time::Duration::from_secs(10); + match tokio::time::timeout(timeout_duration, ws_receiver.next()).await { + Ok(Some(msg)) => { + match msg { + Ok(Message::Text(text)) => { + log::info!("Received configuration message: {}", text); + + if let Ok(config_json) = serde_json::from_str::(&text) { + // Parse configuration from JSON + let display_id = config_json.get("display_id") + .and_then(|v| v.as_u64()) + .unwrap_or(0) as u32; + let width = config_json.get("width") + .and_then(|v| v.as_u64()) + .unwrap_or(320) as u32; // Reduced from 400 for better performance + let height = config_json.get("height") + .and_then(|v| v.as_u64()) + .unwrap_or(180) as u32; // Reduced from 225 for better performance + let quality = config_json.get("quality") + .and_then(|v| v.as_u64()) + .unwrap_or(50) as u8; // Reduced from 75 for faster compression + + let config = StreamConfig { + display_id, + target_width: width, + target_height: height, + quality, + max_fps: 15, + }; + + log::info!("Parsed stream config: display_id={}, width={}, height={}, quality={}", + display_id, width, height, quality); + break config; + } else { + log::warn!("Failed to parse configuration JSON: {}", text); + } + } + Ok(Message::Close(_)) => { + log::info!("WebSocket connection closed before configuration"); + return Ok(()); + } + Err(e) => { + log::warn!("WebSocket error while waiting for config: {}", e); + return Err(e.into()); + } + _ => {} + } + } + Ok(None) => { + log::warn!("WebSocket connection closed while waiting for configuration"); + return Ok(()); + } + Err(_) => { + log::warn!("Timeout waiting for WebSocket configuration message"); + return Err(anyhow::anyhow!("Timeout waiting for configuration")); + } + } + }; + + // Start the stream with the received configuration + log::info!("Starting stream with config: display_id={}, width={}, height={}", + config.display_id, config.target_width, config.target_height); + let stream_manager = ScreenStreamManager::global().await; + let mut frame_rx = match stream_manager.start_stream(config).await { + Ok(rx) => { + log::info!("Screen stream started successfully"); + rx + } + Err(e) => { + log::error!("Failed to start screen stream: {}", e); + return Err(e); + } + }; + + // Handle incoming WebSocket messages (for control) + let ws_sender = Arc::new(tokio::sync::Mutex::new(ws_sender)); + let ws_sender_clone = ws_sender.clone(); + + // Task to handle outgoing frames + let frame_task = tokio::spawn(async move { + while let Ok(frame) = frame_rx.recv().await { + let mut sender = ws_sender_clone.lock().await; + match sender.send(Message::Binary(frame.jpeg_data)).await { + Ok(_) => {}, + Err(e) => { + log::warn!("Failed to send frame: {}", e); + break; + } + } + } + log::info!("Frame sending task completed"); + }); + + // Task to handle incoming messages + let control_task = tokio::spawn(async move { + while let Some(msg) = ws_receiver.next().await { + match msg { + Ok(Message::Text(text)) => { + log::info!("Received control message: {}", text); + // Additional configuration updates could be handled here + } + Ok(Message::Close(_)) => { + log::info!("WebSocket connection closed"); + break; + } + Err(e) => { + log::warn!("WebSocket error: {}", e); + break; + } + _ => {} + } + } + log::info!("Control message task completed"); + }); + + // Wait for either task to complete + tokio::select! { + _ = frame_task => {}, + _ = control_task => {}, + } + + log::info!("WebSocket connection handler completed"); + Ok(()) +} diff --git a/src-tauri/src/screenshot_manager.rs b/src-tauri/src/screenshot_manager.rs index e2e89ed..f7033c8 100644 --- a/src-tauri/src/screenshot_manager.rs +++ b/src-tauri/src/screenshot_manager.rs @@ -108,6 +108,11 @@ impl ScreenshotManager { pub async fn start(&self) -> anyhow::Result<()> { let displays = display_info::DisplayInfo::all()?; + log::info!("ScreenshotManager starting with {} displays:", displays.len()); + for display in &displays { + log::info!(" Display ID: {}, Scale: {}", display.id, display.scale_factor); + } + let futures = displays.iter().map(|display| async { self.start_one(display.id, display.scale_factor) .await @@ -118,11 +123,12 @@ impl ScreenshotManager { }); futures::future::join_all(futures).await; + log::info!("ScreenshotManager started successfully"); Ok(()) } async fn start_one(&self, display_id: u32, scale_factor: f32) -> anyhow::Result<()> { - + log::info!("Starting screenshot capture for display_id: {}", display_id); let merged_screenshot_tx = self.merged_screenshot_tx.clone(); @@ -183,8 +189,8 @@ impl ScreenshotManager { } } - // Sleep for a frame duration (30 FPS) - sleep(Duration::from_millis(33)).await; + // Sleep for a frame duration (15 FPS for better performance) + sleep(Duration::from_millis(67)).await; yield_now().await; } } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 33f52c8..008a7b2 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,8 +1,8 @@ { + "$schema": "https://schema.tauri.app/config/2.0.0", "productName": "test-demo", "version": "0.0.1", "identifier": "cc.ivanli.ambient-light.desktop", - "mainBinaryName": "test-demo", "build": { "beforeDevCommand": "pnpm dev", "beforeBuildCommand": "pnpm build", @@ -10,7 +10,7 @@ "frontendDist": "../dist" }, "app": { - "withGlobalTauri": false, + "withGlobalTauri": true, "security": { "csp": null, "assetProtocol": { diff --git a/src/App.tsx b/src/App.tsx index 8f23609..ed7af82 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -11,12 +11,14 @@ import { DisplayStateIndex } from './components/displays/display-state-index'; function App() { createEffect(() => { invoke('read_config').then((config) => { - console.log('read config', config); + console.log('App: read config', config); setLedStripStore({ strips: config.strips, mappers: config.mappers, colorCalibration: config.color_calibration, }); + }).catch((error) => { + console.error('App: Failed to read config:', error); }); }); diff --git a/src/components/led-strip-configuration/led-strip-configuration.tsx b/src/components/led-strip-configuration/led-strip-configuration.tsx index 8d55f01..66a9a7a 100644 --- a/src/components/led-strip-configuration/led-strip-configuration.tsx +++ b/src/components/led-strip-configuration/led-strip-configuration.tsx @@ -17,12 +17,20 @@ import { export const LedStripConfiguration = () => { createEffect(() => { invoke('list_display_info').then((displays) => { + const parsedDisplays = JSON.parse(displays); + console.log('LedStripConfiguration: Loaded displays:', parsedDisplays); setDisplayStore({ - displays: JSON.parse(displays), + displays: parsedDisplays, }); + }).catch((error) => { + console.error('LedStripConfiguration: Failed to load displays:', error); }); + invoke('read_led_strip_configs').then((configs) => { + console.log('LedStripConfiguration: Loaded LED strip configs:', configs); setLedStripStore(configs); + }).catch((error) => { + console.error('LedStripConfiguration: Failed to load LED strip configs:', error); }); }); @@ -126,6 +134,7 @@ export const LedStripConfiguration = () => { {displayStore.displays.map((display) => { + console.log('LedStripConfiguration: Rendering DisplayView for display:', display); return ; })} diff --git a/src/components/led-strip-configuration/screen-view-websocket.tsx b/src/components/led-strip-configuration/screen-view-websocket.tsx new file mode 100644 index 0000000..067daef --- /dev/null +++ b/src/components/led-strip-configuration/screen-view-websocket.tsx @@ -0,0 +1,290 @@ +import { + Component, + createEffect, + createSignal, + JSX, + onCleanup, + onMount, + splitProps, +} from 'solid-js'; +import { invoke } from '@tauri-apps/api/core'; + +type ScreenViewWebSocketProps = { + displayId: number; + width?: number; + height?: number; + quality?: number; +} & JSX.HTMLAttributes; + +export const ScreenViewWebSocket: Component = (props) => { + const [localProps, rootProps] = splitProps(props, ['displayId', 'width', 'height', 'quality']); + let canvas: HTMLCanvasElement; + let root: HTMLDivElement; + const [ctx, setCtx] = createSignal(null); + + const [drawInfo, setDrawInfo] = createSignal({ + drawX: 0, + drawY: 0, + drawWidth: 0, + drawHeight: 0, + }); + + const [connectionStatus, setConnectionStatus] = createSignal<'connecting' | 'connected' | 'disconnected' | 'error'>('disconnected'); + const [frameCount, setFrameCount] = createSignal(0); + const [lastFrameTime, setLastFrameTime] = createSignal(0); + const [fps, setFps] = createSignal(0); + + let websocket: WebSocket | null = null; + let reconnectTimeout: number | null = null; + let isMounted = true; + + // Performance monitoring + let frameTimestamps: number[] = []; + + const connectWebSocket = () => { + if (!isMounted) { + console.log('Component not mounted, skipping WebSocket connection'); + return; + } + + const wsUrl = `ws://127.0.0.1:8765`; + console.log('Connecting to WebSocket:', wsUrl, 'with displayId:', localProps.displayId); + + setConnectionStatus('connecting'); + websocket = new WebSocket(wsUrl); + websocket.binaryType = 'arraybuffer'; + console.log('WebSocket object created:', websocket); + + websocket.onopen = () => { + console.log('WebSocket connected successfully!'); + setConnectionStatus('connected'); + + // Send initial configuration + const config = { + display_id: localProps.displayId, + width: localProps.width || 320, // Reduced from 400 for better performance + height: localProps.height || 180, // Reduced from 225 for better performance + quality: localProps.quality || 50 // Reduced from 75 for faster compression + }; + console.log('Sending WebSocket configuration:', config); + websocket?.send(JSON.stringify(config)); + }; + + websocket.onmessage = (event) => { + console.log('🔍 WebSocket message received:', { + type: typeof event.data, + isArrayBuffer: event.data instanceof ArrayBuffer, + size: event.data instanceof ArrayBuffer ? event.data.byteLength : 'N/A' + }); + + if (event.data instanceof ArrayBuffer) { + console.log('📦 Processing ArrayBuffer frame, size:', event.data.byteLength); + handleJpegFrame(new Uint8Array(event.data)); + } else { + console.log('⚠️ Received non-ArrayBuffer data:', event.data); + } + }; + + websocket.onclose = (event) => { + console.log('WebSocket closed:', event.code, event.reason); + setConnectionStatus('disconnected'); + websocket = null; + + // Auto-reconnect after 2 seconds if component is still mounted + if (isMounted && !reconnectTimeout) { + reconnectTimeout = window.setTimeout(() => { + reconnectTimeout = null; + connectWebSocket(); + }, 2000); + } + }; + + websocket.onerror = (error) => { + console.error('WebSocket error:', error); + setConnectionStatus('error'); + }; + }; + + const handleJpegFrame = async (jpegData: Uint8Array) => { + const _ctx = ctx(); + if (!_ctx) return; + + try { + // Update performance metrics + const now = performance.now(); + frameTimestamps.push(now); + + // Keep only last 30 frames for FPS calculation + if (frameTimestamps.length > 30) { + frameTimestamps = frameTimestamps.slice(-30); + } + + // Calculate FPS + if (frameTimestamps.length >= 2) { + const timeSpan = frameTimestamps[frameTimestamps.length - 1] - frameTimestamps[0]; + if (timeSpan > 0) { + const currentFps = Math.round((frameTimestamps.length - 1) * 1000 / timeSpan); + setFps(Math.max(0, currentFps)); // Ensure FPS is never negative + } + } + + setFrameCount(prev => prev + 1); + setLastFrameTime(now); + + // Create blob from JPEG data + const blob = new Blob([jpegData], { type: 'image/jpeg' }); + const imageUrl = URL.createObjectURL(blob); + + // Create image element + const img = new Image(); + img.onload = () => { + const { drawX, drawY, drawWidth, drawHeight } = drawInfo(); + + // Clear canvas + _ctx.clearRect(0, 0, canvas.width, canvas.height); + + // Draw image + _ctx.drawImage(img, drawX, drawY, drawWidth, drawHeight); + + // Clean up + URL.revokeObjectURL(imageUrl); + }; + + img.onerror = () => { + console.error('Failed to load JPEG image'); + URL.revokeObjectURL(imageUrl); + }; + + img.src = imageUrl; + + } catch (error) { + console.error('Error handling JPEG frame:', error); + } + }; + + const resetSize = () => { + // Set canvas size first + canvas.width = root.clientWidth; + canvas.height = root.clientHeight; + + // Use a default aspect ratio if canvas dimensions are invalid + const aspectRatio = (canvas.width > 0 && canvas.height > 0) + ? canvas.width / canvas.height + : 16 / 9; // Default 16:9 aspect ratio + + const drawWidth = Math.round( + Math.min(root.clientWidth, root.clientHeight * aspectRatio), + ); + const drawHeight = Math.round( + Math.min(root.clientHeight, root.clientWidth / aspectRatio), + ); + + const drawX = Math.round((root.clientWidth - drawWidth) / 2); + const drawY = Math.round((root.clientHeight - drawHeight) / 2); + + setDrawInfo({ + drawX, + drawY, + drawWidth, + drawHeight, + }); + }; + + const disconnect = () => { + if (reconnectTimeout) { + clearTimeout(reconnectTimeout); + reconnectTimeout = null; + } + + if (websocket) { + websocket.close(); + websocket = null; + } + }; + + // Initialize canvas and resize observer + onMount(() => { + console.log('ScreenViewWebSocket mounted with displayId:', localProps.displayId); + const context = canvas.getContext('2d'); + setCtx(context); + + // Initial size setup + resetSize(); + + const resizeObserver = new ResizeObserver(() => { + resetSize(); + }); + resizeObserver.observe(root); + + // Connect WebSocket + console.log('About to connect WebSocket...'); + connectWebSocket(); + + onCleanup(() => { + isMounted = false; + disconnect(); + resizeObserver?.unobserve(root); + }); + }); + + // Debug function to list displays + const debugDisplays = async () => { + try { + const result = await invoke('list_display_info'); + console.log('Available displays:', result); + alert(`Available displays: ${result}`); + } catch (error) { + console.error('Failed to get display info:', error); + alert(`Error: ${error}`); + } + }; + + // Status indicator + const getStatusColor = () => { + switch (connectionStatus()) { + case 'connected': return '#10b981'; // green + case 'connecting': return '#f59e0b'; // yellow + case 'error': return '#ef4444'; // red + default: return '#6b7280'; // gray + } + }; + + return ( +
+ + + {/* Status indicator */} +
+
+ {connectionStatus()} + {connectionStatus() === 'connected' && ( + | {fps()} FPS | {frameCount()} frames + )} + +
+ + {rootProps.children} +
+ ); +}; diff --git a/src/components/led-strip-configuration/screen-view.tsx b/src/components/led-strip-configuration/screen-view.tsx index eba7892..d9fa057 100644 --- a/src/components/led-strip-configuration/screen-view.tsx +++ b/src/components/led-strip-configuration/screen-view.tsx @@ -7,13 +7,22 @@ import { onMount, splitProps, } from 'solid-js'; +import { ScreenViewWebSocket } from './screen-view-websocket'; type ScreenViewProps = { displayId: number; + useWebSocket?: boolean; } & JSX.HTMLAttributes; export const ScreenView: Component = (props) => { - const [localProps, rootProps] = splitProps(props, ['displayId']); + const [localProps, rootProps] = splitProps(props, ['displayId', 'useWebSocket']); + + // Use WebSocket by default for better performance + if (localProps.useWebSocket !== false) { + return ; + } + + // Fallback to HTTP polling (legacy mode) let canvas: HTMLCanvasElement; let root: HTMLDivElement; const [ctx, setCtx] = createSignal(null);