src-tauri/Cargo.toml

@@ -13,7 +13,7 @@ edition = "2021"
 tauri-build = { version = "1.2", features = [] }
 
 [dependencies]
-tauri = { version = "1.2", features = ["shell-open"] }
+tauri = { version = "1.2", features = [ "protocol-all", "shell-open"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 core-graphics = "0.22.3"
@@ -28,8 +28,8 @@ url-build-parse = "9.0.0"
 color_space = "0.5.3"
 hex = "0.4.3"
 toml = "0.7.3"
-paho-mqtt = "0.12.1"
+# paho-mqtt = "0.12.1"  # Temporarily disabled due to CMake issues
-time = {version="0.3.20", features= ["formatting"] }
+time = {version="0.3.35", features= ["formatting"] }
 itertools = "0.10.5"
 core-foundation = "0.9.3"
 tokio-stream = "0.1.14"
@@ -37,7 +37,7 @@ mdns-sd = "0.7.2"
 futures = "0.3.28"
 ddc-hi = "0.4.1"
 coreaudio-rs = "0.11.2"
-rust_swift_screencapture = { version = "0.1.1", path = "../../../../demo/rust-swift-screencapture" }
+screen-capture-kit = "0.3.1"
 
 [features]
 # this feature is used for production builds or when `devPath` points to the filesystem

src-tauri/src/ambient_light/publisher.rs

@@ -88,21 +88,21 @@ impl LedColorsPublisher {
                     }
                 }
 
-                // match display_colors_tx.send((
+                match display_colors_tx.send((
-                //     display_id,
+                    display_id,
-                //     colors_copy
+                    colors_copy
-                //         .into_iter()
+                        .into_iter()
-                //         .map(|color| color.get_rgb())
+                        .map(|color| color.get_rgb())
-                //         .flatten()
+                        .flatten()
-                //         .collect::<Vec<_>>(),
+                        .collect::<Vec<_>>(),
-                // )) {
+                )) {
-                //     Ok(_) => {
+                    Ok(_) => {
-                //         // log::info!("sent colors: {:?}", color_len);
+                        // log::info!("sent colors: {:?}", color_len);
-                //     }
+                    }
-                //     Err(err) => {
+                    Err(err) => {
-                //         warn!("Failed to send display_colors: {}", err);
+                        warn!("Failed to send display_colors: {}", err);
-                //     }
+                    }
-                // };
+                };
 
                 // Check if the inner task version changed
                 let version = internal_tasks_version.read().await.clone();
@@ -127,7 +127,7 @@ impl LedColorsPublisher {
     ) {
         let sorted_colors_tx = self.sorted_colors_tx.clone();
         let colors_tx = self.colors_tx.clone();
-        log::debug!("start all_colors_worker");
+
 
         tokio::spawn(async move {
             for _ in 0..10 {
@@ -137,7 +137,7 @@ impl LedColorsPublisher {
                 let mut all_colors: Vec<Option<Vec<u8>>> = vec![None; display_ids.len()];
                 let mut start: tokio::time::Instant = tokio::time::Instant::now();
 
-                log::debug!("start all_colors_worker task");
+
                 loop {
                     let color_info = display_colors_rx.recv().await;
 
@@ -186,7 +186,7 @@ impl LedColorsPublisher {
                                 warn!("Failed to send sorted colors: {}", err);
                             }
                         };
-                        log::debug!("tick: {}ms", start.elapsed().as_millis());
+
                         start = tokio::time::Instant::now();
                     }
                 }
@@ -195,7 +195,7 @@ impl LedColorsPublisher {
     }
 
     pub async fn start(&self) {
-        log::info!("start colors worker");
+
 
         let config_manager = ConfigManager::global().await;
         let mut config_receiver = config_manager.clone_config_update_receiver();
@@ -203,9 +203,7 @@ impl LedColorsPublisher {
 
         self.handle_config_change(configs).await;
 
-        log::info!("waiting for config update...");
         while config_receiver.changed().await.is_ok() {
-            log::info!("config updated, restart inner tasks...");
             let configs = config_receiver.borrow().clone();
             self.handle_config_change(configs).await;
         }
@@ -300,16 +298,28 @@ impl LedColorsPublisher {
             if group.end > group.start {
                 for i in group.pos - display_led_offset..group_size + group.pos - display_led_offset
                 {
-                    let bytes = colors[i].as_bytes();
+                    if i < colors.len() {
-                    buffer.append(&mut bytes.to_vec());
+                        let bytes = colors[i].as_bytes();
+                        buffer.append(&mut bytes.to_vec());
+                    } else {
+                        log::warn!("Index {} out of bounds for colors array of length {}", i, colors.len());
+                        // Add black color as fallback
+                        buffer.append(&mut vec![0, 0, 0]);
+                    }
                 }
             } else {
                 for i in (group.pos - display_led_offset
                     ..group_size + group.pos - display_led_offset)
                     .rev()
                 {
-                    let bytes = colors[i].as_bytes();
+                    if i < colors.len() {
-                    buffer.append(&mut bytes.to_vec());
+                        let bytes = colors[i].as_bytes();
+                        buffer.append(&mut bytes.to_vec());
+                    } else {
+                        log::warn!("Index {} out of bounds for colors array of length {}", i, colors.len());
+                        // Add black color as fallback
+                        buffer.append(&mut vec![0, 0, 0]);
+                    }
                 }
             }
 
@@ -350,7 +360,7 @@ impl LedColorsPublisher {
         let mut screenshots = HashMap::new();
 
         loop {
-            log::info!("waiting merged screenshot...");
+
             let screenshot = merged_screenshot_receiver.recv().await;
 
             if let Err(err) = screenshot {
@@ -382,7 +392,7 @@ impl LedColorsPublisher {
                         .filter(|(_, c)| c.display_id == display_id);
 
                     let screenshot = screenshots.get(&display_id).unwrap();
-                    log::debug!("screenshot updated: {:?}", display_id);
+
 
                     let points: Vec<_> = led_strip_configs
                         .clone()
@@ -412,7 +422,7 @@ impl LedColorsPublisher {
                     led_start = led_end;
                 }
 
-                log::debug!("got all colors configs: {:?}", colors_configs.len());
+
 
                 return Ok(AllColorConfig {
                     sample_point_groups: colors_configs,

src-tauri/src/main.rs

@@ -269,7 +269,7 @@ async fn main() {
 
             let url = url.unwrap();
 
-            let re = regex::Regex::new(r"^/displays/(\d+)$").unwrap();
+            let re = regex::Regex::new(r"^/(\d+)$").unwrap();
             let path = url.path;
             let captures = re.captures(path.as_str());
 
@@ -287,6 +287,7 @@ async fn main() {
 
             let bytes = tokio::task::block_in_place(move || {
                 tauri::async_runtime::block_on(async move {
+
                     let screenshot_manager = ScreenshotManager::global().await;
                     let rx: Result<tokio::sync::watch::Receiver<Screenshot>, anyhow::Error> =
                         screenshot_manager.subscribe_by_display_id(display_id).await;
@@ -305,7 +306,7 @@ async fn main() {
                         anyhow::bail!("Display#{}: no screenshot.", display_id);
                     }
 
-                    log::debug!("Display#{}: screenshot size: {}", display_id, bytes.len());
+
 
                     let (scale_factor_x, scale_factor_y, width, height) = if url.query.is_some()
                         && url.query.as_ref().unwrap().contains_key("height")
@@ -368,6 +369,7 @@ async fn main() {
                         }
                     }
 
+
                     Ok(rgba_buffer.clone())
                 })
             });

src-tauri/src/screenshot_manager.rs

@@ -6,9 +6,11 @@ use core_graphics::display::{
 };
 use core_graphics::geometry::{CGPoint, CGRect, CGSize};
 use paris::{info, warn};
-use rust_swift_screencapture::display::CGDisplayId;
+use screen_capture_kit::shareable_content::{SCDisplay, SCShareableContent};
+use screen_capture_kit::stream::{SCStream, SCStreamConfiguration, SCContentFilter, SCStreamOutput};
+use screen_capture_kit::stream::SCStreamDelegate;
 use tauri::async_runtime::RwLock;
-use tokio::sync::{broadcast, watch, Mutex, OnceCell};
+use tokio::sync::{broadcast, watch, OnceCell};
 use tokio::task::yield_now;
 use tokio::time::sleep;
 
@@ -20,7 +22,7 @@ pub fn get_display_colors(
     sample_points: &Vec<Vec<LedSamplePoints>>,
     bound_scale_factor: f32,
 ) -> anyhow::Result<Vec<LedColor>> {
-    log::debug!("take_screenshot");
+
     let cg_display = CGDisplay::new(display_id);
 
     let mut colors = vec![];
@@ -112,7 +114,7 @@ impl ScreenshotManager {
                 .unwrap_or_else(|err| {
                     warn!("start_one failed: display_id: {}, err: {}", display.id, err);
                 });
-            info!("start_one finished: display_id: {}", display.id);
+
         });
 
         futures::future::join_all(futures).await;
@@ -120,6 +122,8 @@ impl ScreenshotManager {
     }
 
     async fn start_one(&self, display_id: u32, scale_factor: f32) -> anyhow::Result<()> {
+
+
         let merged_screenshot_tx = self.merged_screenshot_tx.clone();
 
         let (tx, _) = watch::channel(Screenshot::new(
@@ -138,45 +142,98 @@ impl ScreenshotManager {
 
         drop(channels);
 
+
+
+        // Implement screen capture using screen-capture-kit
         loop {
-            let display = rust_swift_screencapture::display::Display::new(display_id);
+            match Self::capture_display_screenshot(display_id, scale_factor).await {
-            let mut frame_rx = display.subscribe_frame().await;
+                Ok(screenshot) => {
+                    let tx_for_send = tx.read().await;
+                    let merged_screenshot_tx = merged_screenshot_tx.write().await;
 
-            display.start_capture(30).await;
+                    if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
-
+                        // log::warn!("merged_screenshot_tx.send failed: {}", err);
-            let tx_for_send = tx.read().await;
+                    }
-
+                    if let Err(err) = tx_for_send.send(screenshot.clone()) {
-            while frame_rx.changed().await.is_ok() {
+                        log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
-                let frame = frame_rx.borrow().clone();
+                    }
-                let screenshot = Screenshot::new(
-                    display_id,
-                    frame.height as u32,
-                    frame.width as u32,
-                    frame.bytes_per_row as usize,
-                    frame.bytes,
-                    scale_factor,
-                    scale_factor,
-                );
-                let merged_screenshot_tx = merged_screenshot_tx.write().await;
-                if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
-                    // log::warn!("merged_screenshot_tx.send failed: {}", err);
-                }
-                if let Err(err) = tx_for_send.send(screenshot.clone()) {
-                    log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
-                } else {
-                    log::debug!("screenshot: {:?}", screenshot);
                 }
+                Err(err) => {
+                    warn!("Failed to capture screenshot for display {}: {}", display_id, err);
+                    // Create a fallback empty screenshot to maintain the interface
+                    let screenshot = Screenshot::new(
+                        display_id,
+                        1080,
+                        1920,
+                        1920 * 4, // Assuming RGBA format
+                        Arc::new(vec![0u8; 1920 * 1080 * 4]),
+                        scale_factor,
+                        scale_factor,
+                    );
 
-                yield_now().await;
+                    let tx_for_send = tx.read().await;
+                    let merged_screenshot_tx = merged_screenshot_tx.write().await;
+
+                    if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
+                        // log::warn!("merged_screenshot_tx.send failed: {}", err);
+                    }
+                    if let Err(err) = tx_for_send.send(screenshot.clone()) {
+                        log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
+                    }
+                }
             }
-            sleep(Duration::from_secs(5)).await;
+
-            info!(
+            // Sleep for a frame duration (30 FPS)
-                "display {} frame_rx.changed() failed, try to restart",
+            sleep(Duration::from_millis(33)).await;
-                display_id
+            yield_now().await;
-            );
         }
     }
 
+    async fn capture_display_screenshot(display_id: u32, scale_factor: f32) -> anyhow::Result<Screenshot> {
+        // For now, use the existing CGDisplay approach as a fallback
+        // TODO: Implement proper screen-capture-kit integration
+
+
+        let cg_display = CGDisplay::new(display_id);
+        let bounds = cg_display.bounds();
+
+
+
+        let cg_image = CGDisplay::screenshot(
+            bounds,
+            kCGWindowListOptionOnScreenOnly,
+            kCGNullWindowID,
+            kCGWindowImageDefault,
+        )
+        .ok_or_else(|| anyhow::anyhow!("Display#{}: take screenshot failed - possibly no screen recording permission", display_id))?;
+
+        let bitmap = cg_image.data();
+        let width = cg_image.width() as u32;
+        let height = cg_image.height() as u32;
+        let bytes_per_row = cg_image.bytes_per_row();
+
+
+
+        // Convert CFData to Vec<u8>
+        let data_ptr = bitmap.bytes().as_ptr();
+        let data_len = bitmap.len() as usize;
+        let screenshot_data = unsafe {
+            std::slice::from_raw_parts(data_ptr, data_len).to_vec()
+        };
+
+
+
+        Ok(Screenshot::new(
+            display_id,
+            height,
+            width,
+            bytes_per_row,
+            Arc::new(screenshot_data),
+            scale_factor,
+            scale_factor,
+        ))
+    }
+
     pub fn get_sorted_colors(colors: &Vec<u8>, mappers: &Vec<SamplePointMapper>) -> Vec<u8> {
         let total_leds = mappers
             .iter()
@@ -232,7 +289,7 @@ impl ScreenshotManager {
 
     pub async fn subscribe_by_display_id(
         &self,
-        display_id: CGDisplayId,
+        display_id: u32,
     ) -> anyhow::Result<watch::Receiver<Screenshot>> {
         let channels = self.channels.read().await;
         if let Some(tx) = channels.get(&display_id) {

src-tauri/tauri.conf.json

@@ -16,6 +16,13 @@
       "shell": {
         "all": false,
         "open": true
+      },
+      "protocol": {
+        "all": true,
+        "asset": true,
+        "assetScope": [
+          "**"
+        ]
       }
     },
     "bundle": {

src/components/led-strip-configuration/led-strip-configuration.tsx

@@ -13,6 +13,7 @@ import {
   LedStripConfigurationContextType,
 } from '../../contexts/led-strip-configuration.context';
 
+
 export const LedStripConfiguration = () => {
   createEffect(() => {
     invoke<string>('list_display_info').then((displays) => {
@@ -45,11 +46,17 @@ export const LedStripConfiguration = () => {
   // listen to led_colors_changed event
   createEffect(() => {
     const unlisten = listen<Uint8ClampedArray>('led_colors_changed', (event) => {
+      console.log('Received led_colors_changed event:', {
+        hidden: window.document.hidden,
+        colorsLength: event.payload.length,
+        firstFewColors: Array.from(event.payload.slice(0, 12))
+      });
       if (!window.document.hidden) {
         const colors = event.payload;
         setLedStripStore({
           colors,
         });
+        console.log('Updated ledStripStore.colors with length:', colors.length);
       }
     });
 
@@ -61,11 +68,17 @@ export const LedStripConfiguration = () => {
   // listen to led_sorted_colors_changed event
   createEffect(() => {
     const unlisten = listen<Uint8ClampedArray>('led_sorted_colors_changed', (event) => {
+      console.log('Received led_sorted_colors_changed event:', {
+        hidden: window.document.hidden,
+        sortedColorsLength: event.payload.length,
+        firstFewSortedColors: Array.from(event.payload.slice(0, 12))
+      });
       if (!window.document.hidden) {
         const sortedColors = event.payload;
         setLedStripStore({
           sortedColors,
         });
+        console.log('Updated ledStripStore.sortedColors with length:', sortedColors.length);
       }
     });
 

src/components/led-strip-configuration/led-strip-part.tsx

@@ -60,15 +60,18 @@ export const LedStripPart: Component<LedStripPartProps> = (props) => {
     );
 
     if (index === -1) {
+      console.log(`LED strip not found for display ${localProps.config.display_id}, border ${localProps.config.border}`);
       return;
     }
 
     const mapper = ledStripStore.mappers[index];
     if (!mapper) {
+      console.log(`Mapper not found for index ${index}`);
       return;
     }
 
     const offset = mapper.pos * 3;
+    console.log(`Updating LED strip colors for ${localProps.config.border}, offset: ${offset}, colors length: ${ledStripStore.colors.length}`);
 
     const colors = new Array(localProps.config.len).fill(null).map((_, i) => {
       const index = offset + i * 3;
@@ -77,6 +80,7 @@ export const LedStripPart: Component<LedStripPartProps> = (props) => {
       })`;
     });
 
+    console.log(`Generated ${colors.length} colors for ${localProps.config.border}:`, colors.slice(0, 3));
     setColors(colors);
   });
 

src/components/led-strip-configuration/screen-view.tsx

@@ -1,4 +1,3 @@
-import { convertFileSrc } from '@tauri-apps/api/tauri';
 import {
   Component,
   createEffect,
@@ -79,26 +78,43 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
     let stopped = false;
     const frame = async () => {
       const { drawWidth, drawHeight } = drawInfo();
-      const url = convertFileSrc(
+
-        `displays/${localProps.displayId}?width=${drawWidth}&height=${drawHeight}`,
+      // Skip if dimensions are not ready
-        'ambient-light',
+      if (drawWidth <= 0 || drawHeight <= 0) {
-      );
+        console.log('Skipping frame: invalid dimensions', { drawWidth, drawHeight });
-      await fetch(url, {
+        return;
-        mode: 'cors',
+      }
-      })
+
-        .then((res) => res.body?.getReader().read())
+      const url = `ambient-light://displays/${localProps.displayId}?width=${drawWidth}&height=${drawHeight}`;
-        .then((buffer) => {
+
-          if (buffer?.value) {
+      console.log('Fetching screenshot:', url);
-            setImageData({
+
-              buffer: new Uint8ClampedArray(buffer?.value),
+      try {
-              width: drawWidth,
+        const response = await fetch(url, {
-              height: drawHeight,
+          mode: 'cors',
-            });
-          } else {
-            setImageData(null);
-          }
-          draw();
         });
+
+        if (!response.ok) {
+          console.error('Screenshot fetch failed:', response.status, response.statusText);
+          return;
+        }
+
+        const buffer = await response.body?.getReader().read();
+        if (buffer?.value) {
+          console.log('Screenshot received, size:', buffer.value.length);
+          setImageData({
+            buffer: new Uint8ClampedArray(buffer?.value),
+            width: drawWidth,
+            height: drawHeight,
+          });
+        } else {
+          console.log('No screenshot data received');
+          setImageData(null);
+        }
+        draw();
+      } catch (error) {
+        console.error('Screenshot fetch error:', error);
+      }
     };
 
     (async () => {
@@ -107,7 +123,11 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
           await new Promise((resolve) => setTimeout(resolve, 1000));
           continue;
         }
+
         await frame();
+
+        // Add a small delay to prevent overwhelming the backend
+        await new Promise((resolve) => setTimeout(resolve, 33)); // ~30 FPS
       }
     })();
 
@@ -122,9 +142,14 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
 
     onMount(() => {
       setCtx(canvas.getContext('2d'));
-      new ResizeObserver(() => {
+
+      // Initial size setup
+      resetSize();
+
+      resizeObserver = new ResizeObserver(() => {
         resetSize();
-      }).observe(root);
+      });
+      resizeObserver.observe(root);
     });
 
     onCleanup(() => {