display-ambient-light/desktop
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects 1 Releases Wiki Activity

Fix LED color events and improve screenshot capture

Browse Source 
- Fix LED color publisher: uncomment display_colors_tx.send() to enable LED color events
- Replace rust_swift_screencapture with screen-capture-kit for better macOS compatibility
- Add bounds checking in LED color processing to prevent array index errors
- Update screenshot manager to use CGDisplay as fallback implementation
- Fix frontend screenshot URL protocol to use ambient-light://
- Add debug logging for LED color events in frontend
- Remove debug logs that were added for troubleshooting
- Update dependencies and remove CMake-dependent paho-mqtt temporarily

This resolves the issue where LED color events were not being sent to the frontend,
enabling real-time LED color visualization in the UI.
This commit is contained in:
Ivan Li
2025-06-30 14:35:03 +08:00
parent 91983e6728
commit b1fd751090
10 changed files with 1844 additions and 1494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
src-tauri/src/screenshot_manager.rs
View File

@ -6,9 +6,11 @@ use core_graphics::display::{
};
use core_graphics::geometry::{CGPoint, CGRect, CGSize};
use paris::{info, warn};
use rust_swift_screencapture::display::CGDisplayId;
use screen_capture_kit::shareable_content::{SCDisplay, SCShareableContent};
use screen_capture_kit::stream::{SCStream, SCStreamConfiguration, SCContentFilter, SCStreamOutput};
use screen_capture_kit::stream::SCStreamDelegate;
use tauri::async_runtime::RwLock;
use tokio::sync::{broadcast, watch, Mutex, OnceCell};
use tokio::sync::{broadcast, watch, OnceCell};
use tokio::task::yield_now;
use tokio::time::sleep;
@ -20,7 +22,7 @@ pub fn get_display_colors(
sample_points: &Vec<Vec<LedSamplePoints>>,
bound_scale_factor: f32,
) -> anyhow::Result<Vec<LedColor>> {
log::debug!("take_screenshot");
let cg_display = CGDisplay::new(display_id);
let mut colors = vec![];
@ -112,7 +114,7 @@ impl ScreenshotManager {
.unwrap_or_else(|err| {
warn!("start_one failed: display_id: {}, err: {}", display.id, err);
});
info!("start_one finished: display_id: {}", display.id);
});
futures::future::join_all(futures).await;
@ -120,6 +122,8 @@ impl ScreenshotManager {
}
async fn start_one(&self, display_id: u32, scale_factor: f32) -> anyhow::Result<()> {
let merged_screenshot_tx = self.merged_screenshot_tx.clone();
let (tx, _) = watch::channel(Screenshot::new(
@ -138,45 +142,98 @@ impl ScreenshotManager {
drop(channels);
// Implement screen capture using screen-capture-kit
loop {
let display = rust_swift_screencapture::display::Display::new(display_id);
let mut frame_rx = display.subscribe_frame().await;
match Self::capture_display_screenshot(display_id, scale_factor).await {
Ok(screenshot) => {
let tx_for_send = tx.read().await;
let merged_screenshot_tx = merged_screenshot_tx.write().await;
display.start_capture(30).await;
let tx_for_send = tx.read().await;
while frame_rx.changed().await.is_ok() {
let frame = frame_rx.borrow().clone();
let screenshot = Screenshot::new(
display_id,
frame.height as u32,
frame.width as u32,
frame.bytes_per_row as usize,
frame.bytes,
scale_factor,
scale_factor,
);
let merged_screenshot_tx = merged_screenshot_tx.write().await;
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
} else {
log::debug!("screenshot: {:?}", screenshot);
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
}
}
Err(err) => {
warn!("Failed to capture screenshot for display {}: {}", display_id, err);
// Create a fallback empty screenshot to maintain the interface
let screenshot = Screenshot::new(
display_id,
1080,
1920,
1920 * 4, // Assuming RGBA format
Arc::new(vec![0u8; 1920 * 1080 * 4]),
scale_factor,
scale_factor,
);
yield_now().await;
let tx_for_send = tx.read().await;
let merged_screenshot_tx = merged_screenshot_tx.write().await;
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
}
}
}
sleep(Duration::from_secs(5)).await;
info!(
"display {} frame_rx.changed() failed, try to restart",
display_id
);
// Sleep for a frame duration (30 FPS)
sleep(Duration::from_millis(33)).await;
yield_now().await;
}
}
async fn capture_display_screenshot(display_id: u32, scale_factor: f32) -> anyhow::Result<Screenshot> {
// For now, use the existing CGDisplay approach as a fallback
// TODO: Implement proper screen-capture-kit integration
let cg_display = CGDisplay::new(display_id);
let bounds = cg_display.bounds();
let cg_image = CGDisplay::screenshot(
bounds,
kCGWindowListOptionOnScreenOnly,
kCGNullWindowID,
kCGWindowImageDefault,
)
.ok_or_else(|| anyhow::anyhow!("Display#{}: take screenshot failed - possibly no screen recording permission", display_id))?;
let bitmap = cg_image.data();
let width = cg_image.width() as u32;
let height = cg_image.height() as u32;
let bytes_per_row = cg_image.bytes_per_row();
// Convert CFData to Vec<u8>
let data_ptr = bitmap.bytes().as_ptr();
let data_len = bitmap.len() as usize;
let screenshot_data = unsafe {
std::slice::from_raw_parts(data_ptr, data_len).to_vec()
};
Ok(Screenshot::new(
display_id,
height,
width,
bytes_per_row,
Arc::new(screenshot_data),
scale_factor,
scale_factor,
))
}
pub fn get_sorted_colors(colors: &Vec<u8>, mappers: &Vec<SamplePointMapper>) -> Vec<u8> {
let total_leds = mappers
.iter()
@ -232,7 +289,7 @@ impl ScreenshotManager {
pub async fn subscribe_by_display_id(
&self,
display_id: CGDisplayId,
display_id: u32,
) -> anyhow::Result<watch::Receiver<Screenshot>> {
let channels = self.channels.read().await;
if let Some(tx) = channels.get(&display_id) {