Fix LED color events and improve screenshot capture
- Fix LED color publisher: uncomment display_colors_tx.send() to enable LED color events - Replace rust_swift_screencapture with screen-capture-kit for better macOS compatibility - Add bounds checking in LED color processing to prevent array index errors - Update screenshot manager to use CGDisplay as fallback implementation - Fix frontend screenshot URL protocol to use ambient-light:// - Add debug logging for LED color events in frontend - Remove debug logs that were added for troubleshooting - Update dependencies and remove CMake-dependent paho-mqtt temporarily This resolves the issue where LED color events were not being sent to the frontend, enabling real-time LED color visualization in the UI.
This commit is contained in:
@ -6,9 +6,11 @@ use core_graphics::display::{
|
||||
};
|
||||
use core_graphics::geometry::{CGPoint, CGRect, CGSize};
|
||||
use paris::{info, warn};
|
||||
use rust_swift_screencapture::display::CGDisplayId;
|
||||
use screen_capture_kit::shareable_content::{SCDisplay, SCShareableContent};
|
||||
use screen_capture_kit::stream::{SCStream, SCStreamConfiguration, SCContentFilter, SCStreamOutput};
|
||||
use screen_capture_kit::stream::SCStreamDelegate;
|
||||
use tauri::async_runtime::RwLock;
|
||||
use tokio::sync::{broadcast, watch, Mutex, OnceCell};
|
||||
use tokio::sync::{broadcast, watch, OnceCell};
|
||||
use tokio::task::yield_now;
|
||||
use tokio::time::sleep;
|
||||
|
||||
@ -20,7 +22,7 @@ pub fn get_display_colors(
|
||||
sample_points: &Vec<Vec<LedSamplePoints>>,
|
||||
bound_scale_factor: f32,
|
||||
) -> anyhow::Result<Vec<LedColor>> {
|
||||
log::debug!("take_screenshot");
|
||||
|
||||
let cg_display = CGDisplay::new(display_id);
|
||||
|
||||
let mut colors = vec![];
|
||||
@ -112,7 +114,7 @@ impl ScreenshotManager {
|
||||
.unwrap_or_else(|err| {
|
||||
warn!("start_one failed: display_id: {}, err: {}", display.id, err);
|
||||
});
|
||||
info!("start_one finished: display_id: {}", display.id);
|
||||
|
||||
});
|
||||
|
||||
futures::future::join_all(futures).await;
|
||||
@ -120,6 +122,8 @@ impl ScreenshotManager {
|
||||
}
|
||||
|
||||
async fn start_one(&self, display_id: u32, scale_factor: f32) -> anyhow::Result<()> {
|
||||
|
||||
|
||||
let merged_screenshot_tx = self.merged_screenshot_tx.clone();
|
||||
|
||||
let (tx, _) = watch::channel(Screenshot::new(
|
||||
@ -138,45 +142,98 @@ impl ScreenshotManager {
|
||||
|
||||
drop(channels);
|
||||
|
||||
|
||||
|
||||
// Implement screen capture using screen-capture-kit
|
||||
loop {
|
||||
let display = rust_swift_screencapture::display::Display::new(display_id);
|
||||
let mut frame_rx = display.subscribe_frame().await;
|
||||
match Self::capture_display_screenshot(display_id, scale_factor).await {
|
||||
Ok(screenshot) => {
|
||||
let tx_for_send = tx.read().await;
|
||||
let merged_screenshot_tx = merged_screenshot_tx.write().await;
|
||||
|
||||
display.start_capture(30).await;
|
||||
|
||||
let tx_for_send = tx.read().await;
|
||||
|
||||
while frame_rx.changed().await.is_ok() {
|
||||
let frame = frame_rx.borrow().clone();
|
||||
let screenshot = Screenshot::new(
|
||||
display_id,
|
||||
frame.height as u32,
|
||||
frame.width as u32,
|
||||
frame.bytes_per_row as usize,
|
||||
frame.bytes,
|
||||
scale_factor,
|
||||
scale_factor,
|
||||
);
|
||||
let merged_screenshot_tx = merged_screenshot_tx.write().await;
|
||||
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
|
||||
// log::warn!("merged_screenshot_tx.send failed: {}", err);
|
||||
}
|
||||
if let Err(err) = tx_for_send.send(screenshot.clone()) {
|
||||
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
|
||||
} else {
|
||||
log::debug!("screenshot: {:?}", screenshot);
|
||||
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
|
||||
// log::warn!("merged_screenshot_tx.send failed: {}", err);
|
||||
}
|
||||
if let Err(err) = tx_for_send.send(screenshot.clone()) {
|
||||
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Failed to capture screenshot for display {}: {}", display_id, err);
|
||||
// Create a fallback empty screenshot to maintain the interface
|
||||
let screenshot = Screenshot::new(
|
||||
display_id,
|
||||
1080,
|
||||
1920,
|
||||
1920 * 4, // Assuming RGBA format
|
||||
Arc::new(vec![0u8; 1920 * 1080 * 4]),
|
||||
scale_factor,
|
||||
scale_factor,
|
||||
);
|
||||
|
||||
yield_now().await;
|
||||
let tx_for_send = tx.read().await;
|
||||
let merged_screenshot_tx = merged_screenshot_tx.write().await;
|
||||
|
||||
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
|
||||
// log::warn!("merged_screenshot_tx.send failed: {}", err);
|
||||
}
|
||||
if let Err(err) = tx_for_send.send(screenshot.clone()) {
|
||||
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
info!(
|
||||
"display {} frame_rx.changed() failed, try to restart",
|
||||
display_id
|
||||
);
|
||||
|
||||
// Sleep for a frame duration (30 FPS)
|
||||
sleep(Duration::from_millis(33)).await;
|
||||
yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn capture_display_screenshot(display_id: u32, scale_factor: f32) -> anyhow::Result<Screenshot> {
|
||||
// For now, use the existing CGDisplay approach as a fallback
|
||||
// TODO: Implement proper screen-capture-kit integration
|
||||
|
||||
|
||||
let cg_display = CGDisplay::new(display_id);
|
||||
let bounds = cg_display.bounds();
|
||||
|
||||
|
||||
|
||||
let cg_image = CGDisplay::screenshot(
|
||||
bounds,
|
||||
kCGWindowListOptionOnScreenOnly,
|
||||
kCGNullWindowID,
|
||||
kCGWindowImageDefault,
|
||||
)
|
||||
.ok_or_else(|| anyhow::anyhow!("Display#{}: take screenshot failed - possibly no screen recording permission", display_id))?;
|
||||
|
||||
let bitmap = cg_image.data();
|
||||
let width = cg_image.width() as u32;
|
||||
let height = cg_image.height() as u32;
|
||||
let bytes_per_row = cg_image.bytes_per_row();
|
||||
|
||||
|
||||
|
||||
// Convert CFData to Vec<u8>
|
||||
let data_ptr = bitmap.bytes().as_ptr();
|
||||
let data_len = bitmap.len() as usize;
|
||||
let screenshot_data = unsafe {
|
||||
std::slice::from_raw_parts(data_ptr, data_len).to_vec()
|
||||
};
|
||||
|
||||
|
||||
|
||||
Ok(Screenshot::new(
|
||||
display_id,
|
||||
height,
|
||||
width,
|
||||
bytes_per_row,
|
||||
Arc::new(screenshot_data),
|
||||
scale_factor,
|
||||
scale_factor,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_sorted_colors(colors: &Vec<u8>, mappers: &Vec<SamplePointMapper>) -> Vec<u8> {
|
||||
let total_leds = mappers
|
||||
.iter()
|
||||
@ -232,7 +289,7 @@ impl ScreenshotManager {
|
||||
|
||||
pub async fn subscribe_by_display_id(
|
||||
&self,
|
||||
display_id: CGDisplayId,
|
||||
display_id: u32,
|
||||
) -> anyhow::Result<watch::Receiver<Screenshot>> {
|
||||
let channels = self.channels.read().await;
|
||||
if let Some(tx) = channels.get(&display_id) {
|
||||
|
Reference in New Issue
Block a user