display-ambient-light/desktop
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects 1 Releases Wiki Activity

Fix LED color events and improve screenshot capture

Browse Source 
- Fix LED color publisher: uncomment display_colors_tx.send() to enable LED color events
- Replace rust_swift_screencapture with screen-capture-kit for better macOS compatibility
- Add bounds checking in LED color processing to prevent array index errors
- Update screenshot manager to use CGDisplay as fallback implementation
- Fix frontend screenshot URL protocol to use ambient-light://
- Add debug logging for LED color events in frontend
- Remove debug logs that were added for troubleshooting
- Update dependencies and remove CMake-dependent paho-mqtt temporarily

This resolves the issue where LED color events were not being sent to the frontend,
enabling real-time LED color visualization in the UI.
This commit is contained in:
Ivan Li
2025-06-30 14:35:03 +08:00
parent 91983e6728
commit b1fd751090
10 changed files with 1844 additions and 1494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2538
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

498
src-tauri/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

8
src-tauri/Cargo.toml
View File

@ -13,7 +13,7 @@ edition = "2021"
tauri-build = { version = "1.2", features = [] }
[dependencies]
tauri = { version = "1.2", features = ["shell-open"] }
tauri = { version = "1.2", features = [ "protocol-all", "shell-open"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
core-graphics = "0.22.3"
@ -28,8 +28,8 @@ url-build-parse = "9.0.0"
color_space = "0.5.3"
hex = "0.4.3"
toml = "0.7.3"
paho-mqtt = "0.12.1"
time = {version="0.3.20", features= ["formatting"] }
# paho-mqtt = "0.12.1" # Temporarily disabled due to CMake issues
time = {version="0.3.35", features= ["formatting"] }
itertools = "0.10.5"
core-foundation = "0.9.3"
tokio-stream = "0.1.14"
@ -37,7 +37,7 @@ mdns-sd = "0.7.2"
futures = "0.3.28"
ddc-hi = "0.4.1"
coreaudio-rs = "0.11.2"
rust_swift_screencapture = { version = "0.1.1", path = "../../../../demo/rust-swift-screencapture" }
screen-capture-kit = "0.3.1"
[features]
# this feature is used for production builds or when `devPath` points to the filesystem

66
src-tauri/src/ambient_light/publisher.rs
View File

@ -88,21 +88,21 @@ impl LedColorsPublisher {
}
}
// match display_colors_tx.send((
// display_id,
// colors_copy
// .into_iter()
// .map(|color| color.get_rgb())
// .flatten()
// .collect::<Vec<_>>(),
// )) {
// Ok(_) => {
// // log::info!("sent colors: {:?}", color_len);
// }
// Err(err) => {
// warn!("Failed to send display_colors: {}", err);
// }
// };
match display_colors_tx.send((
display_id,
colors_copy
.into_iter()
.map(|color| color.get_rgb())
.flatten()
.collect::<Vec<_>>(),
)) {
Ok(_) => {
// log::info!("sent colors: {:?}", color_len);
}
Err(err) => {
warn!("Failed to send display_colors: {}", err);
}
};
// Check if the inner task version changed
let version = internal_tasks_version.read().await.clone();
@ -127,7 +127,7 @@ impl LedColorsPublisher {
) {
let sorted_colors_tx = self.sorted_colors_tx.clone();
let colors_tx = self.colors_tx.clone();
log::debug!("start all_colors_worker");
tokio::spawn(async move {
for _ in 0..10 {
@ -137,7 +137,7 @@ impl LedColorsPublisher {
let mut all_colors: Vec<Option<Vec<u8>>> = vec![None; display_ids.len()];
let mut start: tokio::time::Instant = tokio::time::Instant::now();
log::debug!("start all_colors_worker task");
loop {
let color_info = display_colors_rx.recv().await;
@ -186,7 +186,7 @@ impl LedColorsPublisher {
warn!("Failed to send sorted colors: {}", err);
}
};
log::debug!("tick: {}ms", start.elapsed().as_millis());
start = tokio::time::Instant::now();
}
}
@ -195,7 +195,7 @@ impl LedColorsPublisher {
}
pub async fn start(&self) {
log::info!("start colors worker");
let config_manager = ConfigManager::global().await;
let mut config_receiver = config_manager.clone_config_update_receiver();
@ -203,9 +203,7 @@ impl LedColorsPublisher {
self.handle_config_change(configs).await;
log::info!("waiting for config update...");
while config_receiver.changed().await.is_ok() {
log::info!("config updated, restart inner tasks...");
let configs = config_receiver.borrow().clone();
self.handle_config_change(configs).await;
}
@ -300,16 +298,28 @@ impl LedColorsPublisher {
if group.end > group.start {
for i in group.pos - display_led_offset..group_size + group.pos - display_led_offset
{
let bytes = colors[i].as_bytes();
buffer.append(&mut bytes.to_vec());
if i < colors.len() {
let bytes = colors[i].as_bytes();
buffer.append(&mut bytes.to_vec());
} else {
log::warn!("Index {} out of bounds for colors array of length {}", i, colors.len());
// Add black color as fallback
buffer.append(&mut vec![0, 0, 0]);
}
}
} else {
for i in (group.pos - display_led_offset
..group_size + group.pos - display_led_offset)
.rev()
{
let bytes = colors[i].as_bytes();
buffer.append(&mut bytes.to_vec());
if i < colors.len() {
let bytes = colors[i].as_bytes();
buffer.append(&mut bytes.to_vec());
} else {
log::warn!("Index {} out of bounds for colors array of length {}", i, colors.len());
// Add black color as fallback
buffer.append(&mut vec![0, 0, 0]);
}
}
}
@ -350,7 +360,7 @@ impl LedColorsPublisher {
let mut screenshots = HashMap::new();
loop {
log::info!("waiting merged screenshot...");
let screenshot = merged_screenshot_receiver.recv().await;
if let Err(err) = screenshot {
@ -382,7 +392,7 @@ impl LedColorsPublisher {
.filter(|(_, c)| c.display_id == display_id);
let screenshot = screenshots.get(&display_id).unwrap();
log::debug!("screenshot updated: {:?}", display_id);
let points: Vec<_> = led_strip_configs
.clone()
@ -412,7 +422,7 @@ impl LedColorsPublisher {
led_start = led_end;
}
log::debug!("got all colors configs: {:?}", colors_configs.len());
return Ok(AllColorConfig {
sample_point_groups: colors_configs,

6
src-tauri/src/main.rs
View File

@ -269,7 +269,7 @@ async fn main() {
let url = url.unwrap();
let re = regex::Regex::new(r"^/displays/(\d+)$").unwrap();
let re = regex::Regex::new(r"^/(\d+)$").unwrap();
let path = url.path;
let captures = re.captures(path.as_str());
@ -287,6 +287,7 @@ async fn main() {
let bytes = tokio::task::block_in_place(move || {
tauri::async_runtime::block_on(async move {
let screenshot_manager = ScreenshotManager::global().await;
let rx: Result<tokio::sync::watch::Receiver<Screenshot>, anyhow::Error> =
screenshot_manager.subscribe_by_display_id(display_id).await;
@ -305,7 +306,7 @@ async fn main() {
anyhow::bail!("Display#{}: no screenshot.", display_id);
}
log::debug!("Display#{}: screenshot size: {}", display_id, bytes.len());
let (scale_factor_x, scale_factor_y, width, height) = if url.query.is_some()
&& url.query.as_ref().unwrap().contains_key("height")
@ -368,6 +369,7 @@ async fn main() {
}
}
Ok(rgba_buffer.clone())
})
});

129
src-tauri/src/screenshot_manager.rs
View File

@ -6,9 +6,11 @@ use core_graphics::display::{
};
use core_graphics::geometry::{CGPoint, CGRect, CGSize};
use paris::{info, warn};
use rust_swift_screencapture::display::CGDisplayId;
use screen_capture_kit::shareable_content::{SCDisplay, SCShareableContent};
use screen_capture_kit::stream::{SCStream, SCStreamConfiguration, SCContentFilter, SCStreamOutput};
use screen_capture_kit::stream::SCStreamDelegate;
use tauri::async_runtime::RwLock;
use tokio::sync::{broadcast, watch, Mutex, OnceCell};
use tokio::sync::{broadcast, watch, OnceCell};
use tokio::task::yield_now;
use tokio::time::sleep;
@ -20,7 +22,7 @@ pub fn get_display_colors(
sample_points: &Vec<Vec<LedSamplePoints>>,
bound_scale_factor: f32,
) -> anyhow::Result<Vec<LedColor>> {
log::debug!("take_screenshot");
let cg_display = CGDisplay::new(display_id);
let mut colors = vec![];
@ -112,7 +114,7 @@ impl ScreenshotManager {
.unwrap_or_else(|err| {
warn!("start_one failed: display_id: {}, err: {}", display.id, err);
});
info!("start_one finished: display_id: {}", display.id);
});
futures::future::join_all(futures).await;
@ -120,6 +122,8 @@ impl ScreenshotManager {
}
async fn start_one(&self, display_id: u32, scale_factor: f32) -> anyhow::Result<()> {
let merged_screenshot_tx = self.merged_screenshot_tx.clone();
let (tx, _) = watch::channel(Screenshot::new(
@ -138,45 +142,98 @@ impl ScreenshotManager {
drop(channels);
// Implement screen capture using screen-capture-kit
loop {
let display = rust_swift_screencapture::display::Display::new(display_id);
let mut frame_rx = display.subscribe_frame().await;
match Self::capture_display_screenshot(display_id, scale_factor).await {
Ok(screenshot) => {
let tx_for_send = tx.read().await;
let merged_screenshot_tx = merged_screenshot_tx.write().await;
display.start_capture(30).await;
let tx_for_send = tx.read().await;
while frame_rx.changed().await.is_ok() {
let frame = frame_rx.borrow().clone();
let screenshot = Screenshot::new(
display_id,
frame.height as u32,
frame.width as u32,
frame.bytes_per_row as usize,
frame.bytes,
scale_factor,
scale_factor,
);
let merged_screenshot_tx = merged_screenshot_tx.write().await;
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
} else {
log::debug!("screenshot: {:?}", screenshot);
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
}
}
Err(err) => {
warn!("Failed to capture screenshot for display {}: {}", display_id, err);
// Create a fallback empty screenshot to maintain the interface
let screenshot = Screenshot::new(
display_id,
1080,
1920,
1920 * 4, // Assuming RGBA format
Arc::new(vec![0u8; 1920 * 1080 * 4]),
scale_factor,
scale_factor,
);
yield_now().await;
let tx_for_send = tx.read().await;
let merged_screenshot_tx = merged_screenshot_tx.write().await;
if let Err(err) = merged_screenshot_tx.send(screenshot.clone()) {
// log::warn!("merged_screenshot_tx.send failed: {}", err);
}
if let Err(err) = tx_for_send.send(screenshot.clone()) {
log::warn!("display {} screenshot_tx.send failed: {}", display_id, err);
}
}
}
sleep(Duration::from_secs(5)).await;
info!(
"display {} frame_rx.changed() failed, try to restart",
display_id
);
// Sleep for a frame duration (30 FPS)
sleep(Duration::from_millis(33)).await;
yield_now().await;
}
}
async fn capture_display_screenshot(display_id: u32, scale_factor: f32) -> anyhow::Result<Screenshot> {
// For now, use the existing CGDisplay approach as a fallback
// TODO: Implement proper screen-capture-kit integration
let cg_display = CGDisplay::new(display_id);
let bounds = cg_display.bounds();
let cg_image = CGDisplay::screenshot(
bounds,
kCGWindowListOptionOnScreenOnly,
kCGNullWindowID,
kCGWindowImageDefault,
)
.ok_or_else(|| anyhow::anyhow!("Display#{}: take screenshot failed - possibly no screen recording permission", display_id))?;
let bitmap = cg_image.data();
let width = cg_image.width() as u32;
let height = cg_image.height() as u32;
let bytes_per_row = cg_image.bytes_per_row();
// Convert CFData to Vec<u8>
let data_ptr = bitmap.bytes().as_ptr();
let data_len = bitmap.len() as usize;
let screenshot_data = unsafe {
std::slice::from_raw_parts(data_ptr, data_len).to_vec()
};
Ok(Screenshot::new(
display_id,
height,
width,
bytes_per_row,
Arc::new(screenshot_data),
scale_factor,
scale_factor,
))
}
pub fn get_sorted_colors(colors: &Vec<u8>, mappers: &Vec<SamplePointMapper>) -> Vec<u8> {
let total_leds = mappers
.iter()
@ -232,7 +289,7 @@ impl ScreenshotManager {
pub async fn subscribe_by_display_id(
&self,
display_id: CGDisplayId,
display_id: u32,
) -> anyhow::Result<watch::Receiver<Screenshot>> {
let channels = self.channels.read().await;
if let Some(tx) = channels.get(&display_id) {

7
src-tauri/tauri.conf.json
View File

@ -16,6 +16,13 @@
"shell": {
"all": false,
"open": true
},
"protocol": {
"all": true,
"asset": true,
"assetScope": [
"**"
]
}
},
"bundle": {

13
src/components/led-strip-configuration/led-strip-configuration.tsx
View File

@ -13,6 +13,7 @@ import {
LedStripConfigurationContextType,
} from '../../contexts/led-strip-configuration.context';
export const LedStripConfiguration = () => {
createEffect(() => {
invoke<string>('list_display_info').then((displays) => {
@ -45,11 +46,17 @@ export const LedStripConfiguration = () => {
// listen to led_colors_changed event
createEffect(() => {
const unlisten = listen<Uint8ClampedArray>('led_colors_changed', (event) => {
console.log('Received led_colors_changed event:', {
hidden: window.document.hidden,
colorsLength: event.payload.length,
firstFewColors: Array.from(event.payload.slice(0, 12))
});
if (!window.document.hidden) {
const colors = event.payload;
setLedStripStore({
colors,
});
console.log('Updated ledStripStore.colors with length:', colors.length);
}
});
@ -61,11 +68,17 @@ export const LedStripConfiguration = () => {
// listen to led_sorted_colors_changed event
createEffect(() => {
const unlisten = listen<Uint8ClampedArray>('led_sorted_colors_changed', (event) => {
console.log('Received led_sorted_colors_changed event:', {
hidden: window.document.hidden,
sortedColorsLength: event.payload.length,
firstFewSortedColors: Array.from(event.payload.slice(0, 12))
});
if (!window.document.hidden) {
const sortedColors = event.payload;
setLedStripStore({
sortedColors,
});
console.log('Updated ledStripStore.sortedColors with length:', sortedColors.length);
}
});

4
src/components/led-strip-configuration/led-strip-part.tsx
View File

@ -60,15 +60,18 @@ export const LedStripPart: Component<LedStripPartProps> = (props) => {
);
if (index === -1) {
console.log(`LED strip not found for display ${localProps.config.display_id}, border ${localProps.config.border}`);
return;
}
const mapper = ledStripStore.mappers[index];
if (!mapper) {
console.log(`Mapper not found for index ${index}`);
return;
}
const offset = mapper.pos * 3;
console.log(`Updating LED strip colors for ${localProps.config.border}, offset: ${offset}, colors length: ${ledStripStore.colors.length}`);
const colors = new Array(localProps.config.len).fill(null).map((_, i) => {
const index = offset + i * 3;
@ -77,6 +80,7 @@ export const LedStripPart: Component<LedStripPartProps> = (props) => {
})`;
});
console.log(`Generated ${colors.length} colors for ${localProps.config.border}:`, colors.slice(0, 3));
setColors(colors);
});

69
src/components/led-strip-configuration/screen-view.tsx
View File

@ -1,4 +1,3 @@
import { convertFileSrc } from '@tauri-apps/api/tauri';
import {
Component,
createEffect,
@ -79,26 +78,43 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
let stopped = false;
const frame = async () => {
const { drawWidth, drawHeight } = drawInfo();
const url = convertFileSrc(
`displays/${localProps.displayId}?width=${drawWidth}&height=${drawHeight}`,
'ambient-light',
);
await fetch(url, {
mode: 'cors',
})
.then((res) => res.body?.getReader().read())
.then((buffer) => {
if (buffer?.value) {
setImageData({
buffer: new Uint8ClampedArray(buffer?.value),
width: drawWidth,
height: drawHeight,
});
} else {
setImageData(null);
}
draw();
// Skip if dimensions are not ready
if (drawWidth <= 0 || drawHeight <= 0) {
console.log('Skipping frame: invalid dimensions', { drawWidth, drawHeight });
return;
}
const url = `ambient-light://displays/${localProps.displayId}?width=${drawWidth}&height=${drawHeight}`;
console.log('Fetching screenshot:', url);
try {
const response = await fetch(url, {
mode: 'cors',
});
if (!response.ok) {
console.error('Screenshot fetch failed:', response.status, response.statusText);
return;
}
const buffer = await response.body?.getReader().read();
if (buffer?.value) {
console.log('Screenshot received, size:', buffer.value.length);
setImageData({
buffer: new Uint8ClampedArray(buffer?.value),
width: drawWidth,
height: drawHeight,
});
} else {
console.log('No screenshot data received');
setImageData(null);
}
draw();
} catch (error) {
console.error('Screenshot fetch error:', error);
}
};
(async () => {
@ -107,7 +123,11 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
await new Promise((resolve) => setTimeout(resolve, 1000));
continue;
}
await frame();
// Add a small delay to prevent overwhelming the backend
await new Promise((resolve) => setTimeout(resolve, 33)); // ~30 FPS
}
})();
@ -122,9 +142,14 @@ export const ScreenView: Component<ScreenViewProps> = (props) => {
onMount(() => {
setCtx(canvas.getContext('2d'));
new ResizeObserver(() => {
// Initial size setup
resetSize();
resizeObserver = new ResizeObserver(() => {
resetSize();
}).observe(root);
});
resizeObserver.observe(root);
});
onCleanup(() => {