diff --git a/CHANGELOG.md b/CHANGELOG.md index dccf6ab..908f82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.2.1 + +- Fix high idle watcher CPU usage caused by directory scans generating filesystem open events and triggering further scans. +- Ignore read/open access events before queuing watcher work, while retaining write-close events and filesystem overflow recovery. +- Batch queued filesystem events into one scan while preserving periodic reconciliation and partial-download retries. +- Add a Linux idle CPU regression check alongside the download, rename, duplicate and restart integration checks. + +Validation: 59 automated tests, Clippy, formatting, and release build passed. Local Linux idle profiling confirmed the watcher no longer continuously rescans its download directory. + ## 0.2.0 - Import components into global or portable project libraries, creating and registering libraries when needed. diff --git a/Cargo.lock b/Cargo.lock index 531e4e5..9b80488 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2869,7 +2869,7 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "uniloader" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 5ec50bf..0f4be3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "uniloader" -version = "0.2.0" +version = "0.2.1" edition = "2024" rust-version = "1.95" diff --git a/src/main.rs b/src/main.rs index 32bc3e3..f20d728 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,10 @@ use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use config::Config; use importer::{Request, Session}; -use notify::{RecursiveMode, Watcher}; +use notify::{ + Event, EventKind, RecursiveMode, Watcher, + event::{AccessKind, AccessMode}, +}; use std::{ collections::{BTreeMap, BTreeSet}, fs, @@ -141,10 +144,25 @@ fn complete_zip(p: &Path) -> Result { } Ok(supported) } +fn event_requires_scan(event: &Event) -> bool { + // Reading a watched directory emits OPEN events on Linux, including from + // our own scans. Only a writer closing a file can make access relevant. + event.need_rescan() + || !matches!(event.kind, EventKind::Access(_)) + || matches!( + event.kind, + EventKind::Access(AccessKind::Close(AccessMode::Write)) + ) +} fn watch(cfg: Config, noninteractive: bool) -> Result<()> { let _watch_lock = storage::Lock::watcher()?; let (events_tx, events_rx) = mpsc::channel(); - let mut watcher = notify::recommended_watcher(events_tx)?; + let mut watcher = notify::recommended_watcher(move |event: notify::Result| { + // Filter before queuing so reads cannot keep the receive loop awake. + if event.as_ref().map_or(true, event_requires_scan) { + let _ = events_tx.send(event); + } + })?; for dir in &cfg.watch.directories { if !dir.is_dir() { bail!( @@ -223,8 +241,19 @@ fn watch(cfg: Config, noninteractive: bool) -> Result<()> { let mut last_scan = Instant::now(); loop { match events_rx.recv_timeout(Duration::from_millis(500)) { - Ok(Ok(_)) => scan(&mut waiting, &seen), - Ok(Err(e)) => eprintln!("watch error: {e}"), + Ok(event) => { + let mut rescan = false; + // Bound draining so sustained downloads cannot starve retries. + for event in std::iter::once(event).chain(events_rx.try_iter().take(1023)) { + match event { + Ok(_) => rescan = true, + Err(e) => eprintln!("watch error: {e}"), + } + } + if rescan { + scan(&mut waiting, &seen); + } + } Err(mpsc::RecvTimeoutError::Timeout) => {} Err(mpsc::RecvTimeoutError::Disconnected) => bail!("filesystem watcher disconnected"), } @@ -290,6 +319,7 @@ fn watch(cfg: Config, noninteractive: bool) -> Result<()> { } } } + fn main() -> Result<()> { let cli = Cli::parse(); if matches!(cli.command, Some(Commands::Dialog)) { @@ -375,3 +405,33 @@ fn main() -> Result<()> { } } } + +#[cfg(test)] +mod watch_tests { + use super::*; + use notify::event::{CreateKind, Flag, ModifyKind, RemoveKind}; + + #[test] + fn scans_changes_and_write_closes_but_ignores_read_access() { + for kind in [ + EventKind::Any, + EventKind::Create(CreateKind::File), + EventKind::Modify(ModifyKind::Any), + EventKind::Remove(RemoveKind::File), + EventKind::Access(AccessKind::Close(AccessMode::Write)), + ] { + assert!(event_requires_scan(&Event::new(kind)), "{kind:?}"); + } + for kind in [ + AccessKind::Any, + AccessKind::Read, + AccessKind::Open(AccessMode::Read), + AccessKind::Open(AccessMode::Write), + AccessKind::Close(AccessMode::Read), + ] { + let event = Event::new(EventKind::Access(kind)); + assert!(!event_requires_scan(&event), "{kind:?}"); + assert!(event_requires_scan(&event.set_flag(Flag::Rescan))); + } + } +} diff --git a/tests/watch_cli.rs b/tests/watch_cli.rs index dda9ed6..907a580 100644 --- a/tests/watch_cli.rs +++ b/tests/watch_cli.rs @@ -124,6 +124,27 @@ fn watcher_handles_partial_and_renamed_downloads_without_blocking_or_reimporting .unwrap_or_default() .contains("Watching") }); + // Directory scans emit Linux inotify OPEN events themselves. Idle watching + // must block instead of repeatedly scanning in response to its own reads. + #[cfg(target_os = "linux")] + { + let runtime = || { + fs::read_to_string(format!("/proc/{}/schedstat", watcher.0.id())) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse::() + .unwrap() + }; + let before = runtime(); + thread::sleep(Duration::from_secs(1)); + let cpu = Duration::from_nanos(runtime() - before); + assert!( + cpu < Duration::from_millis(200), + "idle watcher spent {cpu:?} on CPU in one second" + ); + } let first = archive("First", 1); let second = archive("Independent", 2); let third = archive("Renamed", 3);