monitord/

machines.rs

use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use thiserror::Error;
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, error, warn};

use crate::MachineStats;
use crate::MonitordStats;

/// Cached D-Bus connections to containers, keyed by machine name.
/// The u32 is the leader PID at the time the connection was established.
/// A connection is only reused if the current leader PID matches.
pub type MachineConnections = HashMap<String, (u32, zbus::Connection)>;

/// What action to take for a machine's cached connection.
#[derive(Debug, PartialEq)]
enum CacheAction {
    /// Cached connection exists with matching leader PID — reuse it.
    Reuse,
    /// Cached connection exists but leader PID changed — drop old and create new.
    Replace,
    /// No cached connection — create new.
    Create,
}

#[derive(Error, Debug)]
pub enum MonitordMachinesError {
    #[error("Machines D-Bus error: {0}")]
    ZbusError(#[from] zbus::Error),
}

pub fn filter_machines(
    machines: Vec<crate::dbus::zbus_machines::ListedMachine>,
    allowlist: &HashSet<String>,
    blocklist: &HashSet<String>,
) -> Vec<crate::dbus::zbus_machines::ListedMachine> {
    machines
        .into_iter()
        .filter(|c| c.class == "container")
        .filter(|c| !blocklist.contains(&c.name))
        .filter(|c| allowlist.is_empty() || allowlist.contains(&c.name))
        .collect()
}

pub async fn get_machines(
    connection: &zbus::Connection,
    config: &crate::config::Config,
) -> Result<HashMap<String, u32>, MonitordMachinesError> {
    let c = crate::dbus::zbus_machines::ManagerProxy::builder(connection)
        .cache_properties(zbus::proxy::CacheProperties::No)
        .build()
        .await?;
    let mut results = HashMap::<String, u32>::new();

    let machines = c.list_machines().await?;

    for machine in filter_machines(
        machines,
        &config.machines.allowlist,
        &config.machines.blocklist,
    ) {
        let m = c.get_machine(&machine.name).await?;
        let leader_pid = m.leader().await?;
        results.insert(machine.name, leader_pid);
    }

    Ok(results)
}

/// Determine the cache action for a machine based on its cached and current leader PID.
fn decide_cache_action(cached_pid: Option<u32>, leader_pid: u32) -> CacheAction {
    match cached_pid {
        Some(pid) if pid == leader_pid => CacheAction::Reuse,
        Some(_) => CacheAction::Replace,
        None => CacheAction::Create,
    }
}

/// Remove cached connections for machines that no longer exist.
async fn evict_stale_connections(
    cached_connections: &Mutex<MachineConnections>,
    current_machines: &HashMap<String, u32>,
) {
    let mut cache = cached_connections.lock().await;
    cache.retain(|name, _| current_machines.contains_key(name));
}

/// Evict a cached connection for a machine that experienced errors.
async fn evict_failed_connection(cached_connections: &Mutex<MachineConnections>, machine: &str) {
    debug!(
        "Evicting cached D-Bus connection for {} due to errors",
        machine
    );
    let mut cache = cached_connections.lock().await;
    cache.remove(machine);
}

/// Return a cached D-Bus connection if one exists for the same leader PID,
/// otherwise create a new connection to the container's system bus.
async fn get_or_create_connection(
    config: &crate::config::Config,
    cached_connections: &Mutex<MachineConnections>,
    machine: &str,
    leader_pid: u32,
) -> anyhow::Result<zbus::Connection> {
    // Check cache and return if hit; drop the lock before any async work
    {
        let mut cache = cached_connections.lock().await;
        match decide_cache_action(cache.get(machine).map(|(pid, _)| *pid), leader_pid) {
            CacheAction::Reuse => {
                debug!("Reusing cached D-Bus connection for {}", machine);
                let (_, conn) = cache.get(machine).unwrap();
                return Ok(conn.clone());
            }
            CacheAction::Replace => {
                debug!(
                    "Leader PID changed for {}, dropping stale connection",
                    machine
                );
                cache.remove(machine);
            }
            CacheAction::Create => {}
        }
    }

    // Build connection without holding the lock
    debug!("Creating new D-Bus connection for {}", machine);
    let container_address = format!(
        "unix:path=/proc/{}/root/run/dbus/system_bus_socket",
        leader_pid
    );
    let conn = zbus::connection::Builder::address(container_address.as_str())?
        .method_timeout(std::time::Duration::from_secs(config.monitord.dbus_timeout))
        .build()
        .await?;

    // Re-lock to insert
    {
        let mut cache = cached_connections.lock().await;
        cache.insert(machine.to_string(), (leader_pid, conn.clone()));
    }

    Ok(conn)
}

pub async fn update_machines_stats(
    config: Arc<crate::config::Config>,
    connection: zbus::Connection,
    locked_monitord_stats: Arc<RwLock<MonitordStats>>,
    cached_connections: Arc<Mutex<MachineConnections>>,
) -> anyhow::Result<()> {
    let locked_machine_stats: Arc<RwLock<MachineStats>> =
        Arc::new(RwLock::new(MachineStats::default()));

    let current_machines = get_machines(&connection, &config).await?;

    evict_stale_connections(&cached_connections, &current_machines).await;

    for (machine, leader_pid) in current_machines.into_iter() {
        debug!(
            "Collecting container: machine: {} leader_pid: {}",
            machine, leader_pid
        );

        let sdc = match get_or_create_connection(&config, &cached_connections, &machine, leader_pid)
            .await
        {
            Ok(conn) => conn,
            Err(e) => {
                error!("Failed to connect to container {}: {:?}", machine, e);
                continue;
            }
        };

        let mut join_set = tokio::task::JoinSet::new();

        if config.pid1.enabled {
            join_set.spawn(crate::pid1::update_pid1_stats(
                leader_pid as i32,
                locked_machine_stats.clone(),
            ));
        }

        if config.networkd.enabled {
            let config_clone = Arc::clone(&config);
            let stats_clone = locked_machine_stats.clone();
            let machine_name = machine.clone();
            let no_fallback = config_clone.varlink.no_fallback;
            join_set.spawn(async move {
                if config_clone
                    .use_varlink(&[config_clone.machines.varlink, config_clone.networkd.varlink])
                {
                    let socket_path = format!(
                        "/proc/{}/root{}",
                        leader_pid,
                        crate::varlink_networkd::NETWORK_SOCKET_PATH
                    );
                    match crate::varlink_networkd::get_networkd_state(&socket_path).await {
                        Ok(networkd_stats) => {
                            let mut machine_stats = stats_clone.write().await;
                            machine_stats.networkd = networkd_stats;
                            machine_stats.varlink_usage.networkd =
                                Some(crate::CollectorTransport::Varlink);
                            return Ok(());
                        }
                        Err(err) => {
                            crate::varlink_fallback::report_varlink_failure(
                                no_fallback,
                                &format!("container {machine_name} networkd"),
                                "file-based",
                                err,
                            )?;
                        }
                    }
                }
                stats_clone.write().await.varlink_usage.networkd =
                    Some(crate::CollectorTransport::Dbus);
                // Same fs-root prefixing the units/cgroup collectors use:
                // both the link state files AND the sysfs ifindex map
                // come from inside the container, so host ifindexes are
                // never labelled with container interface names (or vice
                // versa). A container sharing the host netns simply sees
                // identical trees, which is why CI never caught the old
                // host-files/host-bus pairing being self-consistent.
                let container_root = format!("/proc/{leader_pid}/root");
                let container_sysfs = std::path::PathBuf::from(format!("{container_root}/sys"));
                let container_links = std::path::PathBuf::from(format!(
                    "{container_root}{}",
                    config_clone.networkd.link_state_dir.display()
                ));
                crate::networkd::update_networkd_stats(
                    container_links,
                    None,
                    container_sysfs,
                    None,
                    stats_clone,
                )
                .await
            });
        }

        // One Describe per container, shared by its version and system state
        // collectors, against the container's PID 1 varlink socket seen through
        // its leader's procfs root.
        let manager_describe = config
            .use_varlink(&[config.machines.varlink, config.system_state.varlink])
            .then(|| {
                crate::varlink_system::shared_describe(format!(
                    "/proc/{}/root{}",
                    leader_pid,
                    crate::varlink_system::MANAGER_SOCKET_PATH
                ))
            });

        if config.system_state.enabled {
            let sdc_clone = sdc.clone();
            let stats_clone = locked_machine_stats.clone();
            let machine_name = machine.clone();
            let no_fallback = config.varlink.no_fallback;
            let describe = manager_describe.clone();
            join_set.spawn(async move {
                if let Some(describe) = describe {
                    match crate::varlink_system::update_system_stats(describe, stats_clone.clone())
                        .await
                    {
                        Ok(()) => {
                            stats_clone.write().await.varlink_usage.system_state =
                                Some(crate::CollectorTransport::Varlink);
                            return Ok(());
                        }
                        Err(err) => {
                            crate::varlink_fallback::report_varlink_failure(
                                no_fallback,
                                &format!("container {machine_name} system state"),
                                "D-Bus",
                                err,
                            )?;
                        }
                    }
                }
                stats_clone.write().await.varlink_usage.system_state =
                    Some(crate::CollectorTransport::Dbus);
                crate::system::update_system_stats(sdc_clone, stats_clone.clone()).await
            });
        }

        {
            let sdc_clone = sdc.clone();
            let stats_clone = locked_machine_stats.clone();
            let machine_name = machine.clone();
            let no_fallback = config.varlink.no_fallback;
            let describe = manager_describe.clone();
            join_set.spawn(async move {
                if let Some(describe) = describe {
                    match crate::varlink_system::update_version(describe, stats_clone.clone()).await
                    {
                        Ok(()) => {
                            stats_clone.write().await.varlink_usage.version =
                                Some(crate::CollectorTransport::Varlink);
                            return Ok(());
                        }
                        Err(err) => {
                            crate::varlink_fallback::report_varlink_failure(
                                no_fallback,
                                &format!("container {machine_name} version"),
                                "D-Bus",
                                err,
                            )?;
                        }
                    }
                }
                stats_clone.write().await.varlink_usage.version =
                    Some(crate::CollectorTransport::Dbus);
                crate::system::update_version(sdc_clone, stats_clone.clone()).await
            });
        }

        if config.units.enabled {
            if config.use_varlink(&[config.machines.varlink, config.units.varlink]) {
                let config_clone = Arc::clone(&config);
                let sdc_clone = sdc.clone();
                let stats_clone = locked_machine_stats.clone();
                let no_fallback = config_clone.varlink.no_fallback;
                let machine_name = machine.clone();
                let container_socket_path = format!(
                    "/proc/{}/root{}",
                    leader_pid,
                    crate::varlink_units::METRICS_SOCKET_PATH
                );
                join_set.spawn(async move {
                    match crate::varlink_units::update_unit_stats(
                        Arc::clone(&config_clone),
                        stats_clone.clone(),
                        container_socket_path,
                    )
                    .await
                    {
                        Ok(_timer_names) => {
                            // Containers keep the D-Bus backfill rather than the
                            // host's io.systemd.Unit.List path: a container's
                            // varlink sockets are unreachable from here, with
                            // the connection accepted and then reset unless the
                            // caller is inside the container's PID namespace
                            // (observed, see #211). The timer names collected
                            // above go unused for the same reason.
                            let timer_start = std::time::Instant::now();
                            let timer_result = crate::timer::collect_all_timers_dbus(
                                &sdc_clone,
                                &config_clone,
                            )
                            .await;
                            let timer_elapsed_ms =
                                timer_start.elapsed().as_secs_f64() * 1000.0;
                            match timer_result {
                                Ok(timer_stats) => {
                                    let mut ms = stats_clone.write().await;
                                    crate::timer::merge_timer_stats(
                                        &mut ms.units,
                                        timer_stats,
                                        timer_elapsed_ms,
                                    );
                                }
                                Err(err) => {
                                    warn!(
                                        "Varlink timer stats (D-Bus fallback) failed for container {}: {:?}",
                                        machine_name, err
                                    );
                                    let mut ms = stats_clone.write().await;
                                    crate::timer::record_backfill_duration(
                                        &mut ms.units,
                                        timer_elapsed_ms,
                                    );
                                }
                            }
                            // Service type is not exposed via varlink metrics; resolve
                            // it over the container's D-Bus connection (same as host).
                            crate::varlink_units::apply_oneshot_dbus_override(
                                &sdc_clone,
                                &stats_clone,
                                &config_clone.units,
                            )
                            .await;
                            let container_root = format!("/proc/{}/root", leader_pid);
                            if config_clone.units.unit_files {
                                let unit_files =
                                    crate::units::collect_unit_files_stats(&container_root).await;
                                let mut ms = stats_clone.write().await;
                                ms.units.unit_files = unit_files;
                            }
                            stats_clone.write().await.varlink_usage.units =
                                Some(crate::CollectorTransport::Varlink);
                            Ok(())
                        }
                        Err(err) => {
                            crate::varlink_fallback::report_varlink_failure(
                                no_fallback,
                                &format!("container {machine_name} units"),
                                "D-Bus",
                                err,
                            )?;
                            let container_root = format!("/proc/{}/root", leader_pid);
                            // Set before the call (the lib.rs ordering): if
                            // the D-Bus collection errors, the gauge still
                            // says D-Bus rather than going stale or absent.
                            stats_clone.write().await.varlink_usage.units =
                                Some(crate::CollectorTransport::Dbus);
                            crate::units::update_unit_stats(
                                config_clone,
                                sdc_clone,
                                stats_clone,
                                container_root,
                            )
                            .await
                        }
                    }
                });
            } else {
                let container_root = format!("/proc/{}/root", leader_pid);
                let config_clone = Arc::clone(&config);
                let sdc_clone = sdc.clone();
                let stats_clone = locked_machine_stats.clone();
                join_set.spawn(async move {
                    // Set before the call (the lib.rs ordering): if the
                    // collection errors, the gauge still says D-Bus rather
                    // than going stale or absent.
                    stats_clone.write().await.varlink_usage.units =
                        Some(crate::CollectorTransport::Dbus);
                    crate::units::update_unit_stats(
                        config_clone,
                        sdc_clone,
                        stats_clone,
                        container_root,
                    )
                    .await
                });
            }
        }

        if config.dbus_stats.enabled {
            join_set.spawn(crate::dbus_stats::update_machine_dbus_stats(
                Arc::clone(&config),
                sdc.clone(),
                locked_machine_stats.clone(),
            ));
        }

        let mut had_error = false;
        while let Some(res) = join_set.join_next().await {
            match res {
                Ok(r) => match r {
                    Ok(_) => (),
                    Err(e) => {
                        had_error = true;
                        error!(
                            "Collection specific failure (container {}): {:?}",
                            machine, e
                        );
                    }
                },
                Err(e) => {
                    had_error = true;
                    error!("Join error (container {}): {:?}", machine, e);
                }
            }
        }

        if had_error {
            evict_failed_connection(&cached_connections, &machine).await;
        }

        {
            let mut monitord_stats = locked_monitord_stats.write().await;
            let machine_stats = locked_machine_stats.read().await;
            monitord_stats
                .machines
                .insert(machine, machine_stats.clone());
        }
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use std::collections::HashSet;
    use zbus::zvariant::OwnedObjectPath;

    use super::{decide_cache_action, CacheAction};

    #[test]
    fn test_filter_machines() {
        let machines = vec![
            crate::dbus::zbus_machines::ListedMachine {
                name: "foo".to_string(),
                class: "container".to_string(),
                service: "".to_string(),
                path: OwnedObjectPath::try_from("/sample/object").unwrap(),
            },
            crate::dbus::zbus_machines::ListedMachine {
                name: "bar".to_string(),
                class: "container".to_string(),
                service: "".to_string(),
                path: OwnedObjectPath::try_from("/sample/object").unwrap(),
            },
            crate::dbus::zbus_machines::ListedMachine {
                name: "baz".to_string(),
                class: "container".to_string(),
                service: "".to_string(),
                path: OwnedObjectPath::try_from("/sample/object").unwrap(),
            },
        ];
        let allowlist = HashSet::from(["foo".to_string(), "baz".to_string()]);
        let blocklist = HashSet::from(["bar".to_string()]);

        let filtered = super::filter_machines(machines, &allowlist, &blocklist);

        assert_eq!(filtered.len(), 2);
        assert_eq!(filtered[0].name, "foo");
        assert_eq!(filtered[1].name, "baz");
    }

    #[test]
    fn test_decide_cache_action_reuse_on_same_pid() {
        assert_eq!(decide_cache_action(Some(42), 42), CacheAction::Reuse);
    }

    #[test]
    fn test_decide_cache_action_replace_on_pid_change() {
        assert_eq!(decide_cache_action(Some(42), 99), CacheAction::Replace);
    }

    #[test]
    fn test_decide_cache_action_create_on_miss() {
        assert_eq!(decide_cache_action(None, 42), CacheAction::Create);
    }
}