tty: Avoid modeset on adding device if possible

Session resume will still modeset; more work would be needed to support
that (namely, handling changes to the CRTC mapping).

src/backend/tty.rs

@@ -40,9 +40,11 @@ use smithay::desktop::utils::OutputPresentationFeedback;
 use smithay::output::{Mode, Output, OutputModeSource, PhysicalProperties};
 use smithay::reexports::calloop::timer::{TimeoutAction, Timer};
 use smithay::reexports::calloop::{Dispatcher, LoopHandle, RegistrationToken};
+use smithay::reexports::drm::control::atomic::AtomicModeReq;
+use smithay::reexports::drm::control::dumbbuffer::DumbBuffer;
 use smithay::reexports::drm::control::{
-    self, connector, crtc, property, Device, Mode as DrmMode, ModeFlags, ModeTypeFlags,
+    self, connector, crtc, plane, property, AtomicCommitFlags, Device, Mode as DrmMode, ModeFlags,
-    ResourceHandle,
+    ModeTypeFlags, PlaneType, ResourceHandle,
 };
 use smithay::reexports::gbm::Modifier;
 use smithay::reexports::input::Libinput;
@@ -219,6 +221,154 @@ impl OutputDevice {
         };
         info.name.clone()
     }
+
+    fn cleanup_disconnected_resources(
+        &self,
+        should_be_off: &dyn Fn(crtc::Handle, &connector::Info) -> bool,
+    ) -> anyhow::Result<()> {
+        let _span = tracy_client::span!("OutputDevice::cleanup_disconnected_resources");
+
+        let mut req = AtomicModeReq::new();
+        let plane_handles = self
+            .drm
+            .plane_handles()
+            .context("error getting plane handles")?;
+
+        let mut cleanup = HashSet::new();
+        let mut should_be_off_conn = HashSet::new();
+
+        for (conn, info) in self.drm_scanner.connectors() {
+            let Some(enc) = info.current_encoder() else {
+                // Not enabled.
+                continue;
+            };
+
+            let enc = match self.drm.get_encoder(enc) {
+                Ok(x) => x,
+                Err(err) => {
+                    debug!("couldn't get encoder: {err:?}");
+                    continue;
+                }
+            };
+
+            let Some(crtc) = enc.crtc() else {
+                // Encoder has no CRTC?
+                continue;
+            };
+
+            // All Connected connectors are returned by the DrmScanner and will be attempted for
+            // connection.
+            if info.state() == connector::State::Connected {
+                // But we also need to clear the connectors that should be off according to our
+                // config, since those ones will not be cleared elsewhere.
+                if !should_be_off(crtc, info) {
+                    continue;
+                }
+                should_be_off_conn.insert(conn);
+            }
+
+            cleanup.insert(crtc);
+
+            // Clear the connector.
+            let Some((crtc_id, _, _)) = find_drm_property(&self.drm, *conn, "CRTC_ID") else {
+                debug!("couldn't find connector CRTC_ID property");
+                continue;
+            };
+
+            req.add_property(*conn, crtc_id, property::Value::CRTC(None));
+        }
+
+        // Don't cleanup CRTCs that also correspond to some connected connectors.
+        for (conn, crtc) in self.drm_scanner.crtcs() {
+            // If the connector is enabled, but we're about to disable it, then it will be present
+            // in the DrmScanner; keep it in the cleanup list.
+            if should_be_off_conn.contains(&conn.handle()) {
+                continue;
+            }
+
+            cleanup.remove(&crtc);
+        }
+
+        // Legacy fallback.
+        if !self.drm.is_atomic() {
+            if let Ok(res_handles) = self.drm.resource_handles() {
+                for crtc in res_handles.crtcs() {
+                    #[allow(deprecated)]
+                    let _ = self.drm.set_cursor(*crtc, Option::<&DumbBuffer>::None);
+                }
+            }
+            for crtc in cleanup {
+                let _ = self.drm.set_crtc(crtc, None, (0, 0), &[], None);
+            }
+            return Ok(());
+        }
+
+        // Disable non-primary planes, and planes belonging to disabled CRTCs.
+        let is_primary = |plane: plane::Handle| {
+            if let Some((_, info, value)) = find_drm_property(&self.drm, plane, "type") {
+                match info.value_type().convert_value(value) {
+                    property::Value::Enum(Some(val)) => val.value() == PlaneType::Primary as u64,
+                    _ => false,
+                }
+            } else {
+                debug!("couldn't find plane type property");
+                false
+            }
+        };
+
+        for plane in plane_handles {
+            let info = match self.drm.get_plane(plane) {
+                Ok(x) => x,
+                Err(err) => {
+                    debug!("error getting plane: {err:?}");
+                    continue;
+                }
+            };
+
+            let Some(crtc) = info.crtc() else {
+                continue;
+            };
+
+            if !cleanup.contains(&crtc) && is_primary(plane) {
+                continue;
+            }
+
+            let Some((crtc_id, _, _)) = find_drm_property(&self.drm, plane, "CRTC_ID") else {
+                debug!("couldn't find plane CRTC_ID property");
+                continue;
+            };
+
+            let Some((fb_id, _, _)) = find_drm_property(&self.drm, plane, "FB_ID") else {
+                debug!("couldn't find plane FB_ID property");
+                continue;
+            };
+
+            req.add_property(plane, crtc_id, property::Value::CRTC(None));
+            req.add_property(plane, fb_id, property::Value::Framebuffer(None));
+        }
+
+        // Disable the CRTCs.
+        for crtc in cleanup {
+            let Some((mode_id, _, _)) = find_drm_property(&self.drm, crtc, "MODE_ID") else {
+                debug!("couldn't find CRTC MODE_ID property");
+                continue;
+            };
+
+            let Some((active, _, _)) = find_drm_property(&self.drm, crtc, "ACTIVE") else {
+                debug!("couldn't find CRTC ACTIVE property");
+                continue;
+            };
+
+            req.add_property(crtc, mode_id, property::Value::Unknown(0));
+            req.add_property(crtc, active, property::Value::Boolean(false));
+        }
+
+        self.drm
+            .atomic_commit(AtomicCommitFlags::ALLOW_MODESET, req)
+            .context("error doing atomic commit")?;
+
+        Ok(())
+    }
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -571,7 +721,7 @@ impl Tty {
 
         let (drm, drm_notifier) = {
             let _span = tracy_client::span!("DrmDevice::new");
-            DrmDevice::new(device_fd.clone(), true)
+            DrmDevice::new(device_fd.clone(), false)
         }?;
         let gbm = {
             let _span = tracy_client::span!("GbmDevice::new");
@@ -727,6 +877,11 @@ impl Tty {
             return;
         };
 
+        // If the DrmScanner connectors is empty here, then this will be the very first scan after
+        // the device had just been added.
+        let just_created = device.drm_scanner.connectors().is_empty();
+
+        // DrmScanner will preserve any existing connector-CRTC mapping.
         let scan_result = match device.drm_scanner.scan_connectors(&device.drm) {
             Ok(x) => x,
             Err(err) => {
@@ -810,6 +965,34 @@ impl Tty {
             device.known_crtcs.insert(crtc, info);
         }
 
+        // If the device was just added, we need to cleanup any disconnected connectors and planes.
+        if just_created {
+            let device = self.devices.get(&node).unwrap();
+
+            // Follow the logic in on_output_config_changed().
+            let disable_laptop_panels = self.should_disable_laptop_panels(niri.is_lid_closed);
+            let should_disable = |conn: &str| disable_laptop_panels && is_laptop_panel(conn);
+
+            let config = self.config.borrow();
+            let disable_monitor_names = config.debug.disable_monitor_names;
+
+            let should_be_off = |crtc, conn: &connector::Info| {
+                let output_name = device.known_crtc_name(&crtc, conn, disable_monitor_names);
+
+                let config = config
+                    .outputs
+                    .find(&output_name)
+                    .cloned()
+                    .unwrap_or_default();
+
+                config.off || should_disable(&output_name.connector)
+            };
+
+            if let Err(err) = device.cleanup_disconnected_resources(&should_be_off) {
+                warn!("error cleaning up connectors: {err:?}");
+            }
+        }
+
         // This will connect any new connectors if needed, and apply other changes, such as
         // connecting back the internal laptop monitor once it becomes the only monitor left.
         //