Add layer matcher to layer-rule

docs/wiki/Configuration:-Layer-Rules.md

@@ -14,6 +14,7 @@ Here are all matchers and properties that a layer rule could have:
 layer-rule {
     match namespace="waybar"
     match at-startup=true
+    match layer="top"
 
     // Properties that apply continuously.
     opacity 0.5
@@ -69,6 +70,22 @@ layer-rule {
 }
 ```
 
+#### `layer`
+
+<sup>Since: next release</sup>
+
+Matches surfaces on this layer-shell layer.
+Can be `"background"`, `"bottom"`, `"top"`, or `"overlay"`.
+
+```kdl
+// Make all overlay-layer surfaces FLOAT.
+layer-rule {
+    match layer="overlay"
+
+    baba-is-float true
+}
+```
+
 ### Dynamic Properties
 
 These properties apply continuously to open layer-shell surfaces.

niri-config/src/layer_rule.rs

@@ -28,4 +28,6 @@ pub struct Match {
     pub namespace: Option<RegexEq>,
     #[knuffel(property)]
     pub at_startup: Option<bool>,
+    #[knuffel(property, str)]
+    pub layer: Option<niri_ipc::Layer>,
 }

niri-config/src/lib.rs

@@ -1859,6 +1859,7 @@ mod tests {
                                 ),
                             ),
                             at_startup: None,
+                            layer: None,
                         },
                     ],
                     excludes: [],

niri-ipc/src/lib.rs

@@ -1868,6 +1868,20 @@ impl FromStr for Transform {
     }
 }
 
+impl FromStr for Layer {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "background" => Ok(Self::Background),
+            "bottom" => Ok(Self::Bottom),
+            "top" => Ok(Self::Top),
+            "overlay" => Ok(Self::Overlay),
+            _ => Err("invalid layer, can be \"background\", \"bottom\", \"top\" or \"overlay\""),
+        }
+    }
+}
+
 impl FromStr for ModeToSet {
     type Err = &'static str;
 

src/handlers/layer_shell.rs

@@ -3,10 +3,10 @@ use smithay::desktop::{layer_map_for_output, LayerSurface, PopupKind, WindowSurf
 use smithay::output::Output;
 use smithay::reexports::wayland_server::protocol::wl_output::WlOutput;
 use smithay::reexports::wayland_server::protocol::wl_surface::WlSurface;
-use smithay::wayland::compositor::{get_parent, with_states};
+use smithay::wayland::compositor::{add_pre_commit_hook, get_parent, with_states, HookId};
 use smithay::wayland::shell::wlr_layer::{
-    self, Layer, LayerSurface as WlrLayerSurface, LayerSurfaceData, WlrLayerShellHandler,
-    WlrLayerShellState,
+    self, Layer, LayerSurface as WlrLayerSurface, LayerSurfaceCachedState, LayerSurfaceData,
+    WlrLayerShellHandler, WlrLayerShellState,
 };
 use smithay::wayland::shell::xdg::PopupSurface;
 
@@ -126,8 +126,10 @@ impl State {
                 let output_size = output_size(&output);
                 let scale = output.current_scale().fractional_scale();
 
+                let hook = add_mapped_layer_pre_commit_hook(layer);
                 let mapped = MappedLayer::new(
                     layer.clone(),
+                    hook,
                     rules,
                     output_size,
                     scale,
@@ -142,6 +144,21 @@ impl State {
                 if prev.is_some() {
                     error!("MappedLayer was present for an unmapped surface");
                 }
+            } else {
+                // The surface remains mapped.
+                if let Some(mapped) = self.niri.mapped_layer_surfaces.get_mut(layer) {
+                    // Check if the layer changed.
+                    if mapped.take_recompute_rules_on_commit() {
+                        let config = self.niri.config.borrow();
+                        if mapped
+                            .recompute_layer_rules(&config.layer_rules, self.niri.is_at_startup)
+                        {
+                            mapped.update_config(&config);
+                        }
+                    }
+                } else {
+                    error!("MappedLayer missing for a mapped surface");
+                }
             }
 
             // Give focus to newly mapped on-demand surfaces. Some launchers like lxqt-runner rely
@@ -204,3 +221,23 @@ impl State {
         true
     }
 }
+
+fn add_mapped_layer_pre_commit_hook(layer: &LayerSurface) -> HookId {
+    add_pre_commit_hook::<State, _>(layer.wl_surface(), move |state, _dh, surface| {
+        let layer_changed = with_states(surface, |states| {
+            let mut guard = states.cached_state.get::<LayerSurfaceCachedState>();
+            let pending_layer = guard.pending().layer;
+            let current_layer = guard.current().layer;
+            pending_layer != current_layer
+        });
+
+        if layer_changed {
+            for mapped in state.niri.mapped_layer_surfaces.values_mut() {
+                if mapped.surface().wl_surface() == surface {
+                    mapped.set_recompute_rules_on_commit();
+                    break;
+                }
+            }
+        }
+    })
+}

src/layer/mapped.rs

@@ -4,6 +4,7 @@ use smithay::backend::renderer::element::surface::WaylandSurfaceRenderElement;
 use smithay::backend::renderer::element::Kind;
 use smithay::desktop::{LayerSurface, PopupManager};
 use smithay::utils::{Logical, Point, Scale, Size};
+use smithay::wayland::compositor::{remove_pre_commit_hook, HookId};
 use smithay::wayland::shell::wlr_layer::{ExclusiveZone, Layer};
 
 use super::ResolvedLayerRules;
@@ -22,9 +23,17 @@ pub struct MappedLayer {
     /// The surface itself.
     surface: LayerSurface,
 
+    /// Pre-commit hook that we have on all mapped layer surfaces.
+    pre_commit_hook: HookId,
+
     /// Up-to-date rules.
     rules: ResolvedLayerRules,
 
+    /// Whether to recompute layer rules on the next commit.
+    ///
+    /// Set in the pre-commit hook when the layer changes; consumed in the commit handler.
+    recompute_rules_on_commit: bool,
+
     /// Buffer to draw instead of the surface when it should be blocked out.
     block_out_buffer: SolidColorBuffer,
 
@@ -52,6 +61,7 @@ niri_render_elements! {
 impl MappedLayer {
     pub fn new(
         surface: LayerSurface,
+        pre_commit_hook: HookId,
         rules: ResolvedLayerRules,
         view_size: Size<f64, Logical>,
         scale: f64,
@@ -65,7 +75,9 @@ impl MappedLayer {
 
         Self {
             surface,
+            pre_commit_hook,
             rules,
+            recompute_rules_on_commit: false,
             block_out_buffer: SolidColorBuffer::new((0., 0.), [0., 0., 0., 1.]),
             view_size,
             scale,
@@ -128,6 +140,14 @@ impl MappedLayer {
         true
     }
 
+    pub fn set_recompute_rules_on_commit(&mut self) {
+        self.recompute_rules_on_commit = true;
+    }
+
+    pub fn take_recompute_rules_on_commit(&mut self) -> bool {
+        std::mem::take(&mut self.recompute_rules_on_commit)
+    }
+
     pub fn place_within_backdrop(&self) -> bool {
         if !self.rules.place_within_backdrop {
             return false;
@@ -232,3 +252,9 @@ impl MappedLayer {
         }
     }
 }
+
+impl Drop for MappedLayer {
+    fn drop(&mut self) {
+        remove_pre_commit_hook(self.surface.wl_surface(), self.pre_commit_hook.clone());
+    }
+}

src/layer/mod.rs

@@ -2,6 +2,7 @@ use niri_config::layer_rule::{LayerRule, Match};
 use niri_config::utils::MergeWith as _;
 use niri_config::{BlockOutFrom, CornerRadius, ShadowRule};
 use smithay::desktop::LayerSurface;
+use smithay::wayland::shell::wlr_layer::Layer;
 
 pub mod mapped;
 pub use mapped::MappedLayer;
@@ -83,5 +84,17 @@ fn surface_matches(surface: &LayerSurface, m: &Match) -> bool {
         }
     }
 
+    if let Some(layer) = m.layer {
+        let surface_layer = match surface.layer() {
+            Layer::Background => niri_ipc::Layer::Background,
+            Layer::Bottom => niri_ipc::Layer::Bottom,
+            Layer::Top => niri_ipc::Layer::Top,
+            Layer::Overlay => niri_ipc::Layer::Overlay,
+        };
+        if layer != surface_layer {
+            return false;
+        }
+    }
+
     true
 }