feat: let master schedule low-count WAL tails automatically

crates/lance-context-master/src/config.rs

@@ -14,6 +14,8 @@ pub struct MasterConfig {
     pub catchup: crate::catchup::CatchupConfig,
     #[command(flatten)]
     pub append: crate::rollout_append::AppendConfig,
+    #[command(flatten)]
+    pub wal_tail: crate::wal_tail::WalTailConfig,
     /// Data directory / object-store prefix shared with the data-plane server.
     #[arg(long, env = "DATA_DIR", default_value = "./data")]
     pub data_dir: String,

crates/lance-context-master/src/lib.rs

@@ -14,3 +14,4 @@ pub mod scheduler;
 pub mod state;
 pub mod stats_store;
 pub mod task_store;
+pub mod wal_tail;

crates/lance-context-master/src/routes.rs

@@ -803,6 +803,7 @@ mod tests {
         MasterConfig {
             append: Default::default(),
             catchup: Default::default(),
+            wal_tail: Default::default(),
             maintenance: Default::default(),
             merge_rollout: Default::default(),
             data_dir: dir.path().to_string_lossy().to_string(),

crates/lance-context-master/src/scheduler.rs

@@ -889,6 +889,7 @@ async fn enqueue_merge_request(
 /// not every scheduler task -- adequate for tests, which drop the whole
 /// `MasterState` immediately after.
 pub fn spawn_scheduler(state: &Arc<MasterState>) -> JoinHandle<()> {
+    crate::wal_tail::spawn(state);
     // Retry metadata independently of coarse stats sweeps. Every replica may
     // enqueue; the existing task dedupe/claim transaction elects one executor.
     let retry_state = state.clone();
@@ -1129,6 +1130,7 @@ mod tests {
         MasterConfig {
             append: Default::default(),
             catchup: Default::default(),
+            wal_tail: Default::default(),
             maintenance: Default::default(),
             merge_rollout: lance_context_merge::rollout::MergeRollout {
                 owned_targets: ["exp", "generic:gs", "broken"]

crates/lance-context-master/src/state.rs

@@ -309,6 +309,7 @@ mod tests {
         MasterConfig {
             append: Default::default(),
             catchup: Default::default(),
+            wal_tail: Default::default(),
             maintenance: Default::default(),
             merge_rollout: Default::default(),
             data_dir: dir.path().to_string_lossy().to_string(),

crates/lance-context-master/src/task_store.rs

@@ -2216,6 +2216,7 @@ mod tests {
         MasterConfig {
             append: Default::default(),
             catchup: Default::default(),
+            wal_tail: Default::default(),
             maintenance: Default::default(),
             merge_rollout: Default::default(),
             data_dir: dir.path().to_string_lossy().to_string(),