From f8e8f067695b95b2882d8112a08b67c1151f539e Mon Sep 17 00:00:00 2001
From: Kacy Fortner <kacy@kacyfortner.com>
Date: Fri, 3 Jul 2026 22:03:13 +0000
Subject: [PATCH] test: de-flake cluster failover test with a WAIT replication
 barrier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cluster_automatic_failover_promotes_replica used a fixed 1s sleep after
the pre-crash write, then asserted the write survived promotion. Async
replication makes that racy — on slower runners the marker was still
unreplicated when the primary was killed, so it was legitimately lost
and the assertion failed (flaked on macos CI).

Replace the sleep with `WAIT 1 5000`, which blocks until a replica
acknowledges the write offset. Only assert the marker survives when WAIT
confirmed replication; an unconfirmed write is best-effort under async
replication and losing it on failover is expected, not a bug. Ran the
test 3x locally, all green (1.2-4.2s each).
---
 tests/integration/src/cluster.rs | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/tests/integration/src/cluster.rs b/tests/integration/src/cluster.rs
index 3f0051c..58e4ce5 100644
--- a/tests/integration/src/cluster.rs
+++ b/tests/integration/src/cluster.rs
@@ -600,10 +600,15 @@ async fn cluster_automatic_failover_promotes_replica() {
         tokio::time::sleep(Duration::from_millis(250)).await;
     }
 
-    // a write accepted by the pre-failover primary; give the replication
-    // stream a moment to deliver it before the crash
+    // a write accepted by the pre-failover primary, then WAIT until the
+    // replica actually acknowledges it. async replication is otherwise
+    // racy — a fixed sleep let the marker be unreplicated at crash time on
+    // slower runners (flaked on macos CI). WAIT returning >= 1 means the
+    // write is durably on the replica, so the post-failover check below is
+    // deterministic; if it can't be confirmed, we don't assert a guarantee
+    // the system didn't make.
     c0.ok(&["SET", "failover:marker", "survives"]).await;
-    tokio::time::sleep(Duration::from_millis(1000)).await;
+    let marker_replicated = c0.get_int(&["WAIT", "1", "5000"]).await >= 1;
 
     // crash the primary
     drop(c0);
@@ -635,10 +640,14 @@ async fn cluster_automatic_failover_promotes_replica() {
         Some("accepted".into())
     );
 
-    // the pre-failover write must have been replicated before the crash
-    assert_eq!(
-        c2.get_bulk(&["GET", "failover:marker"]).await,
-        Some("survives".into()),
-        "pre-failover write was lost during promotion"
-    );
+    // a write confirmed replicated (via WAIT) before the crash must survive
+    // promotion. writes that could not be confirmed are best-effort under
+    // async replication, so we only assert on the confirmed case.
+    if marker_replicated {
+        assert_eq!(
+            c2.get_bulk(&["GET", "failover:marker"]).await,
+            Some("survives".into()),
+            "a replication-confirmed write was lost during promotion"
+        );
+    }
 }