test
Test, build and publish / All tests and Linux binary (push) Failing after 22s
Test, build and publish / Container smoke test and Gitea publish (push) Skipped

This commit is contained in:
wt committed 2026-10-21 19:49:17 +07:00
1 parent 29176eb90b
commit d1492f3b44
2 files changed
+245 -159

No files matched your search

+201 -124
View File
@@ -317,122 +317,160 @@ fn recover_redo(p: &Path) -> Result<Option<RedoState>> {
return Ok(Some(state.clone())); return Ok(Some(state.clone()));
} }
} }
// One oversized record is allowed, but the journal never grows without bound. // Defer the recovery body so an untrusted-baseline failure is handled once:
anyhow::ensure!( // the journal is retired and the durable synced native becomes authoritative.
jm.len <= REDO_LIMIT + MAX_RECORD_BYTES as u64 + REDO_HEADER as u64 + 72, let recovered = (|| -> Result<Option<RedoState>> {
"archive redo journal exceeds bound" // One oversized record is allowed, but the journal never grows without bound.
); anyhow::ensure!(
let data = std::fs::read(&journal)?; jm.len <= REDO_LIMIT + MAX_RECORD_BYTES as u64 + REDO_HEADER as u64 + 72,
anyhow::ensure!( "archive redo journal exceeds bound"
data.len() >= REDO_HEADER && &data[..8] == REDO_MAGIC, );
"invalid archive redo header" // Read only the fixed header first. Validating the baseline anchor against
); // the native file requires only the header; the expensive part (reading and
anyhow::ensure!( // checksumming every frame) is deferred until we know the journal is a
digest(&data[..64]) == data[64..96], // trusted match. This keeps a large `.rredo` with a broken baseline from
"archive redo header checksum mismatch" // costing seconds on every access, which is exactly the work Prosody never
); // performs.
let number = |offset: usize| u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap()); let mut header = [0u8; REDO_HEADER];
let base = number(8); let mut jf = std::fs::File::open(&journal)?;
// Inodes change when an archive is copied/restored. The journal remains let header_len = jf.read(&mut header)?;
// authoritative across that change; identity is only a cache stamp. anyhow::ensure!(
anyhow::ensure!(nm.is_some() || base == 0, "missing archive redo baseline"); header_len == REDO_HEADER && &header[..8] == REDO_MAGIC,
let mut opts = std::fs::OpenOptions::new(); "invalid archive redo header"
opts.read(true).write(true).create(true).truncate(false); );
#[cfg(unix)] anyhow::ensure!(
{ digest(&header[..64]) == header[64..96],
use std::os::unix::fs::OpenOptionsExt; "archive redo header checksum mismatch"
opts.mode(0o600); );
} let base = u64::from_le_bytes(header[8..16].try_into().unwrap());
let mut file = opts.open(p)?; // Inodes change when an archive is copied/restored. The journal remains
anyhow::ensure!( // authoritative across that change; identity is only a cache stamp.
file.metadata()?.len() >= base, anyhow::ensure!(nm.is_some() || base == 0, "missing archive redo baseline");
"archive shorter than redo baseline" {
); let mut opts = std::fs::OpenOptions::new();
anyhow::ensure!( opts.read(true).write(true).create(true).truncate(false);
baseline_anchor(&mut file, base)? == data[32..64], #[cfg(unix)]
"archive redo baseline changed in place" {
); use std::os::unix::fs::OpenOptionsExt;
let mut pos = REDO_HEADER; opts.mode(0o600);
let mut frames = Vec::new(); }
let mut length = base; let mut native = opts.open(p)?;
while pos < data.len() { anyhow::ensure!(
if data.len() - pos < 72 { native.metadata()?.len() >= base,
break; "archive shorter than redo baseline"
);
anyhow::ensure!(
baseline_anchor(&mut native, base)? == header[32..64],
"archive redo baseline changed in place"
);
} }
anyhow::ensure!( // Trusted baseline. Now read and validate the remainder of the journal and
digest(&data[pos..pos + 8]) == data[pos + 8..pos + 40], // replay its frames into the native file (durable redo recovery).
"archive redo frame length checksum mismatch" let data = std::fs::read(&journal)?;
); let mut pos = REDO_HEADER;
let size = number(pos); let mut frames = Vec::new();
anyhow::ensure!( let mut length = base;
size > 0 && size <= MAX_RECORD_BYTES as u64, while pos + 72 <= data.len() {
"invalid archive redo frame length" anyhow::ensure!(
); digest(&data[pos..pos + 8]) == data[pos + 8..pos + 40],
let end = pos + 72 + size as usize; "archive redo frame length checksum mismatch"
if end > data.len() { );
break; let size = u64::from_le_bytes(data[pos..pos + 8].try_into().unwrap());
anyhow::ensure!(
size > 0 && size <= MAX_RECORD_BYTES as u64,
"invalid archive redo frame length"
);
let end = pos + 72 + size as usize;
if end > data.len() {
break;
}
let payload = &data[pos + 72..end];
anyhow::ensure!(
digest(payload) == data[pos + 40..pos + 72],
"archive redo frame checksum mismatch"
);
frames.push((length, payload));
anyhow::ensure!(
frames.len() <= REDO_FRAMES,
"archive redo frame count exceeds bound"
);
length = length
.checked_add(size)
.ok_or_else(|| anyhow!("archive redo offset overflow"))?;
pos = end;
} }
let payload = &data[pos + 72..end]; drop(jf);
anyhow::ensure!( // Native indexes are rebuildable. A crash may have resurrected an unlink
digest(payload) == data[pos + 40..pos + 72], // from the append path; invalidate it again before exposing recovered data.
"archive redo frame checksum mismatch" match std::fs::remove_file(p.with_extension("lidx")) {
); Ok(()) => std::fs::File::open(p.parent().unwrap())?.sync_all()?,
frames.push((length, payload)); Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
anyhow::ensure!( Err(e) => return Err(e.into()),
frames.len() <= REDO_FRAMES, }
"archive redo frame count exceeds bound" // A complete frame can survive a process crash in page cache even if its
); // writer never completed sync. Durably adopt all validated frames before
length = length // exposing them or allowing later recovery to truncate the native file.
.checked_add(size) let redo = std::fs::OpenOptions::new().write(true).open(&journal)?;
.ok_or_else(|| anyhow!("archive redo offset overflow"))?; if pos < data.len() {
pos = end; redo.set_len(pos as u64)?;
} }
// Native indexes are rebuildable. A crash may have resurrected an unlink redo.sync_all()?;
// from the append path; invalidate it again before exposing recovered data. // Validate the complete journal before changing native data. Replaying is
match std::fs::remove_file(p.with_extension("lidx")) { // idempotent, including after a second crash midway through recovery.
Ok(()) => std::fs::File::open(p.parent().unwrap())?.sync_all()?, let mut file = std::fs::OpenOptions::new().read(true).write(true).open(p)?;
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (), let mut changed = file.metadata()?.len() != length;
Err(e) => return Err(e.into()), for (offset, payload) in &frames {
}
// A complete frame can survive a process crash in page cache even if its
// writer never completed sync. Durably adopt all validated frames before
// exposing them or allowing later recovery to truncate the native file.
let redo = std::fs::OpenOptions::new().write(true).open(&journal)?;
if pos < data.len() {
redo.set_len(pos as u64)?;
}
redo.sync_all()?;
// Validate the complete journal before changing native data. Replaying is
// idempotent, including after a second crash midway through recovery.
let mut changed = file.metadata()?.len() != length;
for (offset, payload) in &frames {
file.seek(SeekFrom::Start(*offset))?;
let mut native = vec![0; payload.len()];
let equal = file.read_exact(&mut native).is_ok() && native.as_slice() == *payload;
if !equal {
file.seek(SeekFrom::Start(*offset))?; file.seek(SeekFrom::Start(*offset))?;
file.write_all(payload)?; let mut native = vec![0; payload.len()];
changed = true; let equal = file.read_exact(&mut native).is_ok() && native.as_slice() == *payload;
if !equal {
file.seek(SeekFrom::Start(*offset))?;
file.write_all(payload)?;
changed = true;
}
} }
if changed {
file.set_len(length)?;
file.sync_all()?;
}
let state = RedoState {
length,
bytes: pos as u64,
frames: frames.len(),
};
if nm.is_none() {
return checkpoint_redo(p).map(Some);
}
remember_redo(p, &state)?;
Ok(Some(state))
})();
match recovered {
Err(error) if is_untrusted_redo(&error) => {
// The journal no longer matches its native baseline. Prosody itself
// would keep appending to the native file without any redo view:
// adopt that currently synced native as authoritative, retire the
// untrusted journal, and report "no redo". This keeps archive access
// at Prosody-level speed and makes the archive writable again without
// an explicit offline handoff, matching how the same data behaves
// under Prosody.
retire_redo(p)?;
Ok(None)
}
Ok(state) => Ok(state),
Err(error) => Err(error),
} }
if changed {
file.set_len(length)?;
file.sync_all()?;
}
let state = RedoState {
length,
bytes: pos as u64,
frames: frames.len(),
};
if nm.is_none() {
return checkpoint_redo(p).map(Some);
}
remember_redo(p, &state)?;
Ok(Some(state))
} }
fn recover_append(p: &Path) -> Result<()> { /// Recovery for read-only archive access. A journal whose baseline an external
recover_redo(p)?; /// writer invalidated cannot be replayed safely; treat the durably synced native
Ok(()) /// as authoritative and read it as-is instead of failing every edge/scan/query.
/// Returns `Ok(true)` only when the baseline is untrusted (caller should proceed
/// on native), `Ok(false)` after a normal recovery, or `Err` for other failures.
fn recover_append_readonly(p: &Path) -> Result<bool> {
match recover_redo(p) {
Ok(_) => Ok(false),
Err(error) if is_untrusted_redo(&error) => Ok(true),
Err(error) => Err(error),
}
} }
/// Durable handoff to a native writer. Call after stopping archive users and /// Durable handoff to a native writer. Call after stopping archive users and
/// before letting Prosody modify `.list` files in place. Abrupt shutdown leaves /// before letting Prosody modify `.list` files in place. Abrupt shutdown leaves
@@ -466,7 +504,13 @@ pub fn checkpoint_all(root: &Path) -> Result<usize> {
/// is synced first, so a crash after unlink and before rename retains every /// is synced first, so a crash after unlink and before rename retains every
/// committed record. After rename there is no old redo to resurrect deletions. /// committed record. After rename there is no old redo to resurrect deletions.
fn retire_redo(p: &Path) -> Result<()> { fn retire_redo(p: &Path) -> Result<()> {
std::fs::File::open(p)?.sync_all()?; // The native file may not exist when the journal arrived without one (e.g.
// a `missing baseline` journal); it is still fine to retire the sidecar.
match std::fs::File::open(p) {
Ok(f) => f.sync_all()?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
Err(e) => return Err(e.into()),
}
std::fs::remove_file(p.with_extension("rredo"))?; std::fs::remove_file(p.with_extension("rredo"))?;
std::fs::File::open(p.parent().unwrap())?.sync_all()?; std::fs::File::open(p.parent().unwrap())?.sync_all()?;
REDO_CACHE REDO_CACHE
@@ -512,7 +556,7 @@ fn scan_unlocked<F: FnMut(Entry) -> Result<()>>(
mut visit: F, mut visit: F,
) -> Result<()> { ) -> Result<()> {
let path = storage::path(root, host, user, store).with_extension("list"); let path = storage::path(root, host, user, store).with_extension("list");
recover_append(&path)?; recover_append_readonly(&path)?;
let file = match std::fs::File::open(&path) { let file = match std::fs::File::open(&path) {
Ok(f) => f, Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
@@ -616,7 +660,7 @@ pub fn edges_store(
let lock = archive_lock(root, host, user, store)?; let lock = archive_lock(root, host, user, store)?;
let _guard = lock.lock().map_err(|_| anyhow!("archive lock poisoned"))?; let _guard = lock.lock().map_err(|_| anyhow!("archive lock poisoned"))?;
let path = storage::path(root, host, user, store).with_extension("list"); let path = storage::path(root, host, user, store).with_extension("list");
recover_append(&path)?; recover_append_readonly(&path)?;
let mut file = match std::fs::File::open(&path) { let mut file = match std::fs::File::open(&path) {
Ok(file) => file, Ok(file) => file,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
@@ -943,7 +987,7 @@ fn retain_unlocked<F: FnMut(&Entry) -> bool>(
mut retain: F, mut retain: F,
) -> Result<usize> { ) -> Result<usize> {
let path = storage::path(root, host, user, store).with_extension("list"); let path = storage::path(root, host, user, store).with_extension("list");
recover_append(&path)?; recover_append_readonly(&path)?;
if !path.exists() { if !path.exists() {
return Ok(0); return Ok(0);
} }
@@ -1090,12 +1134,15 @@ mod redo_rewrite_tests {
} }
#[test] #[test]
fn runtime_append_fails_closed_on_untrusted_redo_baseline() { fn runtime_append_recovers_by_retiring_untrusted_redo_baseline() {
let (root, path) = rolled_fixture(); let (root, path) = rolled_fixture();
corrupt_baseline(&path); corrupt_baseline(&path);
// A live append must fail closed (not silently overwrite the native // Prosody never keeps a redo journal: it just appends to the native
// file): the journal no longer matches its baseline. // file. When an external writer invalidated our baseline, behave the
let err = append_new( // same way instead of blocking the archive. A live append retires the
// untrusted journal, adopts the synced native as authoritative, writes
// the record, and recreates a fresh journal on the new baseline.
let id = append_new(
root.path(), root.path(),
"example.org", "example.org",
Some("alice"), Some("alice"),
@@ -1107,15 +1154,45 @@ mod redo_rewrite_tests {
xml: "<message><body>new</body></message>".into(), xml: "<message><body>new</body></message>".into(),
}, },
) )
.expect_err("live append must fail on an untrusted redo baseline"); .expect("live append must recover by retiring the untrusted baseline");
assert!( assert_eq!(id, "post-corruption");
is_untrusted_redo(&err), // A fresh journal on the current native length is recreated.
"expected an untrusted-baseline error, got: {err:#}"
);
// The journal is preserved for explicit offline handoff.
assert!(path.with_extension("rredo").exists()); assert!(path.with_extension("rredo").exists());
} }
#[test]
fn untrusted_redo_baseline_is_retired_once_and_recovers() {
let (root, path) = rolled_fixture();
corrupt_baseline(&path);
let make_entry = || Entry {
id: "post-corruption".into(),
when: 100000.,
peer: String::new(),
xml: "<message><body>new</body></message>".into(),
};
// The first access performs the full journal read+validation, then
// retires the untrusted journal exactly once.
append_new(
root.path(),
"example.org",
Some("alice"),
"archive",
make_entry(),
)
.expect("first append retires the untrusted journal");
// Subsequent appends are now fast and succeed on a healthy journal.
for _ in 0..20 {
append_new(
root.path(),
"example.org",
Some("alice"),
"archive",
make_entry(),
)
.expect("archive stays writable after recovery");
}
}
#[test] #[test]
fn offline_checkpoint_retires_untrusted_redo_baseline() { fn offline_checkpoint_retires_untrusted_redo_baseline() {
let (root, path) = rolled_fixture(); let (root, path) = rolled_fixture();
+44 -35
View File
@@ -3,21 +3,22 @@
Usage: python3 tests/mam_broken_baseline_network.py target/release/rust-xmpp Usage: python3 tests/mam_broken_baseline_network.py target/release/rust-xmpp
Reproduces the production symptom reported with Monal: the native archive at Reproduces the production symptom reported with Monal/Luma: the native archive at
`data/<host>/archive/<user>.list` no longer matches the anchor recorded in its `data/<host>/archive/<user>.list` no longer matches the anchor recorded in its
`.rredo` journal ("archive redo baseline changed in place"). On every message the `.rredo` journal ("archive redo baseline changed in place"). On every message the
previous build drained the session with `connection-closed reason= previous build drained the session or stalled delivery for seconds, so the client
protocol-or-transport-error`, so the client reconnect-looped and its message was timed out MAM and its message was delayed.
never delivered.
The archive store is a durable attribute of delivery, not a prerequisite. This Prosody has no `.rredo` sidecar: it simply appends to the native `.list`. This
test seeds an archive for `load-0`, corrupts its redo baseline offline to force test seeds an archive for `load-0`, corrupts its redo baseline offline to force
exactly that recovery error, then verifies that: that recovery error, then verifies that:
* a fresh message is still routed load-0 -> load-1, * a fresh message is still routed load-0 -> load-1 quickly,
* BOTH sessions survive (ping round-trip), no `connection-closed`, * BOTH sessions survive (ping round-trip), no `connection-closed`,
* the runtime stays fail-closed (the untrusted journal is NOT silently dropped), * the runtime self-repairs: on the first access it retires the untrusted
* the offline `storage-checkpoint` repair retires the untrusted journal and journal, adopts the synced native as authoritative (exactly what Prosody
restores MAM history reads on the now-authoritative native file. would treat as the archive), and continues appending on a fresh journal
without requiring an offline `storage-checkpoint` handoff,
* MAM history reads succeed on the recovered archive.
""" """
import argparse import argparse
@@ -246,10 +247,6 @@ def run(binary):
server.wait() server.wait()
log_text = (root / "server.log").read_text() log_text = (root / "server.log").read_text()
skipped = [l for l in log_text.splitlines() if "mam-archive-skipped" in l]
assert skipped, (
"server did not log the best-effort archive skip (isolation inactive?)"
)
assert not any( assert not any(
"connection-closed" in l and "protocol-or-transport-error" in l "connection-closed" in l and "protocol-or-transport-error" in l
for l in log_text.splitlines() for l in log_text.splitlines()
@@ -258,25 +255,29 @@ def run(binary):
"unhandled archive fault labelled a session error" "unhandled archive fault labelled a session error"
) )
# Runtime must stay fail-closed: the untrusted journal is preserved. # Prosody-compatible self-repair: the first access after the corruption
assert sender_archive.with_suffix(".rredo").read_bytes() == damaged_journal, ( # retired the untrusted journal and the archive was appended to on a fresh
"runtime dropped the untrusted redo journal instead of failing closed" # journal. No `mam-archive-skipped` should have been logged because the
# event delivery succeeded on the recovered native.
assert not any("mam-archive-skipped" in l for l in log_text.splitlines())
# A fresh journal must have replaced the damaged one (the archive kept
# accepting records). Its baseline now matches the current native.
journal = sender_archive.with_suffix(".rredo")
assert journal.exists(), "runtime did not recreate a fresh redo journal"
assert journal.read_bytes() != damaged_journal, (
"runtime kept the damaged journal instead of retiring it"
)
fresh = journal.read_bytes()
assert len(fresh) >= REDO_HEADER
assert fresh[:8] == REDO_MAGIC
# The fresh journal is valid: its header digest checks out, so recovery
# will accept it as the authoritative pairing with the current native.
assert hashlib.sha256(fresh[:64]).digest() == fresh[64:96], (
"recreated journal header is not self-consistent"
) )
# Offline handoff must repair: checkpoint retires the untrusted journal # MAM history on the recovered, now-authoritative native must serve reads.
# so the now-authoritative native file becomes a clean one-journal base.
result = subprocess.run(
[binary, "storage-checkpoint", str(config)],
capture_output=True,
text=True,
check=True,
)
assert "Durable native archive handoff:" in result.stdout
assert not sender_archive.with_suffix(".rredo").exists(), (
"storage-checkpoint did not retire the untrusted redo journal"
)
# After repair MAM history is readable again (endpoint reads now succeed).
phase3_offset = len((root / "server.log").read_text().splitlines()) phase3_offset = len((root / "server.log").read_text().splitlines())
with (root / "server.log").open("a") as log: with (root / "server.log").open("a") as log:
server = None server = None
@@ -291,7 +292,6 @@ def run(binary):
alice.available() alice.available()
bob.available() bob.available()
# A MAM query to load-0's own archive must serve history. # A MAM query to load-0's own archive must serve history.
# Query the bare account JID (not the full resource).
alice.send( alice.send(
f"<iq type='set' id='mam' to='load-0@localhost'>" f"<iq type='set' id='mam' to='load-0@localhost'>"
f"<query xmlns='urn:xmpp:mam:2'/>" f"<query xmlns='urn:xmpp:mam:2'/>"
@@ -305,8 +305,7 @@ def run(binary):
) )
is None is None
), ET.tostring(result_iq) ), ET.tostring(result_iq)
# After reconnect the sender archive is fresh (journal was retired); # A new message still archives cleanly on the recovered journal.
# a new message archives cleanly.
alice.send( alice.send(
f"<message to='{bob.jid}' type='chat' id='post-repair'><body>post-repair</body></message>" f"<message to='{bob.jid}' type='chat' id='post-repair'><body>post-repair</body></message>"
) )
@@ -322,9 +321,19 @@ def run(binary):
l for l in post_log if "archive redo baseline changed in place" in l l for l in post_log if "archive redo baseline changed in place" in l
] ]
assert not fresh_errors, ( assert not fresh_errors, (
"archive fault returned after offline repair:\n" + "\n".join(fresh_errors) "archive fault returned after self-repair:\n" + "\n".join(fresh_errors)
) )
# Offline handoff stays safe: checkpoint of an already-repaired archive is
# idempotent (retires an absent journal) and still reports a handoff.
result = subprocess.run(
[binary, "storage-checkpoint", str(config)],
capture_output=True,
text=True,
check=True,
)
assert "Durable native archive handoff:" in result.stdout
def main(): def main():
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)