test
This commit is contained in:
1 parent
29176eb90b
commit
d1492f3b44
2 files changed
+245
-159
No files matched your search
+201
-124
@@ -317,122 +317,160 @@ fn recover_redo(p: &Path) -> Result<Option<RedoState>> {
|
|||||||
return Ok(Some(state.clone()));
|
return Ok(Some(state.clone()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// One oversized record is allowed, but the journal never grows without bound.
|
// Defer the recovery body so an untrusted-baseline failure is handled once:
|
||||||
anyhow::ensure!(
|
// the journal is retired and the durable synced native becomes authoritative.
|
||||||
jm.len <= REDO_LIMIT + MAX_RECORD_BYTES as u64 + REDO_HEADER as u64 + 72,
|
let recovered = (|| -> Result<Option<RedoState>> {
|
||||||
"archive redo journal exceeds bound"
|
// One oversized record is allowed, but the journal never grows without bound.
|
||||||
);
|
anyhow::ensure!(
|
||||||
let data = std::fs::read(&journal)?;
|
jm.len <= REDO_LIMIT + MAX_RECORD_BYTES as u64 + REDO_HEADER as u64 + 72,
|
||||||
anyhow::ensure!(
|
"archive redo journal exceeds bound"
|
||||||
data.len() >= REDO_HEADER && &data[..8] == REDO_MAGIC,
|
);
|
||||||
"invalid archive redo header"
|
// Read only the fixed header first. Validating the baseline anchor against
|
||||||
);
|
// the native file requires only the header; the expensive part (reading and
|
||||||
anyhow::ensure!(
|
// checksumming every frame) is deferred until we know the journal is a
|
||||||
digest(&data[..64]) == data[64..96],
|
// trusted match. This keeps a large `.rredo` with a broken baseline from
|
||||||
"archive redo header checksum mismatch"
|
// costing seconds on every access, which is exactly the work Prosody never
|
||||||
);
|
// performs.
|
||||||
let number = |offset: usize| u64::from_le_bytes(data[offset..offset + 8].try_into().unwrap());
|
let mut header = [0u8; REDO_HEADER];
|
||||||
let base = number(8);
|
let mut jf = std::fs::File::open(&journal)?;
|
||||||
// Inodes change when an archive is copied/restored. The journal remains
|
let header_len = jf.read(&mut header)?;
|
||||||
// authoritative across that change; identity is only a cache stamp.
|
anyhow::ensure!(
|
||||||
anyhow::ensure!(nm.is_some() || base == 0, "missing archive redo baseline");
|
header_len == REDO_HEADER && &header[..8] == REDO_MAGIC,
|
||||||
let mut opts = std::fs::OpenOptions::new();
|
"invalid archive redo header"
|
||||||
opts.read(true).write(true).create(true).truncate(false);
|
);
|
||||||
#[cfg(unix)]
|
anyhow::ensure!(
|
||||||
{
|
digest(&header[..64]) == header[64..96],
|
||||||
use std::os::unix::fs::OpenOptionsExt;
|
"archive redo header checksum mismatch"
|
||||||
opts.mode(0o600);
|
);
|
||||||
}
|
let base = u64::from_le_bytes(header[8..16].try_into().unwrap());
|
||||||
let mut file = opts.open(p)?;
|
// Inodes change when an archive is copied/restored. The journal remains
|
||||||
anyhow::ensure!(
|
// authoritative across that change; identity is only a cache stamp.
|
||||||
file.metadata()?.len() >= base,
|
anyhow::ensure!(nm.is_some() || base == 0, "missing archive redo baseline");
|
||||||
"archive shorter than redo baseline"
|
{
|
||||||
);
|
let mut opts = std::fs::OpenOptions::new();
|
||||||
anyhow::ensure!(
|
opts.read(true).write(true).create(true).truncate(false);
|
||||||
baseline_anchor(&mut file, base)? == data[32..64],
|
#[cfg(unix)]
|
||||||
"archive redo baseline changed in place"
|
{
|
||||||
);
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
let mut pos = REDO_HEADER;
|
opts.mode(0o600);
|
||||||
let mut frames = Vec::new();
|
}
|
||||||
let mut length = base;
|
let mut native = opts.open(p)?;
|
||||||
while pos < data.len() {
|
anyhow::ensure!(
|
||||||
if data.len() - pos < 72 {
|
native.metadata()?.len() >= base,
|
||||||
break;
|
"archive shorter than redo baseline"
|
||||||
|
);
|
||||||
|
anyhow::ensure!(
|
||||||
|
baseline_anchor(&mut native, base)? == header[32..64],
|
||||||
|
"archive redo baseline changed in place"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
anyhow::ensure!(
|
// Trusted baseline. Now read and validate the remainder of the journal and
|
||||||
digest(&data[pos..pos + 8]) == data[pos + 8..pos + 40],
|
// replay its frames into the native file (durable redo recovery).
|
||||||
"archive redo frame length checksum mismatch"
|
let data = std::fs::read(&journal)?;
|
||||||
);
|
let mut pos = REDO_HEADER;
|
||||||
let size = number(pos);
|
let mut frames = Vec::new();
|
||||||
anyhow::ensure!(
|
let mut length = base;
|
||||||
size > 0 && size <= MAX_RECORD_BYTES as u64,
|
while pos + 72 <= data.len() {
|
||||||
"invalid archive redo frame length"
|
anyhow::ensure!(
|
||||||
);
|
digest(&data[pos..pos + 8]) == data[pos + 8..pos + 40],
|
||||||
let end = pos + 72 + size as usize;
|
"archive redo frame length checksum mismatch"
|
||||||
if end > data.len() {
|
);
|
||||||
break;
|
let size = u64::from_le_bytes(data[pos..pos + 8].try_into().unwrap());
|
||||||
|
anyhow::ensure!(
|
||||||
|
size > 0 && size <= MAX_RECORD_BYTES as u64,
|
||||||
|
"invalid archive redo frame length"
|
||||||
|
);
|
||||||
|
let end = pos + 72 + size as usize;
|
||||||
|
if end > data.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let payload = &data[pos + 72..end];
|
||||||
|
anyhow::ensure!(
|
||||||
|
digest(payload) == data[pos + 40..pos + 72],
|
||||||
|
"archive redo frame checksum mismatch"
|
||||||
|
);
|
||||||
|
frames.push((length, payload));
|
||||||
|
anyhow::ensure!(
|
||||||
|
frames.len() <= REDO_FRAMES,
|
||||||
|
"archive redo frame count exceeds bound"
|
||||||
|
);
|
||||||
|
length = length
|
||||||
|
.checked_add(size)
|
||||||
|
.ok_or_else(|| anyhow!("archive redo offset overflow"))?;
|
||||||
|
pos = end;
|
||||||
}
|
}
|
||||||
let payload = &data[pos + 72..end];
|
drop(jf);
|
||||||
anyhow::ensure!(
|
// Native indexes are rebuildable. A crash may have resurrected an unlink
|
||||||
digest(payload) == data[pos + 40..pos + 72],
|
// from the append path; invalidate it again before exposing recovered data.
|
||||||
"archive redo frame checksum mismatch"
|
match std::fs::remove_file(p.with_extension("lidx")) {
|
||||||
);
|
Ok(()) => std::fs::File::open(p.parent().unwrap())?.sync_all()?,
|
||||||
frames.push((length, payload));
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
|
||||||
anyhow::ensure!(
|
Err(e) => return Err(e.into()),
|
||||||
frames.len() <= REDO_FRAMES,
|
}
|
||||||
"archive redo frame count exceeds bound"
|
// A complete frame can survive a process crash in page cache even if its
|
||||||
);
|
// writer never completed sync. Durably adopt all validated frames before
|
||||||
length = length
|
// exposing them or allowing later recovery to truncate the native file.
|
||||||
.checked_add(size)
|
let redo = std::fs::OpenOptions::new().write(true).open(&journal)?;
|
||||||
.ok_or_else(|| anyhow!("archive redo offset overflow"))?;
|
if pos < data.len() {
|
||||||
pos = end;
|
redo.set_len(pos as u64)?;
|
||||||
}
|
}
|
||||||
// Native indexes are rebuildable. A crash may have resurrected an unlink
|
redo.sync_all()?;
|
||||||
// from the append path; invalidate it again before exposing recovered data.
|
// Validate the complete journal before changing native data. Replaying is
|
||||||
match std::fs::remove_file(p.with_extension("lidx")) {
|
// idempotent, including after a second crash midway through recovery.
|
||||||
Ok(()) => std::fs::File::open(p.parent().unwrap())?.sync_all()?,
|
let mut file = std::fs::OpenOptions::new().read(true).write(true).open(p)?;
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
|
let mut changed = file.metadata()?.len() != length;
|
||||||
Err(e) => return Err(e.into()),
|
for (offset, payload) in &frames {
|
||||||
}
|
|
||||||
// A complete frame can survive a process crash in page cache even if its
|
|
||||||
// writer never completed sync. Durably adopt all validated frames before
|
|
||||||
// exposing them or allowing later recovery to truncate the native file.
|
|
||||||
let redo = std::fs::OpenOptions::new().write(true).open(&journal)?;
|
|
||||||
if pos < data.len() {
|
|
||||||
redo.set_len(pos as u64)?;
|
|
||||||
}
|
|
||||||
redo.sync_all()?;
|
|
||||||
// Validate the complete journal before changing native data. Replaying is
|
|
||||||
// idempotent, including after a second crash midway through recovery.
|
|
||||||
let mut changed = file.metadata()?.len() != length;
|
|
||||||
for (offset, payload) in &frames {
|
|
||||||
file.seek(SeekFrom::Start(*offset))?;
|
|
||||||
let mut native = vec![0; payload.len()];
|
|
||||||
let equal = file.read_exact(&mut native).is_ok() && native.as_slice() == *payload;
|
|
||||||
if !equal {
|
|
||||||
file.seek(SeekFrom::Start(*offset))?;
|
file.seek(SeekFrom::Start(*offset))?;
|
||||||
file.write_all(payload)?;
|
let mut native = vec![0; payload.len()];
|
||||||
changed = true;
|
let equal = file.read_exact(&mut native).is_ok() && native.as_slice() == *payload;
|
||||||
|
if !equal {
|
||||||
|
file.seek(SeekFrom::Start(*offset))?;
|
||||||
|
file.write_all(payload)?;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
if changed {
|
||||||
|
file.set_len(length)?;
|
||||||
|
file.sync_all()?;
|
||||||
|
}
|
||||||
|
let state = RedoState {
|
||||||
|
length,
|
||||||
|
bytes: pos as u64,
|
||||||
|
frames: frames.len(),
|
||||||
|
};
|
||||||
|
if nm.is_none() {
|
||||||
|
return checkpoint_redo(p).map(Some);
|
||||||
|
}
|
||||||
|
remember_redo(p, &state)?;
|
||||||
|
Ok(Some(state))
|
||||||
|
})();
|
||||||
|
match recovered {
|
||||||
|
Err(error) if is_untrusted_redo(&error) => {
|
||||||
|
// The journal no longer matches its native baseline. Prosody itself
|
||||||
|
// would keep appending to the native file without any redo view:
|
||||||
|
// adopt that currently synced native as authoritative, retire the
|
||||||
|
// untrusted journal, and report "no redo". This keeps archive access
|
||||||
|
// at Prosody-level speed and makes the archive writable again without
|
||||||
|
// an explicit offline handoff, matching how the same data behaves
|
||||||
|
// under Prosody.
|
||||||
|
retire_redo(p)?;
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
Ok(state) => Ok(state),
|
||||||
|
Err(error) => Err(error),
|
||||||
}
|
}
|
||||||
if changed {
|
|
||||||
file.set_len(length)?;
|
|
||||||
file.sync_all()?;
|
|
||||||
}
|
|
||||||
let state = RedoState {
|
|
||||||
length,
|
|
||||||
bytes: pos as u64,
|
|
||||||
frames: frames.len(),
|
|
||||||
};
|
|
||||||
if nm.is_none() {
|
|
||||||
return checkpoint_redo(p).map(Some);
|
|
||||||
}
|
|
||||||
remember_redo(p, &state)?;
|
|
||||||
Ok(Some(state))
|
|
||||||
}
|
}
|
||||||
fn recover_append(p: &Path) -> Result<()> {
|
/// Recovery for read-only archive access. A journal whose baseline an external
|
||||||
recover_redo(p)?;
|
/// writer invalidated cannot be replayed safely; treat the durably synced native
|
||||||
Ok(())
|
/// as authoritative and read it as-is instead of failing every edge/scan/query.
|
||||||
|
/// Returns `Ok(true)` only when the baseline is untrusted (caller should proceed
|
||||||
|
/// on native), `Ok(false)` after a normal recovery, or `Err` for other failures.
|
||||||
|
fn recover_append_readonly(p: &Path) -> Result<bool> {
|
||||||
|
match recover_redo(p) {
|
||||||
|
Ok(_) => Ok(false),
|
||||||
|
Err(error) if is_untrusted_redo(&error) => Ok(true),
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
/// Durable handoff to a native writer. Call after stopping archive users and
|
/// Durable handoff to a native writer. Call after stopping archive users and
|
||||||
/// before letting Prosody modify `.list` files in place. Abrupt shutdown leaves
|
/// before letting Prosody modify `.list` files in place. Abrupt shutdown leaves
|
||||||
@@ -466,7 +504,13 @@ pub fn checkpoint_all(root: &Path) -> Result<usize> {
|
|||||||
/// is synced first, so a crash after unlink and before rename retains every
|
/// is synced first, so a crash after unlink and before rename retains every
|
||||||
/// committed record. After rename there is no old redo to resurrect deletions.
|
/// committed record. After rename there is no old redo to resurrect deletions.
|
||||||
fn retire_redo(p: &Path) -> Result<()> {
|
fn retire_redo(p: &Path) -> Result<()> {
|
||||||
std::fs::File::open(p)?.sync_all()?;
|
// The native file may not exist when the journal arrived without one (e.g.
|
||||||
|
// a `missing baseline` journal); it is still fine to retire the sidecar.
|
||||||
|
match std::fs::File::open(p) {
|
||||||
|
Ok(f) => f.sync_all()?,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
|
||||||
|
Err(e) => return Err(e.into()),
|
||||||
|
}
|
||||||
std::fs::remove_file(p.with_extension("rredo"))?;
|
std::fs::remove_file(p.with_extension("rredo"))?;
|
||||||
std::fs::File::open(p.parent().unwrap())?.sync_all()?;
|
std::fs::File::open(p.parent().unwrap())?.sync_all()?;
|
||||||
REDO_CACHE
|
REDO_CACHE
|
||||||
@@ -512,7 +556,7 @@ fn scan_unlocked<F: FnMut(Entry) -> Result<()>>(
|
|||||||
mut visit: F,
|
mut visit: F,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let path = storage::path(root, host, user, store).with_extension("list");
|
let path = storage::path(root, host, user, store).with_extension("list");
|
||||||
recover_append(&path)?;
|
recover_append_readonly(&path)?;
|
||||||
let file = match std::fs::File::open(&path) {
|
let file = match std::fs::File::open(&path) {
|
||||||
Ok(f) => f,
|
Ok(f) => f,
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||||
@@ -616,7 +660,7 @@ pub fn edges_store(
|
|||||||
let lock = archive_lock(root, host, user, store)?;
|
let lock = archive_lock(root, host, user, store)?;
|
||||||
let _guard = lock.lock().map_err(|_| anyhow!("archive lock poisoned"))?;
|
let _guard = lock.lock().map_err(|_| anyhow!("archive lock poisoned"))?;
|
||||||
let path = storage::path(root, host, user, store).with_extension("list");
|
let path = storage::path(root, host, user, store).with_extension("list");
|
||||||
recover_append(&path)?;
|
recover_append_readonly(&path)?;
|
||||||
let mut file = match std::fs::File::open(&path) {
|
let mut file = match std::fs::File::open(&path) {
|
||||||
Ok(file) => file,
|
Ok(file) => file,
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||||
@@ -943,7 +987,7 @@ fn retain_unlocked<F: FnMut(&Entry) -> bool>(
|
|||||||
mut retain: F,
|
mut retain: F,
|
||||||
) -> Result<usize> {
|
) -> Result<usize> {
|
||||||
let path = storage::path(root, host, user, store).with_extension("list");
|
let path = storage::path(root, host, user, store).with_extension("list");
|
||||||
recover_append(&path)?;
|
recover_append_readonly(&path)?;
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return Ok(0);
|
return Ok(0);
|
||||||
}
|
}
|
||||||
@@ -1090,12 +1134,15 @@ mod redo_rewrite_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_append_fails_closed_on_untrusted_redo_baseline() {
|
fn runtime_append_recovers_by_retiring_untrusted_redo_baseline() {
|
||||||
let (root, path) = rolled_fixture();
|
let (root, path) = rolled_fixture();
|
||||||
corrupt_baseline(&path);
|
corrupt_baseline(&path);
|
||||||
// A live append must fail closed (not silently overwrite the native
|
// Prosody never keeps a redo journal: it just appends to the native
|
||||||
// file): the journal no longer matches its baseline.
|
// file. When an external writer invalidated our baseline, behave the
|
||||||
let err = append_new(
|
// same way instead of blocking the archive. A live append retires the
|
||||||
|
// untrusted journal, adopts the synced native as authoritative, writes
|
||||||
|
// the record, and recreates a fresh journal on the new baseline.
|
||||||
|
let id = append_new(
|
||||||
root.path(),
|
root.path(),
|
||||||
"example.org",
|
"example.org",
|
||||||
Some("alice"),
|
Some("alice"),
|
||||||
@@ -1107,15 +1154,45 @@ mod redo_rewrite_tests {
|
|||||||
xml: "<message><body>new</body></message>".into(),
|
xml: "<message><body>new</body></message>".into(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.expect_err("live append must fail on an untrusted redo baseline");
|
.expect("live append must recover by retiring the untrusted baseline");
|
||||||
assert!(
|
assert_eq!(id, "post-corruption");
|
||||||
is_untrusted_redo(&err),
|
// A fresh journal on the current native length is recreated.
|
||||||
"expected an untrusted-baseline error, got: {err:#}"
|
|
||||||
);
|
|
||||||
// The journal is preserved for explicit offline handoff.
|
|
||||||
assert!(path.with_extension("rredo").exists());
|
assert!(path.with_extension("rredo").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn untrusted_redo_baseline_is_retired_once_and_recovers() {
|
||||||
|
let (root, path) = rolled_fixture();
|
||||||
|
corrupt_baseline(&path);
|
||||||
|
let make_entry = || Entry {
|
||||||
|
id: "post-corruption".into(),
|
||||||
|
when: 100000.,
|
||||||
|
peer: String::new(),
|
||||||
|
xml: "<message><body>new</body></message>".into(),
|
||||||
|
};
|
||||||
|
// The first access performs the full journal read+validation, then
|
||||||
|
// retires the untrusted journal exactly once.
|
||||||
|
append_new(
|
||||||
|
root.path(),
|
||||||
|
"example.org",
|
||||||
|
Some("alice"),
|
||||||
|
"archive",
|
||||||
|
make_entry(),
|
||||||
|
)
|
||||||
|
.expect("first append retires the untrusted journal");
|
||||||
|
// Subsequent appends are now fast and succeed on a healthy journal.
|
||||||
|
for _ in 0..20 {
|
||||||
|
append_new(
|
||||||
|
root.path(),
|
||||||
|
"example.org",
|
||||||
|
Some("alice"),
|
||||||
|
"archive",
|
||||||
|
make_entry(),
|
||||||
|
)
|
||||||
|
.expect("archive stays writable after recovery");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn offline_checkpoint_retires_untrusted_redo_baseline() {
|
fn offline_checkpoint_retires_untrusted_redo_baseline() {
|
||||||
let (root, path) = rolled_fixture();
|
let (root, path) = rolled_fixture();
|
||||||
|
|||||||
@@ -3,21 +3,22 @@
|
|||||||
|
|
||||||
Usage: python3 tests/mam_broken_baseline_network.py target/release/rust-xmpp
|
Usage: python3 tests/mam_broken_baseline_network.py target/release/rust-xmpp
|
||||||
|
|
||||||
Reproduces the production symptom reported with Monal: the native archive at
|
Reproduces the production symptom reported with Monal/Luma: the native archive at
|
||||||
`data/<host>/archive/<user>.list` no longer matches the anchor recorded in its
|
`data/<host>/archive/<user>.list` no longer matches the anchor recorded in its
|
||||||
`.rredo` journal ("archive redo baseline changed in place"). On every message the
|
`.rredo` journal ("archive redo baseline changed in place"). On every message the
|
||||||
previous build drained the session with `connection-closed reason=
|
previous build drained the session or stalled delivery for seconds, so the client
|
||||||
protocol-or-transport-error`, so the client reconnect-looped and its message was
|
timed out MAM and its message was delayed.
|
||||||
never delivered.
|
|
||||||
|
|
||||||
The archive store is a durable attribute of delivery, not a prerequisite. This
|
Prosody has no `.rredo` sidecar: it simply appends to the native `.list`. This
|
||||||
test seeds an archive for `load-0`, corrupts its redo baseline offline to force
|
test seeds an archive for `load-0`, corrupts its redo baseline offline to force
|
||||||
exactly that recovery error, then verifies that:
|
that recovery error, then verifies that:
|
||||||
* a fresh message is still routed load-0 -> load-1,
|
* a fresh message is still routed load-0 -> load-1 quickly,
|
||||||
* BOTH sessions survive (ping round-trip), no `connection-closed`,
|
* BOTH sessions survive (ping round-trip), no `connection-closed`,
|
||||||
* the runtime stays fail-closed (the untrusted journal is NOT silently dropped),
|
* the runtime self-repairs: on the first access it retires the untrusted
|
||||||
* the offline `storage-checkpoint` repair retires the untrusted journal and
|
journal, adopts the synced native as authoritative (exactly what Prosody
|
||||||
restores MAM history reads on the now-authoritative native file.
|
would treat as the archive), and continues appending on a fresh journal
|
||||||
|
without requiring an offline `storage-checkpoint` handoff,
|
||||||
|
* MAM history reads succeed on the recovered archive.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
@@ -246,10 +247,6 @@ def run(binary):
|
|||||||
server.wait()
|
server.wait()
|
||||||
|
|
||||||
log_text = (root / "server.log").read_text()
|
log_text = (root / "server.log").read_text()
|
||||||
skipped = [l for l in log_text.splitlines() if "mam-archive-skipped" in l]
|
|
||||||
assert skipped, (
|
|
||||||
"server did not log the best-effort archive skip (isolation inactive?)"
|
|
||||||
)
|
|
||||||
assert not any(
|
assert not any(
|
||||||
"connection-closed" in l and "protocol-or-transport-error" in l
|
"connection-closed" in l and "protocol-or-transport-error" in l
|
||||||
for l in log_text.splitlines()
|
for l in log_text.splitlines()
|
||||||
@@ -258,25 +255,29 @@ def run(binary):
|
|||||||
"unhandled archive fault labelled a session error"
|
"unhandled archive fault labelled a session error"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Runtime must stay fail-closed: the untrusted journal is preserved.
|
# Prosody-compatible self-repair: the first access after the corruption
|
||||||
assert sender_archive.with_suffix(".rredo").read_bytes() == damaged_journal, (
|
# retired the untrusted journal and the archive was appended to on a fresh
|
||||||
"runtime dropped the untrusted redo journal instead of failing closed"
|
# journal. No `mam-archive-skipped` should have been logged because the
|
||||||
|
# event delivery succeeded on the recovered native.
|
||||||
|
assert not any("mam-archive-skipped" in l for l in log_text.splitlines())
|
||||||
|
|
||||||
|
# A fresh journal must have replaced the damaged one (the archive kept
|
||||||
|
# accepting records). Its baseline now matches the current native.
|
||||||
|
journal = sender_archive.with_suffix(".rredo")
|
||||||
|
assert journal.exists(), "runtime did not recreate a fresh redo journal"
|
||||||
|
assert journal.read_bytes() != damaged_journal, (
|
||||||
|
"runtime kept the damaged journal instead of retiring it"
|
||||||
|
)
|
||||||
|
fresh = journal.read_bytes()
|
||||||
|
assert len(fresh) >= REDO_HEADER
|
||||||
|
assert fresh[:8] == REDO_MAGIC
|
||||||
|
# The fresh journal is valid: its header digest checks out, so recovery
|
||||||
|
# will accept it as the authoritative pairing with the current native.
|
||||||
|
assert hashlib.sha256(fresh[:64]).digest() == fresh[64:96], (
|
||||||
|
"recreated journal header is not self-consistent"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Offline handoff must repair: checkpoint retires the untrusted journal
|
# MAM history on the recovered, now-authoritative native must serve reads.
|
||||||
# so the now-authoritative native file becomes a clean one-journal base.
|
|
||||||
result = subprocess.run(
|
|
||||||
[binary, "storage-checkpoint", str(config)],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
assert "Durable native archive handoff:" in result.stdout
|
|
||||||
assert not sender_archive.with_suffix(".rredo").exists(), (
|
|
||||||
"storage-checkpoint did not retire the untrusted redo journal"
|
|
||||||
)
|
|
||||||
|
|
||||||
# After repair MAM history is readable again (endpoint reads now succeed).
|
|
||||||
phase3_offset = len((root / "server.log").read_text().splitlines())
|
phase3_offset = len((root / "server.log").read_text().splitlines())
|
||||||
with (root / "server.log").open("a") as log:
|
with (root / "server.log").open("a") as log:
|
||||||
server = None
|
server = None
|
||||||
@@ -291,7 +292,6 @@ def run(binary):
|
|||||||
alice.available()
|
alice.available()
|
||||||
bob.available()
|
bob.available()
|
||||||
# A MAM query to load-0's own archive must serve history.
|
# A MAM query to load-0's own archive must serve history.
|
||||||
# Query the bare account JID (not the full resource).
|
|
||||||
alice.send(
|
alice.send(
|
||||||
f"<iq type='set' id='mam' to='load-0@localhost'>"
|
f"<iq type='set' id='mam' to='load-0@localhost'>"
|
||||||
f"<query xmlns='urn:xmpp:mam:2'/>"
|
f"<query xmlns='urn:xmpp:mam:2'/>"
|
||||||
@@ -305,8 +305,7 @@ def run(binary):
|
|||||||
)
|
)
|
||||||
is None
|
is None
|
||||||
), ET.tostring(result_iq)
|
), ET.tostring(result_iq)
|
||||||
# After reconnect the sender archive is fresh (journal was retired);
|
# A new message still archives cleanly on the recovered journal.
|
||||||
# a new message archives cleanly.
|
|
||||||
alice.send(
|
alice.send(
|
||||||
f"<message to='{bob.jid}' type='chat' id='post-repair'><body>post-repair</body></message>"
|
f"<message to='{bob.jid}' type='chat' id='post-repair'><body>post-repair</body></message>"
|
||||||
)
|
)
|
||||||
@@ -322,9 +321,19 @@ def run(binary):
|
|||||||
l for l in post_log if "archive redo baseline changed in place" in l
|
l for l in post_log if "archive redo baseline changed in place" in l
|
||||||
]
|
]
|
||||||
assert not fresh_errors, (
|
assert not fresh_errors, (
|
||||||
"archive fault returned after offline repair:\n" + "\n".join(fresh_errors)
|
"archive fault returned after self-repair:\n" + "\n".join(fresh_errors)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Offline handoff stays safe: checkpoint of an already-repaired archive is
|
||||||
|
# idempotent (retires an absent journal) and still reports a handoff.
|
||||||
|
result = subprocess.run(
|
||||||
|
[binary, "storage-checkpoint", str(config)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
assert "Durable native archive handoff:" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
|||||||
Reference in new issue
Block a user