initial commit

scripts/docker-entrypoint.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+# Never recursively chown an existing Prosody volume. Match its existing owner.
+if [ "$(id -u)" = 0 ]; then
+    data_dir=${XMPP_DATA_DIR:-/var/lib/prosody}
+    if [ ! -d "$data_dir" ]; then
+        mkdir -p "$data_dir"
+        chown "${XMPP_UID:-1000}:${XMPP_GID:-1000}" "$data_dir"
+    fi
+    runtime_uid=${XMPP_UID:-$(stat -c %u "$data_dir")}
+    runtime_gid=${XMPP_GID:-$(stat -c %g "$data_dir")}
+    case "$runtime_uid:$runtime_gid" in
+        *[!0-9:]*|:*|*:) echo 'XMPP_UID and XMPP_GID must be numeric' >&2; exit 1 ;;
+    esac
+    if [ "$runtime_uid" != 0 ]; then
+        exec gosu "$runtime_uid:$runtime_gid" rust-xmpp "$@"
+    fi
+fi
+exec rust-xmpp "$@"

scripts/filter-android-pep-log.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Read client log lines on stdin; print only fixed PEP warning classifications.
+
+Usage: adb logcat -v brief | python3 scripts/filter-android-pep-log.py
+Or: python3 scripts/filter-android-pep-log.py < client.log
+Raw input, JIDs, XML, tokens, key material and message text are never printed.
+These phrases describe inspected client source, not every client/version.
+"""
+import json
+import sys
+
+PHRASES = {
+    "Own device publish attempt threshold exceeded": "empty-device-list-retry-limit",
+    "Own device list empty, attempting to publish": "empty-device-list-publish",
+    "Error received while publishing own device id": "device-list-publish-error",
+    "request for device bundles came back with something other than item-not-found": "bundle-read-error",
+    "Error received while publishing bundle": "bundle-publish-error",
+}
+for line in sys.stdin:
+    normalized = " ".join(line.split())
+    for phrase, reason in PHRASES.items():
+        if phrase in normalized:
+            print(json.dumps({"event": "client-pep-warning", "reason": reason}), flush=True)
+            break

scripts/integration_http.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""Real binary integration: TLS WebSocket/BOSH auth, roster/chat and XEP-0363.
+Requires Python websockets and openssl. Uses temporary data and loopback ports only.
+"""
+import argparse
+import asyncio
+import base64
+import json
+import http.client
+import urllib.parse
+import pathlib
+import socket
+import ssl
+import subprocess
+import tempfile
+import time
+import urllib.request
+import urllib.error
+import xml.etree.ElementTree as ET
+import websockets
+
+CLIENT = 'jabber:client'
+BIND = 'urn:ietf:params:xml:ns:xmpp-bind'
+BOSH = 'http://jabber.org/protocol/httpbind'
+SASL = 'urn:ietf:params:xml:ns:xmpp-sasl'
+UPLOAD = 'urn:xmpp:http:upload:0'
+SM = 'urn:xmpp:sm:3'
+FRAMING = 'urn:ietf:params:xml:ns:xmpp-framing'
+
+
+def port():
+    with socket.socket() as sock:
+        sock.bind(('127.0.0.1', 0))
+        return sock.getsockname()[1]
+
+
+def plain(user):
+    return base64.b64encode(f'\0{user}\0integration-password'.encode()).decode()
+
+
+class BoshClient:
+    def __init__(self, url, context):
+        self.url, self.context, self.sid, self.rid = url, context, None, 100
+
+    def send(self, stanza='', **attributes):
+        attributes.update({'xmlns': BOSH, 'rid': str(self.rid)})
+        self.rid += 1
+        if self.sid:
+            attributes['sid'] = self.sid
+        # Values are controlled constants in this integration fixture.
+        attrs = ' '.join(f'{k}="{v}"' for k, v in attributes.items())
+        request = urllib.request.Request(self.url, f'<body {attrs}>{stanza}</body>'.encode(),
+                                         headers={'Content-Type': 'text/xml'}, method='POST')
+        with urllib.request.urlopen(request, context=self.context, timeout=8) as response:
+            xml = response.read()
+        body = ET.fromstring(xml)
+        assert body.get('type') != 'terminate' or attributes.get('type') == 'terminate', xml
+        if body.get('sid'):
+            self.sid = body.get('sid')
+        return body
+
+    def login(self, user):
+        first = self.send(to='localhost', wait='1', hold='1', ver='1.11')
+        assert first.find(f'.//{{{SASL}}}mechanism') is not None, ET.tostring(first)
+        auth = self.send(f'<auth xmlns="{SASL}" mechanism="PLAIN">{plain(user)}</auth>')
+        assert auth.find(f'{{{SASL}}}success') is not None, ET.tostring(auth)
+        self.send(**{'xmlns:xmpp': 'urn:xmpp:xbosh', 'xmpp:restart': 'true', 'to': 'localhost'})
+        bound = self.send(f'<iq xmlns="{CLIENT}" type="set" id="bind"><bind xmlns="{BIND}"><resource>bosh</resource></bind></iq>')
+        jid = bound.find(f'.//{{{BIND}}}jid')
+        assert jid is not None and jid.text == f'{user}@localhost/bosh', ET.tostring(bound)
+
+    def enable_resume(self):
+        body = self.send(f'<enable xmlns="{SM}" resume="true"/>')
+        enabled = body.find(f'{{{SM}}}enabled')
+        assert enabled is not None and enabled.get('id') and enabled.get('resume') in ('true', '1'), ET.tostring(body)
+
+
+async def ws_receive(ws, predicate, timeout=8):
+    async with asyncio.timeout(timeout):
+        while True:
+            raw = await ws.recv()
+            node = ET.fromstring(raw)
+            if predicate(node):
+                return node
+
+
+async def ws_login(ws, user):
+    opening = f'<open xmlns="{FRAMING}" to="localhost" version="1.0"/>'
+    await ws.send(opening)
+    features = await ws_receive(ws, lambda n: n.tag.endswith('}features'))
+    assert features.find(f'.//{{{SASL}}}mechanism') is not None
+    await ws.send(f'<auth xmlns="{SASL}" mechanism="PLAIN">{plain(user)}</auth>')
+    await ws_receive(ws, lambda n: n.tag == f'{{{SASL}}}success')
+    await ws.send(opening)
+    await ws_receive(ws, lambda n: n.tag.endswith('}features'))
+    await ws.send(f'<iq xmlns="{CLIENT}" type="set" id="bind"><bind xmlns="{BIND}"><resource>websocket</resource></bind></iq>')
+    result = await ws_receive(ws, lambda n: n.tag == f'{{{CLIENT}}}iq' and n.get('id') == 'bind')
+    jid = result.find(f'.//{{{BIND}}}jid')
+    assert result.get('type') == 'result' and jid is not None and jid.text == f'{user}@localhost/websocket', ET.tostring(result)
+
+
+async def ws_enable_resume(ws):
+    await ws.send(f'<enable xmlns="{SM}" resume="true"/>')
+    enabled = await ws_receive(ws, lambda n: n.tag in (f'{{{SM}}}enabled', f'{{{SM}}}failed'))
+    assert enabled.tag == f'{{{SM}}}enabled' and enabled.get('id') and enabled.get('resume') in ('true', '1'), ET.tostring(enabled)
+
+
+async def exercise(https_port, context, custom_upload=False):
+    endpoint = f'https://localhost:{https_port}'
+    bosh = BoshClient(endpoint + '/http-bind', context)
+    await asyncio.to_thread(bosh.login, 'alice')
+    async with websockets.connect(f'wss://localhost:{https_port}/xmpp-websocket', ssl=context,
+                                  subprotocols=['xmpp'], open_timeout=8) as ws:
+        await ws_login(ws, 'bob')
+        await ws.send('<inactive xmlns="urn:xmpp:csi:0"/>')
+        await ws.send('<active xmlns="urn:xmpp:csi:0"/>')
+        await ws.send(f'<iq xmlns="{CLIENT}" type="get" id="after-csi" to="localhost"><ping xmlns="urn:xmpp:ping"/></iq>')
+        await ws_receive(ws, lambda n: n.get('id') == 'after-csi' and n.get('type') == 'result')
+        namespaced = await asyncio.to_thread(bosh.send, '<c:iq type="get" id="inherited-ns" to="localhost"><ping xmlns="urn:xmpp:ping"/></c:iq>', **{'xmlns:c': CLIENT})
+        assert any(n.get('id') == 'inherited-ns' and n.get('type') == 'result' for n in namespaced), ET.tostring(namespaced)
+        # Both become available before ordinary bare-JID routing.
+        await ws.send(f'<presence xmlns="{CLIENT}"/>')
+        await asyncio.to_thread(bosh.send, f'<presence xmlns="{CLIENT}"/>')
+        roster = await asyncio.to_thread(bosh.send, f'<iq xmlns="{CLIENT}" type="set" id="r1"><query xmlns="jabber:iq:roster"><item jid="bob@localhost" name="Bob"/></query></iq>')
+        assert any(n.get('id') == 'r1' and n.get('type') == 'result' for n in roster), ET.tostring(roster)
+        roster = await asyncio.to_thread(bosh.send, f'<iq xmlns="{CLIENT}" type="get" id="r2"><query xmlns="jabber:iq:roster"/></iq>')
+        assert roster.find('.//{jabber:iq:roster}item').get('jid') == 'bob@localhost'
+        await asyncio.to_thread(bosh.send, f'<message xmlns="{CLIENT}" to="bob@localhost" type="chat" id="hello"><body>from-bosh</body></message>')
+        incoming = await ws_receive(ws, lambda n: n.tag == f'{{{CLIENT}}}message' and n.get('id') == 'hello')
+        assert incoming.find(f'{{{CLIENT}}}body').text == 'from-bosh'
+        await ws.send(f'<message xmlns="{CLIENT}" to="alice@localhost" type="chat" id="reply"><body>from-websocket</body></message>')
+        poll = await asyncio.to_thread(bosh.send)
+        assert any(n.get('id') == 'reply' for n in poll), ET.tostring(poll)
+        await ws.send(f'<iq xmlns="{CLIENT}" type="get" id="ws-roster"><query xmlns="jabber:iq:roster"/></iq>')
+        await ws_receive(ws, lambda n: n.tag == f'{{{CLIENT}}}iq' and n.get('id') == 'ws-roster' and n.get('type') == 'result')
+        data = b'actual integrated HTTP upload\n' * 16384
+        slots = await asyncio.to_thread(bosh.send, f'<iq xmlns="{CLIENT}" to="upload.localhost" type="get" id="slot"><request xmlns="{UPLOAD}" filename="sample.txt" size="{len(data)}" content-type="text/plain"/></iq>')
+        slot = slots.find(f'.//{{{UPLOAD}}}slot')
+        assert slot is not None, ET.tostring(slots)
+        put = slot.find(f'{{{UPLOAD}}}put')
+        if custom_upload:
+            assert urllib.parse.urlsplit(put.get('url')).path.startswith('/custom/upload-upload.localhost/'), put.get('url')
+        authorization = put.find(f'{{{UPLOAD}}}header').text
+        request = urllib.request.Request(put.get('url'), data, headers={'Authorization': authorization}, method='PUT')
+        def transfer():
+            with urllib.request.urlopen(request, context=context, timeout=8) as response:
+                assert response.status == 201
+            with urllib.request.urlopen(slot.find(f'{{{UPLOAD}}}get').get('url'), context=context, timeout=8) as response:
+                assert response.read() == data
+            parsed = urllib.parse.urlsplit(put.get('url'))
+            duplicate = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=8)
+            duplicate.putrequest('PUT', parsed.path)
+            duplicate.putheader('Authorization', authorization)
+            duplicate.putheader('Content-Length', str(len(data)))
+            duplicate.endheaders()
+            # Server rejects a completed slot before accepting its body.
+            assert duplicate.getresponse().status == 409
+            duplicate.close()
+        await asyncio.to_thread(transfer)
+        # Graceful transport shutdown must release even a resumable SM resource.
+        # kick_new makes an accidentally hibernating old resource fail this bind.
+        await asyncio.to_thread(bosh.enable_resume)
+        terminated = await asyncio.to_thread(bosh.send, type='terminate')
+        assert terminated.get('type') == 'terminate', ET.tostring(terminated)
+        again = BoshClient(endpoint + '/http-bind', context)
+        await asyncio.to_thread(again.login, 'alice')
+        await asyncio.to_thread(again.enable_resume)
+        ping = await asyncio.to_thread(again.send, f'<iq xmlns="{CLIENT}" type="get" id="bosh-relogin"><ping xmlns="urn:xmpp:ping"/></iq>')
+        assert any(n.get('id') == 'bosh-relogin' and n.get('type') == 'result' for n in ping), ET.tostring(ping)
+        await asyncio.to_thread(again.send, type='terminate')
+        await ws_enable_resume(ws)
+        await ws.send(f'<close xmlns="{FRAMING}"/>')
+        await ws_receive(ws, lambda n: n.tag == f'{{{FRAMING}}}close')
+    async with websockets.connect(f'wss://localhost:{https_port}/xmpp-websocket', ssl=context,
+                                  subprotocols=['xmpp'], open_timeout=8) as ws:
+        await ws_login(ws, 'bob')
+        await ws_enable_resume(ws)
+        await ws.send(f'<iq xmlns="{CLIENT}" type="get" id="ws-relogin"><ping xmlns="urn:xmpp:ping"/></iq>')
+        await ws_receive(ws, lambda n: n.get('id') == 'ws-relogin' and n.get('type') == 'result')
+        await ws.send(f'<close xmlns="{FRAMING}"/>')
+        await ws_receive(ws, lambda n: n.tag == f'{{{FRAMING}}}close')
+    print('PASS: integrated WSS + HTTPS BOSH SASL/bind, CSI, inherited BOSH namespaces, roster, bidirectional chat, upload IQ/PUT/GET/reuse, graceful SM close and same-resource relogin under kick_new')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--binary', default=str(pathlib.Path(__file__).resolve().parents[1] / 'target/release/rust-xmpp'))
+    parser.add_argument('--custom-upload-path', action='store_true')
+    args = parser.parse_args()
+    with tempfile.TemporaryDirectory(prefix='rust-xmpp-http-integration-') as work:
+        work = pathlib.Path(work)
+        cert, key = work / 'cert.pem', work / 'key.pem'
+        subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
+                        '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost,DNS:upload.localhost',
+                        '-addext', 'basicConstraints=critical,CA:FALSE', '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True)
+        tcp_port, http_port, https_port = port(), port(), port()
+        config = {'global': {'data_path': str(work / 'data'), 'interfaces': ['127.0.0.1'], 'http_interfaces': ['127.0.0.1'],
+                            'https_interfaces': ['127.0.0.1'], 'c2s_ports': [tcp_port], 's2s_ports': [],
+                            'http_ports': [http_port], 'https_ports': [https_port], 'allow_registration': False,
+                            'consider_bosh_secure': True, 'consider_websocket_secure': True,
+                            'modules_enabled': ['tls', 'saslauth', 'disco', 'roster', 'websocket', 'bosh', 'csi_simple', 'smacks', 'ping'],
+                            'ssl': {'certificate': str(cert), 'key': str(key)}},
+                  'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'password_hash': 'SHA-256', 'conflict_resolve': 'kick_new'}},
+                            'upload.localhost': {'component': 'http_file_share', 'options': {'http_host': 'localhost', 'http_external_url': f'https://localhost:{https_port}',
+                                 'http_file_share_secret': 'integration-test-secret', 'http_file_share_size_limit': 1024*1024,
+                                 'http_file_share_daily_quota': 10*1024*1024}}}}
+        if args.custom_upload_path:
+            config['hosts']['upload.localhost']['options']['http_paths'] = {'file_share': '/custom/upload-$host'}
+        path = work / 'config.json'
+        path.write_text(json.dumps(config))
+        for user in ['alice', 'bob']:
+            subprocess.run([args.binary, 'create-user', str(path), user, '--password-stdin'], input='integration-password\n', text=True, check=True, capture_output=True)
+        log = open(work / 'server.log', 'w+')
+        process = subprocess.Popen([args.binary, 'serve', str(path), '--experimental'], stdout=log, stderr=log)
+        try:
+            for _ in range(100):
+                if process.poll() is not None:
+                    raise RuntimeError('server exited during startup')
+                try:
+                    with socket.create_connection(('127.0.0.1', tcp_port), timeout=.1):
+                        break
+                except OSError:
+                    time.sleep(.05)
+            else:
+                raise RuntimeError('server startup timed out')
+            context = ssl.create_default_context(cafile=str(cert))
+            asyncio.run(exercise(https_port, context, args.custom_upload_path))
+            if args.custom_upload_path:
+                print('PASS: configured upload HTTP path with $host through integrated listener')
+        except BaseException:
+            log.flush()
+            log.seek(0)
+            print('SERVER LOG:\n' + log.read())
+            raise
+        finally:
+            process.terminate()
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait()
+            log.close()
+
+
+if __name__ == '__main__':
+    main()

scripts/integration_xmpp.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Independent TCP integration tests for integrated Rust XMPP modules (isolated data)."""
+import base64, json, signal, socket, ssl, subprocess, tempfile, time
+from pathlib import Path
+import xml.etree.ElementTree as ET
+ROOT=Path(__file__).resolve().parents[1]
+BIN=ROOT/'target/release/rust-xmpp'
+C='jabber:client'; S='http://etherx.jabber.org/streams'
+OPEN=f"<stream:stream xmlns='{C}' xmlns:stream='{S}' to='localhost' version='1.0'>"
+def local(e):return e.tag.split('}')[-1]
+class Client:
+ def __init__(self,port,cert,user,password,resource='main',bind=True):
+  self.sock=socket.create_connection(('127.0.0.1',port),timeout=5);self.buf=b'';self.queue=[];self.user=user;self.port=port;self.cert=cert;self.password=password
+  self.send(OPEN);self.until(b'</stream:features>');self.send("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>");self.until(b'/>')
+  self.sock=ssl.create_default_context(cafile=str(cert)).wrap_socket(self.sock,server_hostname='localhost');self.send(OPEN);self.until(b'</stream:features>')
+  payload=base64.b64encode(f'\0{user}\0{password}'.encode()).decode();self.send(f"<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='PLAIN'>{payload}</auth>");self.until(b'/>');self.send(OPEN);self.until(b'</stream:features>')
+  self.jid=f'{user}@localhost/{resource}'
+  if bind:self.send(f"<iq type='set' id='bind'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>{resource}</resource></bind></iq>");self.iq('bind')
+ def send(self,s):self.sock.sendall(s.encode())
+ def until(self,mark):
+  while mark not in self.buf:
+   chunk=self.sock.recv(65536)
+   if not chunk:raise RuntimeError('EOF '+repr(self.buf))
+   self.buf+=chunk
+  end=self.buf.index(mark)+len(mark);out,self.buf=self.buf[:end],self.buf[end:];return out
+ def read(self):
+  parser=ET.XMLPullParser(events=('start','end'));parser.feed(f"<root xmlns='{C}' xmlns:stream='{S}'>");list(parser.read_events());depth=0
+  while True:
+   if not self.buf:
+    self.buf=self.sock.recv(65536)
+    if not self.buf:raise RuntimeError('EOF')
+   byte,self.buf=self.buf[:1],self.buf[1:];parser.feed(byte)
+   # Expat reparse deferral may withhold a short end-tag when fed bytewise.
+   # Flush at markup boundaries so an empty IQ result needs no following stanza.
+   if byte==b'>' and hasattr(parser,'flush'):parser.flush()
+   for event,node in parser.read_events():
+    depth+=1 if event=='start' else -1
+    if event=='end' and depth==0:return node
+ def take(self,predicate):
+  for i,e in enumerate(self.queue):
+   if predicate(e):return self.queue.pop(i)
+  while True:
+   e=self.read()
+   if predicate(e):return e
+   self.queue.append(e)
+ def iq(self,id,error=False):
+  e=self.take(lambda e:local(e)=='iq' and e.get('id')==id and e.get('type') in ('result','error'))
+  assert e.get('type')==('error' if error else 'result'),ET.tostring(e)
+  return e
+ def sync(self):self.send("<iq type='get' id='sync'><ping xmlns='urn:xmpp:ping'/></iq>");self.iq('sync')
+ def available(self):self.send('<presence/>');self.sync()
+ def body(self,text):return self.take(lambda e:local(e)=='message' and any(local(c)=='body' and c.text==text for c in e))
+ def close(self):self.sock.close()
+
+with tempfile.TemporaryDirectory(prefix='rust-xmpp-network-') as folder:
+ path=Path(folder);port=socket.socket();port.bind(('127.0.0.1',0));portnum=port.getsockname()[1];port.close()
+ subprocess.run(['openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','2','-keyout',str(path/'key.pem'),'-out',str(path/'cert.pem'),'-subj','/CN=localhost','-addext','subjectAltName=DNS:localhost'],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ modules=['tls','saslauth','sasl2','sasl_ssdp','roster','disco','private','vcard_legacy','blocklist','mam','pep','bookmarks','smacks','carbons','ping','version','register','cloud_notify','time','uptime','csi_simple']
+ config={'global':{'data_path':str(path/'data'),'interfaces':['127.0.0.1'],'c2s_ports':[portnum],'s2s_ports':[],'http_ports':[],'https_ports':[],'allow_registration':False,'default_archive_policy':True,'modules_enabled':modules},'hosts':{'localhost':{'options':{'authentication':'internal_hashed','password_hash':'SHA-256','ssl':{'key':str(path/'key.pem'),'certificate':str(path/'cert.pem')}}},'conference.localhost':{'component':'muc','options':{'modules_enabled':['muc_mam'],'muc_room_default_persistent':True}},'pubsub.localhost':{'component':'pubsub','options':{'restrict_node_creation':False}}}}
+ (path/'config.json').write_text(json.dumps(config))
+ for user in ['alice','bob']:
+  subprocess.run([str(BIN),'create-user',str(path/'config.json'),user,'--password-stdin'],input=f'{user}-secret\n',text=True,check=True,stdout=subprocess.DEVNULL)
+ server=subprocess.Popen([str(BIN),'serve',str(path/'config.json'),'--experimental'],stdout=subprocess.PIPE,stderr=subprocess.PIPE);clients=[]
+ try:
+  for _ in range(100):
+   if server.poll() is not None:raise RuntimeError(server.stderr.read().decode())
+   try:s=socket.create_connection(('127.0.0.1',portnum),timeout=.1);s.close();break
+   except OSError:time.sleep(.05)
+  a=Client(portnum,path/'cert.pem','alice','alice-secret');b=Client(portnum,path/'cert.pem','bob','bob-secret');clients=[a,b];a.available();b.available()
+  a.send("<iq type='set' id='roster'><query xmlns='jabber:iq:roster'><item jid='bob@localhost' name='Bob'><group>Friends</group></item></query></iq>");a.iq('roster')
+  a.send("<iq type='get' id='getroster'><query xmlns='jabber:iq:roster'/></iq>");assert any(e.get('jid')=='bob@localhost' for e in a.iq('getroster').iter())
+  a.send("<presence type='subscribe' to='bob@localhost'/>");b.take(lambda e:local(e)=='presence' and e.get('type')=='subscribe')
+  b.send("<presence type='subscribed' to='alice@localhost'/>");b.sync();a.sync()
+  a.send("<iq type='get' id='substate'><query xmlns='jabber:iq:roster'/></iq>");assert any(e.get('subscription')=='to' for e in a.iq('substate').iter())
+  a.send(f"<iq type='get' id='client-iq' to='{b.jid}'><query xmlns='jabber:iq:version'/></iq>")
+  ask=b.take(lambda e:local(e)=='iq' and e.get('id')=='client-iq');assert ask.get('from')==a.jid
+  b.send(f"<iq type='result' id='client-iq' from='forged@localhost' to='{a.jid}'><query xmlns='jabber:iq:version'><name>Bob Client</name></query></iq>");assert a.iq('client-iq').get('from')==b.jid
+  a.send("<message type='chat' id='chat1' to='bob@localhost'><body>archived</body></message>");b.body('archived')
+  a.send("<message to='bob@localhost/obsolete-resource' type='chat'><body>resource fallback</body></message>");assert b.body('resource fallback').get('to')=='bob@localhost/obsolete-resource'
+  # Inactive CSI holds chat-state noise, but a real message flushes it in order.
+  b.send("<inactive xmlns='urn:xmpp:csi:0'/>");time.sleep(.05)
+  a.send("<message to='bob@localhost' id='csi-noise'><composing xmlns='http://jabber.org/protocol/chatstates'/></message>");a.sync()
+  assert not b.buf,repr(b.buf)
+  b.sock.settimeout(.15)
+  try:
+   unexpected=b.sock.recv(1);raise AssertionError(('CSI leaked unimportant output',unexpected))
+  except socket.timeout:pass
+  finally:b.sock.settimeout(5)
+  a.send("<message to='bob@localhost'><body>CSI flush</body></message>")
+  noise=b.take(lambda e:e.get('id')=='csi-noise');assert any(n.tag=='{urn:xmpp:delay}delay' for n in noise)
+  b.body('CSI flush')
+  b.send("<inactive xmlns='urn:xmpp:csi:0'/>");time.sleep(.05)
+  a.send("<message to='bob@localhost' id='csi-keepalive'><composing xmlns='http://jabber.org/protocol/chatstates'/></message>");a.sync();time.sleep(.05)
+  b.send(' ');b.take(lambda e:e.get('id')=='csi-keepalive')
+  b.send("<active xmlns='urn:xmpp:csi:0'/>")
+  a.send("<iq type='set' id='mam'><query xmlns='urn:xmpp:mam:2' queryid='q1'/></iq>");a.iq('mam');assert any(any(n.tag=='{urn:xmpp:mam:2}result' for n in e) for e in a.queue)
+  b.send("<iq type='set' id='block'><block xmlns='urn:xmpp:blocking'><item jid='alice@localhost'/></block></iq>");b.iq('block')
+  a.send("<message to='bob@localhost'><body>blocked</body></message>");a.sync()
+  b.send("<iq type='set' id='unblock'><unblock xmlns='urn:xmpp:blocking'/></iq>");b.iq('unblock')
+  a.send("<iq type='set' id='pep'><pubsub xmlns='http://jabber.org/protocol/pubsub'><publish node='eu.siacs.conversations.axolotl.devicelist'><item id='current'><list xmlns='eu.siacs.conversations.axolotl'><device id='123'/></list></item></publish></pubsub></iq>");a.iq('pep')
+  a.send("<iq type='get' id='pepget'><pubsub xmlns='http://jabber.org/protocol/pubsub'><items node='eu.siacs.conversations.axolotl.devicelist'/></pubsub></iq>");assert any(n.tag=='{eu.siacs.conversations.axolotl}device' for n in a.iq('pepget').iter())
+  for c,nick in [(a,'Alice'),(b,'Bob')]:
+   c.send(f"<presence to='room@conference.localhost/{nick}'><x xmlns='http://jabber.org/protocol/muc'/></presence>");c.take(lambda e:local(e)=='presence' and any(n.get('code')=='110' for n in e.iter()))
+   if c is a:c.send("<iq type='set' id='unlock' to='room@conference.localhost'><query xmlns='http://jabber.org/protocol/muc#owner'><x xmlns='jabber:x:data' type='submit'/></query></iq>");c.iq('unlock')
+  a.send("<message to='room@conference.localhost' type='groupchat'><body>room chat</body></message>");assert b.body('room chat').get('from')=='room@conference.localhost/Alice'
+  b.close();clients.remove(b);time.sleep(.1)
+  a.send("<message to='bob@localhost'><body>offline replay</body></message>");a.sync()
+  a.send("<message to='bob@localhost' type='headline' id='offline-headline'><body>discard headline</body></message>");a.sync()
+  assert not any(e.get('id')=='offline-headline' and e.get('type')=='error' for e in a.queue),'offline headline must be silently discarded'
+  # no-permanent-store forbids MAM, but must not discard transient offline delivery.
+  a.send("<message to='bob@localhost' id='offline-transient'><body>transient offline</body><no-permanent-store xmlns='urn:xmpp:hints'/></message>")
+  # Bodyless receipts are also messages handled by native mod_message/mod_offline.
+  a.send("<message to='bob@localhost' id='offline-receipt'><received xmlns='urn:xmpp:receipts' id='old-message'/></message>");a.sync()
+  b=Client(portnum,path/'cert.pem','bob','bob-secret','second');clients.append(b);b.available();b.body('offline replay');b.body('transient offline')
+  receipt=b.take(lambda e:e.get('id')=='offline-receipt');assert any(n.tag=='{urn:xmpp:receipts}received' for n in receipt)
+  b.send("<iq type='set' id='offline-mam'><query xmlns='urn:xmpp:mam:2' queryid='offline-mam'/></iq>");b.iq('offline-mam')
+  assert not any(n.tag=='{jabber:client}body' and n.text=='transient offline' for e in b.queue for n in e.iter()),'no-permanent-store leaked into MAM' 
+  b.send("<enable xmlns='urn:xmpp:sm:3' resume='true'/>");enabled=b.take(lambda e:e.tag=='{urn:xmpp:sm:3}enabled');resume=enabled.get('id');b.close();clients.remove(b);time.sleep(.1)
+  a.send("<message to='bob@localhost'><body>resume replay</body></message>");a.sync()
+  b=Client(portnum,path/'cert.pem','bob','bob-secret',bind=False);clients.append(b);b.send(f"<resume xmlns='urn:xmpp:sm:3' previd='{resume}' h='0'/>");b.take(lambda e:e.tag=='{urn:xmpp:sm:3}resumed');b.body('resume replay');b.send("<a xmlns='urn:xmpp:sm:3' h='1'/>");b.sync()
+  print('PASS: integrated TCP roster/subscriptions, client IQ routing, CSI buffering/keepalive flush, MAM, blocking, OMEMO PEP, MUC, offline replay and SM resumption')
+ finally:
+  for c in clients:c.close()
+  server.send_signal(signal.SIGTERM)
+  try:server.wait(timeout=5)
+  except subprocess.TimeoutExpired:server.kill();server.wait()
+
+ assert server.returncode==0, f'SIGTERM shutdown failed: {server.returncode}: '+server.stderr.read().decode()
+ print('PASS: graceful SIGTERM shutdown')

scripts/load_smoke.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+"""Isolated Linux multi-account TCP/STARTTLS load smoke; stdlib and openssl only.
+Reports observations, not production capacity guarantees or memory-leak proof.
+"""
+import argparse
+import asyncio
+import base64
+import json
+import os
+import pathlib
+import socket
+import ssl
+import statistics
+import subprocess
+import tempfile
+import time
+import xml.etree.ElementTree as ET
+
+STREAM = "<stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' to='localhost' version='1.0'>"
+
+
+def proc_pid(child_pid):
+    """Resolve this direct child's procfs PID when /proc exposes the host namespace."""
+    status = pathlib.Path('/proc/self/status').read_text()
+    parent = next(int(line.split()[1]) for line in status.splitlines() if line.startswith('Pid:'))
+    children = [p.name for p in pathlib.Path('/proc').iterdir() if p.name.isdigit()]
+    for candidate in children:
+        try:
+            fields = dict(line.split(':', 1) for line in pathlib.Path(f'/proc/{candidate}/status').read_text().splitlines() if ':' in line)
+        except (OSError, ProcessLookupError):
+            continue
+        namespaces = fields.get('NSpid', fields['Pid']).split()
+        if int(namespaces[-1]) == child_pid and int(fields['PPid'].strip()) == parent and fields['Name'].strip() == 'rust-xmpp':
+            return int(candidate)
+    raise RuntimeError('cannot resolve owned server process in procfs')
+
+
+def process_metrics(pid):
+    try:
+        status = pathlib.Path(f'/proc/{pid}/status').read_text()
+        rss = next(int(line.split()[1]) for line in status.splitlines() if line.startswith('VmRSS:'))
+        fields = pathlib.Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()
+        cpu = (int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK')
+        return {'rss_kib': rss, 'cpu_seconds': cpu, 'fd_count': len(list(pathlib.Path(f'/proc/{pid}/fd').iterdir()))}
+    except (OSError, StopIteration):
+        return {'unavailable': True}
+
+
+def distribution(values):
+    if not values:
+        return {}
+    values = sorted(values)
+    return {'samples': len(values), 'median_ms': round(statistics.median(values)*1000, 3),
+            'p95_ms': round(values[min(len(values)-1, int(len(values)*.95))]*1000, 3),
+            'max_ms': round(values[-1]*1000, 3)}
+
+
+class Client:
+    def __init__(self, number, user, reader, writer):
+        self.number, self.user, self.reader, self.writer = number, user, reader, writer
+        self.jid = f"{user}@localhost/load-{number}"
+
+    async def send(self, text):
+        self.writer.write(text.encode())
+        await self.writer.drain()
+
+    async def until(self, end):
+        return await asyncio.wait_for(self.reader.readuntil(end), 60)
+
+    @classmethod
+    async def connect(cls, number, port, context, accounts):
+        reader, writer = await asyncio.wait_for(asyncio.open_connection('127.0.0.1', port), 30)
+        user = "load" if accounts == 1 else f"load-{number % accounts}"
+        client = cls(number, user, reader, writer)
+        try:
+            await client.send(STREAM)
+            assert b'starttls' in await client.until(b'</stream:features>')
+            await client.send("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>")
+            assert b'proceed' in await client.until(b'/>')
+            await writer.start_tls(context, server_hostname='localhost', ssl_handshake_timeout=30)
+            await client.send(STREAM)
+            assert b'PLAIN' in await client.until(b'</stream:features>')
+            auth = base64.b64encode(f'\0{user}\0load-test-password'.encode()).decode()
+            await client.send(f"<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='PLAIN'>{auth}</auth>")
+            assert b'success' in await client.until(b'/>')
+            await client.send(STREAM)
+            assert b'xmpp-bind' in await client.until(b'</stream:features>')
+            await client.send(f"<iq type='set' id='bind'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>load-{number}</resource></bind></iq>")
+            reply = await client.until(b'</iq>')
+            assert client.jid.encode() in reply, reply
+            return client
+        except BaseException:
+            writer.close()
+            raise
+
+    async def ping(self, round_number):
+        start = time.perf_counter()
+        await self.send(f"<iq type='get' id='ping-{round_number}' to='localhost'><ping xmlns='urn:xmpp:ping'/></iq>")
+        reply = ET.fromstring(await self.until(b'</iq>'))
+        assert reply.get('type') == 'result' and reply.get('id') == f'ping-{round_number}'
+        return time.perf_counter()-start
+
+    async def close(self):
+        try:
+            await self.send('</stream:stream>')
+        except (OSError, ssl.SSLError):
+            pass
+        self.writer.close()
+        try:
+            await asyncio.wait_for(self.writer.wait_closed(), 10)
+        except (OSError, ssl.SSLError, TimeoutError):
+            pass
+
+
+async def run_load(count, accounts, concurrency, rounds, message_rounds, port, context, pid):
+    clients, failures, setup_times, samples = [], [], [], []
+    sampling = True
+
+    async def sample():
+        while sampling:
+            samples.append(process_metrics(pid))
+            await asyncio.sleep(.1)
+
+    sampler = asyncio.create_task(sample())
+    gate = asyncio.Semaphore(concurrency)
+    started = time.perf_counter()
+
+    async def connect(number):
+        async with gate:
+            start = time.perf_counter()
+            try:
+                client = await Client.connect(number, port, context, accounts)
+                clients.append(client)
+                setup_times.append(time.perf_counter()-start)
+            except Exception as error:
+                failures.append({'phase': 'connect', 'resource': number, 'error': repr(error)})
+
+    before = process_metrics(pid)
+    try:
+        await asyncio.gather(*(connect(i) for i in range(count)))
+        setup_seconds = time.perf_counter()-started
+        connected_metrics = process_metrics(pid)
+        ping_times = []
+        for round_number in range(rounds):
+            results = await asyncio.gather(*(c.ping(round_number) for c in clients), return_exceptions=True)
+            for client, result in zip(clients, results):
+                if isinstance(result, Exception):
+                    failures.append({'phase': 'ping', 'resource': client.number, 'error': repr(result)})
+                else:
+                    ping_times.append(result)
+        # Explicit full JIDs avoid presence fanout and deliver exactly one message per resource.
+        message_start = time.perf_counter()
+        clients.sort(key=lambda c: c.number)
+        delivered = 0
+        for batch in range(message_rounds):
+            for i, client in enumerate(clients):
+                target = clients[(i+1) % len(clients)]
+                await client.send(f"<message to='{target.jid}' type='chat' id='m-{batch}-{client.number}'><body>load-smoke-{client.number}</body></message>")
+
+            async def receive_message(index, client):
+                reply = ET.fromstring(await client.until(b'</message>'))
+                expected = clients[(index-1) % len(clients)]
+                assert reply.get('id') == f'm-{batch}-{expected.number}', ET.tostring(reply)
+                assert reply.get('from') == expected.jid, ET.tostring(reply)
+                assert reply.find('{jabber:client}body').text == f'load-smoke-{expected.number}'
+
+            results = await asyncio.gather(*(receive_message(i, c) for i, c in enumerate(clients)), return_exceptions=True)
+            for client, result in zip(clients, results):
+                if isinstance(result, Exception):
+                    failures.append({'phase': 'message', 'batch': batch, 'resource': client.number, 'error': repr(result)})
+                else:
+                    delivered += 1
+        message_seconds = time.perf_counter()-message_start
+        steady = process_metrics(pid)
+        await asyncio.gather(*(c.close() for c in clients))
+        await asyncio.sleep(2)
+        after = process_metrics(pid)
+        return {'total_load_wall_seconds': round(time.perf_counter()-started, 3), 'requested_resources': count, 'authenticated_bound_resources': len(clients),
+                'connection_setup_concurrency': concurrency, 'setup_wall_seconds': round(setup_seconds, 3),
+                'setup_latency': distribution(setup_times), 'ping_rounds': rounds, 'ping_round_trip': distribution(ping_times),
+                'accounts': accounts, 'message_rounds': message_rounds, 'messages_sent': len(clients)*message_rounds, 'messages_received': delivered, 'message_batch_seconds': round(message_seconds, 3),
+                'metrics_before': before, 'metrics_connected': connected_metrics, 'metrics_after_workload': steady,
+                'metrics_two_seconds_after_disconnect': after,
+                'sampled_peak_rss_kib': max((s['rss_kib'] for s in samples if 'rss_kib' in s), default=None),
+                'fd_delta_after_disconnect': after['fd_count']-before['fd_count'] if 'fd_count' in after and 'fd_count' in before else None,
+                'failures': failures,
+                'scope': f'One loopback host, {accounts} accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.'}
+    finally:
+        sampling = False
+        await sampler
+        await asyncio.gather(*(c.close() for c in clients), return_exceptions=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--binary', default=str(pathlib.Path(__file__).resolve().parents[1] / 'target/release/rust-xmpp'))
+    parser.add_argument('--clients', type=int, default=500)
+    parser.add_argument('--archive', action='store_true', help='enable persistent MAM and verify both account archives')
+    parser.add_argument('--accounts', type=int, default=1)
+    parser.add_argument('--message-rounds', type=int, default=1)
+    parser.add_argument('--concurrency', type=int, default=50)
+    parser.add_argument('--rounds', type=int, default=3)
+    args = parser.parse_args()
+    assert 1 <= args.clients <= 4000 and args.concurrency > 0 and args.rounds > 0 and 1 <= args.accounts <= args.clients and args.message_rounds > 0
+    with tempfile.TemporaryDirectory(prefix='rust-xmpp-load-smoke-') as directory:
+        work = pathlib.Path(directory)
+        cert, key = work/'cert.pem', work/'key.pem'
+        subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256',
+                        '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost',
+                        '-addext', 'basicConstraints=critical,CA:FALSE', '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True)
+        with socket.socket() as probe:
+            probe.bind(('127.0.0.1', 0))
+            port = probe.getsockname()[1]
+        config = {'global': {'data_path': str(work/'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [port],
+                            's2s_ports': [], 'http_ports': [], 'https_ports': [], 'allow_registration': False,
+                            'modules_enabled': ['tls', 'saslauth', 'disco', 'ping'],
+                            'ssl': {'certificate': str(cert), 'key': str(key)}},
+                  'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'password_hash': 'SHA-256'}}}}
+        if args.archive:
+            config['global']['modules_enabled'].append('mam')
+            config['global']['default_archive_policy'] = True
+            config['global']['archive_expires_after'] = 'never'
+        path = work/'config.json'
+        path.write_text(json.dumps(config))
+        for account in range(args.accounts):
+            user = 'load' if args.accounts == 1 else f'load-{account}'
+            subprocess.run([args.binary, 'create-user', str(path), user, '--password-stdin'], input='load-test-password\n', text=True, capture_output=True, check=True)
+        with open(work/'server.log', 'w+') as log:
+            process = subprocess.Popen([args.binary, 'serve', str(path), '--experimental'], stdout=log, stderr=log)
+            try:
+                for _ in range(100):
+                    if process.poll() is not None:
+                        raise RuntimeError('server exited during startup')
+                    try:
+                        with socket.create_connection(('127.0.0.1', port), timeout=.1):
+                            break
+                    except OSError:
+                        time.sleep(.05)
+                else:
+                    raise RuntimeError('startup timed out')
+                time.sleep(.2)
+                context = ssl.create_default_context(cafile=str(cert))
+                report = asyncio.run(run_load(args.clients, args.accounts, args.concurrency, args.rounds, args.message_rounds, port, context, proc_pid(process.pid)))
+                if args.archive:
+                    archives = list((work/'data'/'localhost'/'archive').glob('*.list'))
+                    records = sum(sum(1 for line in archive.open() if line.startswith('item(')) for archive in archives)
+                    expected = report['messages_received'] * (1 if args.accounts == 1 else 2)
+                    report['mam'] = {'archive_files': len(archives), 'records': records, 'expected_records': expected}
+                    assert records == expected, report['mam']
+                print(json.dumps(report, indent=2))
+                if report['failures'] or report['authenticated_bound_resources'] != args.clients:
+                    raise SystemExit(1)
+            except BaseException:
+                log.flush()
+                log.seek(0)
+                print('SERVER LOG:\n'+log.read())
+                raise
+            finally:
+                process.terminate()
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                    process.wait()
+
+
+if __name__ == '__main__':
+    main()

scripts/make-demo.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+mkdir -p demo
+if [ -e demo/config.toml ]; then
+  echo 'demo/config.toml already exists; refusing to replace demo' >&2
+  exit 1
+fi
+umask 077
+openssl req -x509 -newkey rsa:2048 -nodes -days 30 \
+  -keyout demo/key.pem -out demo/cert.pem \
+  -subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost,IP:127.0.0.1' 2>/dev/null
+python3 - <<'PY'
+from pathlib import Path
+import json
+p=Path('demo').resolve()
+config={'global':{'data_path':str(p/'data'),'interfaces':['127.0.0.1'],'c2s_ports':[55222],
+'s2s_ports':[],'http_ports':[],'https_ports':[],'allow_registration':False,
+'modules_enabled':['tls','saslauth','disco','ping','version']},
+'hosts':{'localhost':{'options':{'authentication':'internal_hashed','password_hash':'SHA-256',
+'ssl':{'key':str(p/'key.pem'),'certificate':str(p/'cert.pem')}}}}}
+(p/'config.json').write_text(json.dumps(config,indent=2))
+PY
+./target/release/rust-xmpp convert demo/config.json toml demo/config.toml
+printf '%s\n' 'alice-test-only' | ./target/release/rust-xmpp create-user demo/config.toml alice --password-stdin
+printf '%s\n' 'bob-test-only' | ./target/release/rust-xmpp create-user demo/config.toml bob --password-stdin
+cat <<'TEXT'
+Demo created. Local test credentials:
+  alice@localhost / alice-test-only
+  bob@localhost   / bob-test-only
+Start: ./target/release/rust-xmpp serve demo/config.toml --experimental
+Manual clients: server localhost, port 55222, STARTTLS; trust demo/cert.pem.
+Automated test: python3 scripts/smoke.py (starts and stops its own server).
+TEXT

scripts/native-federation.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Native transport interop: built Prosody cafd74b9c5ea + Lua 5.4/TLS dependencies.
+set -euo pipefail
+: "${PROSODY_SOURCE:?Set PROSODY_SOURCE to the built Prosody cafd74b9c5ea source tree}"
+export PROSODY_SOURCE
+export PROSODY_LUA="${PROSODY_LUA:-lua5.4}"
+# Optional Debian multiarch dependency bundle, e.g. /tmp/xmpp-native/usr.
+if [[ -n "${PROSODY_NATIVE_PREFIX:-}" ]]; then
+    export LD_LIBRARY_PATH="$PROSODY_NATIVE_PREFIX/lib/x86_64-linux-gnu${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+    export LUA_PATH="$PROSODY_NATIVE_PREFIX/share/lua/5.4/?.lua;$PROSODY_NATIVE_PREFIX/share/lua/5.4/?/init.lua;${LUA_PATH:-;}"
+    export LUA_CPATH="$PROSODY_NATIVE_PREFIX/lib/x86_64-linux-gnu/lua/5.4/?.so;${LUA_CPATH:-;}"
+fi
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+exec "${CARGO:-cargo}" test --locked --test federation_tls native_prosody_tls_external_bidirectional -- --ignored --nocapture

scripts/prosody_compat/README.md

@@ -0,0 +1,66 @@
+# Exact Prosody compatibility oracle
+
+This harness executes the supplied **Prosody cafd74b9c5ea** implementation with a real Lua 5.4 interpreter and its compiled native modules. It uses the original `startup.make_dummy_hosts` (the upstream prosodyctl bootstrap), `core.modulemanager`, `mod_storage_internal`, and `mod_auth_internal_hashed`. Storage, authentication, stanza serialization and binary archive indexing are not replaced with test doubles. It does not launch a network listener or access a production data directory.
+
+The storage/auth ignored integration test creates a new temporary directory for each of SHA-1 and SHA-256. It verifies:
+
+1. Rust creates mixed-key/binary key/value data, preferences, accounts and archives.
+2. Original Prosody loads those records, verifies passwords, rotates passwords, creates another account with a binary salt, reads XML namespace attributes, appends and replaces archive entries, and builds a real `.lidx` binary index.
+3. Rust reads the modified records and verifies the new passwords, appends another record without rewriting history, and invalidates the upstream index.
+4. Original Prosody reads the final records again and authenticates the password changed by Rust.
+
+This caught and fixed real namespace interoperability problems: canonical Prosody attributes use the `\1` namespace separator, and reserved XML attributes are represented as literal `xml:lang`, etc. Reading also accepts the historical `|` separator.
+
+## Reproduce on Debian/Ubuntu
+
+Install build dependencies using your normal package manager: `lua5.4`, `liblua5.4-dev`, `lua-filesystem`, `lua-expat`, `lua-socket`, `lua-sec`, `libssl-dev`, `libicu-dev`, a C compiler, and `make`.
+
+Extract the supplied source archive into a new test-only directory, then build:
+
+```bash
+cd prosody-cafd74b9c5ea
+./configure --ostype=debian --lua-version=5.4 --lua-suffix=5.4
+make -j2
+```
+
+From the Rust project:
+
+```bash
+scripts/prosody_compat/verify.sh /absolute/path/to/prosody-cafd74b9c5ea lua5.4
+```
+
+`CARGO` can select a cargo executable. `RUSTUP_TOOLCHAIN` defaults to the project toolchain `1.99.1`. If Lua dependencies are extracted to a private prefix instead of installed, set `LUA_PATH` and `LUA_CPATH` to that prefix before invoking the script.
+
+## Validation performed in this workspace
+
+The source was built with GCC against Lua 5.4.6, OpenSSL 3, and ICU 74. Real Lua modules came from Ubuntu packages extracted into a private prefix. The oracle completed all four native phases for SHA-1 and SHA-256 successfully. This verifies the tested persistence and password interoperability, **not full Prosody protocol parity**.
+
+## Additional native module oracles
+
+The same verification script also runs additional ignored tests against unmodified native modules:
+
+- `preauth_native`: actual mod_invites consumes a Rust token and creates a native token consumed by Rust; scoped tokens remain inaccessible to native code.
+- `pubsub_native`: actual `mod_pep`/`util.pubsub` read Rust items, publish a replacement, and Rust reads it.
+- `muc_native`: actual `mod_muc` restores a Rust room/affiliations; `mod_muc_mam` reads Rust `message<groupchat` records and archives a real native room broadcast; Rust reads the result.
+- `push` (ignored native lookup): original `util.stanza` confirms body extraction excludes foreign namespaces and nested extension elements.
+- `csi_native`: fifteen importance decisions are compared with the actual `mod_csi_simple`, including presence, chatstates, carbons, EME, Jingle, MUC invites and a configured custom payload.
+
+These use actual Prosody modules and storage. They do not emulate a networked fleet of real clients or external servers.
+
+## Native federation transport test
+
+The optional harness launches the built original Prosody in a temporary directory
+and exchanges real stanzas with Rust over two TCP STARTTLS + SASL EXTERNAL
+connections. A generated test CA anchors both certificate chains. Identity and
+chain checks remain enabled. Only the test TCP routing resolver is overridden;
+the asserted XMPP domain is unchanged.
+
+```bash
+PROSODY_SOURCE=/absolute/path/to/built/prosody-cafd74b9c5ea \
+PROSODY_LUA=lua5.4 scripts/native-federation.sh
+```
+
+For dependencies extracted to a private prefix, set `PROSODY_NATIVE_PREFIX` to
+its `usr` directory. The test checks the expected archive revision before
+launching. It is isolated from live configuration/data and validates both message
+IDs and bodies, plus Prosody's TLS/EXTERNAL authentication status.

scripts/prosody_compat/oracle.lua

@@ -0,0 +1,77 @@
+-- Runs the supplied Prosody's REAL bootstrap/module manager/storage/auth implementations.
+local source, root, phase = assert(arg[1]), assert(arg[2]), assert(arg[3]);
+package.path = source .. '/?.lua;' .. package.path;
+package.cpath = source .. '/?.so;' .. package.cpath;
+dofile(source .. '/loader.lua');
+CFG_SOURCEDIR, CFG_PLUGINDIR, CFG_DATADIR = source, source .. '/plugins', root;
+local startup = require 'prosody.util.startup';
+startup.init_global_state();
+local config = require 'prosody.core.configmanager';
+config.set('*', 'data_path', root);
+config.set('*', 'network_backend', 'select');
+config.set('example.org', 'authentication', 'internal_hashed');
+config.set('example.org', 'password_hash', arg[4] or 'SHA-256');
+startup.setup_datadir();
+startup.load_libraries();
+startup.make_dummy_hosts(); -- The original prosodyctl bootstrap, not a test replacement.
+local sm = require 'prosody.core.storagemanager';
+sm.initialize_host('example.org');
+local kv = assert(sm.open('example.org', 'oracle'));
+local archive = assert(sm.open('example.org', 'archive', 'archive'));
+local st = require 'prosody.util.stanza';
+local mm = require 'prosody.core.modulemanager';
+assert(mm.load('example.org', 'auth_internal_hashed'));
+local authmod = assert(mm.get_module('example.org', 'auth_internal_hashed'));
+local auth = assert(mm.get_items('auth-provider', 'example.org')[1]);
+if phase == 'probe' then
+  assert(kv:set('probe', {hello='actual Prosody'}));
+  assert(kv:get('probe').hello == 'actual Prosody');
+  print('PROSODY_NATIVE_PROBE_OK');
+elseif phase == 'roundtrip' then
+  local value = assert(kv:get('alice'));
+  assert(value[false] == 'roster');
+  assert(value[3] == 'sparse');
+  assert(value.binary == string.char(0, 1, 10, 255));
+  assert(value.text == 'Привет & <xml>');
+  value.native = {changed=true, number=1.25};
+  assert(kv:set('alice', value));
+  local prefs = assert(sm.open('example.org', 'archive_prefs'));
+  assert(prefs:get('alice')[false] == 'roster');
+  assert(prefs:set('alice', {[false]=false, ['bob@example.org']=true}));
+  assert(auth.test_password('alice', 'пароль-Original'));
+  assert(not auth.test_password('alice', 'wrong'));
+  assert(auth.set_password('alice', 'пароль-Native'));
+  assert(auth.create_user('bob', 'native-bob'));
+  local ok, stored, server = require('prosody.util.sasl.scram').hashers[arg[4] or 'SHA-256']('binary-password', string.char(0,255)..'salt', 10000);
+  assert(ok);
+  local hex = require 'prosody.util.hex';
+  assert(sm.open('example.org', 'accounts'):set('binary', {salt=string.char(0,255)..'salt', iteration_count=10000,stored_key=hex.encode(stored),server_key=hex.encode(server)}));
+  assert(auth.test_password('binary', 'binary-password'));
+  local found = {};
+  for id, stanza, when, peer in assert(archive:find('alice')) do
+    found[#found+1] = id;
+    assert(stanza.name == 'message');
+    assert(stanza.attr['xml:lang'] == 'ru');
+    assert(stanza.attr['urn:oracle'..string.char(1)..'flag'] == 'yes');
+    assert(stanza:get_child_text('body') == 'Rust body');
+    assert(when == 1767225600 + #found - 1);
+    assert(peer == 'bob@example.org');
+    assert(stanza.attr.stamp == nil);
+  end
+  assert(#found == 2 and found[1]=='rust-one' and found[2]=='rust-two');
+  local native = st.message({xmlns='jabber:client',from='bob@example.org',to='alice@example.org',['xml:lang']='ru', ['urn:oracle'..string.char(1)..'flag']='native'}, 'Native body');
+  assert(archive:append('alice', 'native-three', native, 1767225602, 'bob@example.org'));
+  assert(archive:append('alice', 'rust-two', st.message({xmlns='jabber:client'}, 'Replaced by native'), 1767225603, 'bob@example.org'));
+  local iter = assert(archive:find('alice', {after='rust-one', with='bob@example.org'}));
+  assert(iter() == 'native-three');assert(iter() == 'rust-two');assert(iter() == nil);
+  -- This exercises original binary .lidx creation, not just .list loading.
+  print('PROSODY_NATIVE_ROUNDTRIP_OK '..(arg[4] or 'SHA-256'));
+elseif phase == 'verify-final' then
+  assert(kv:get('alice').rust_final == true);
+  assert(auth.test_password('bob', 'rust-bob-new'));
+  local count=0;for id, stanza in assert(archive:find('alice')) do count=count+1;if count==4 then assert(id=='rust-final');assert(stanza:get_child_text('body')=='Final');end end
+  assert(count==4);
+  print('PROSODY_NATIVE_FINAL_OK '..(arg[4] or 'SHA-256'));
+else
+  error('unknown phase');
+end

scripts/prosody_compat/verify.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+if [[ $# -lt 1 || $# -gt 2 ]]; then
+  echo 'Usage: verify.sh /absolute/path/to/built/prosody-cafd74b9c5ea [lua5.4]' >&2
+  exit 2
+fi
+project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)
+reference_dir=$(cd -- "$1" && pwd)
+if ! grep -q '^node: cafd74b9c5ea' "$reference_dir/.hg_archival.txt"; then
+  echo 'Expected the supplied Prosody cafd74b9c5ea source revision.' >&2
+  exit 2
+fi
+for required in util/encodings.so util/hashes.so util/pposix.so plugins/mod_storage_internal.lua; do
+  test -f "$reference_dir/$required" || { echo "Missing $required; build Prosody first." >&2; exit 2; }
+done
+export PROSODY_REFERENCE_SOURCE="$reference_dir"
+export PROSODY_REFERENCE_LUA="${2:-lua5.4}"
+export RUSTUP_TOOLCHAIN="${RUSTUP_TOOLCHAIN:-1.99.1}"
+cd -- "$project_dir"
+"${CARGO:-cargo}" test --locked --test reference_prosody --test pubsub_native --test pep_item_order --test muc_native --test muc_state_native --test csi_native --test push --test preauth_native -- --ignored --nocapture

scripts/smoke.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Real TCP/TLS integration test; stdlib only; requires make-demo.sh first."""
+import base64
+import hashlib
+import hmac
+import re
+import signal
+import socket
+import ssl
+import subprocess
+import time
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+ROOT=Path(__file__).resolve().parents[1]
+STREAM="<stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' to='localhost' version='1.0'>"
+class Client:
+    def __init__(self):
+        self.sock=socket.create_connection(('127.0.0.1',55222),timeout=5)
+        self.buf=b''
+        self.send(STREAM)
+        assert b'<required' in self.until(b'</stream:features>')
+        self.send("<starttls xmlns='urn:ietf:params:xml:ns:xmpp-tls'/>")
+        assert b'proceed' in self.until(b'/>')
+        ctx=ssl.create_default_context(cafile=str(ROOT/'demo/cert.pem'))
+        self.sock=ctx.wrap_socket(self.sock,server_hostname='localhost')
+        self.send(STREAM)
+        assert b'PLAIN' in self.until(b'</stream:features>')
+    def send(self,s): self.sock.sendall(s.encode())
+    def until(self,mark):
+        while mark not in self.buf:
+            chunk=self.sock.recv(65536)
+            if not chunk: raise RuntimeError('unexpected EOF')
+            self.buf+=chunk
+        end=self.buf.index(mark)+len(mark)
+        out,self.buf=self.buf[:end],self.buf[end:]
+        return out
+    def auth(self,user,password,fail=False):
+        encoded=base64.b64encode(('\0'+user+'\0'+password).encode()).decode()
+        self.send(f"<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='PLAIN'>{encoded}</auth>")
+        if fail:
+            assert b'not-authorized' in self.until(b'</failure>');return
+        assert b'success' in self.until(b'/>')
+        self.send(STREAM)
+        assert b'xmpp-bind' in self.until(b'</stream:features>')
+        self.send("<iq type='set' id='bind'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'><resource>smoke</resource></bind></iq>")
+        assert f'{user}@localhost/smoke'.encode() in self.until(b'</iq>')
+    def close(self): self.sock.close()
+
+def account_hash_crosscheck():
+    record=(ROOT/'demo/data/localhost/accounts/alice.dat').read_text()
+    def value(k): return re.search(rf'{k}\s*=\s*"([^"]+)"',record).group(1)
+    iterations=int(re.search(r'iteration_count\s*=\s*(\d+)',record).group(1))
+    salted=hashlib.pbkdf2_hmac('sha256',b'alice-test-only',value('salt').encode(),iterations)
+    stored=hashlib.sha256(hmac.new(salted,b'Client Key','sha256').digest()).hexdigest()
+    server=hmac.new(salted,b'Server Key','sha256').hexdigest()
+    assert stored==value('stored_key') and server==value('server_key')
+
+p=subprocess.Popen([str(ROOT/'target/release/rust-xmpp'),'serve',str(ROOT/'demo/config.toml'),'--experimental'],cwd=ROOT,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+clients=[]
+try:
+    for _ in range(100):
+        if p.poll() is not None: raise RuntimeError(p.stderr.read().decode())
+        try:
+            probe=socket.create_connection(('127.0.0.1',55222),timeout=.1);probe.close();break
+        except OSError: time.sleep(.05)
+    else: raise RuntimeError('server did not start')
+    account_hash_crosscheck()
+    a=Client();clients.append(a)
+    a.auth('alice','wrong',fail=True);a.auth('alice','alice-test-only')
+    b=Client();clients.append(b);b.auth('bob','bob-test-only')
+    a.send("<presence/>");b.send("<presence/>")
+    b.send("<iq type='get' id='ready'><ping xmlns='urn:xmpp:ping'/></iq>");b.until(b'</iq>')
+    a.send("<iq type='get' id='p1'><ping xmlns='urn:xmpp:ping'/></iq>")
+    assert b"type='result'" in a.until(b'</iq>')
+    a.send("<message type='chat' id='m1' from='admin@localhost' to='bob@localhost'><body>Hello &amp; привет</body></message>")
+    xml=b.until(b'</message>');m=ET.fromstring(xml)
+    assert m.attrib['from']=='alice@localhost/smoke'
+    assert m.find('{jabber:client}body').text=='Hello & привет'
+    b.send("<message type='chat' id='m2' to='alice@localhost/smoke'><body>Reply</body></message>")
+    assert ET.fromstring(a.until(b'</message>')).find('{jabber:client}body').text=='Reply'
+    # Forwarding keeps extension namespace declarations from the stanza root.
+    a.send("<message xmlns:ext='urn:test:extension' xml:lang='ru' to='bob@localhost'><ext:payload>data</ext:payload></message>")
+    extension=ET.fromstring(b.until(b'</message>'))
+    assert extension.find('{urn:test:extension}payload').text=='data'
+    assert extension.attrib['{http://www.w3.org/XML/1998/namespace}lang']=='ru'
+    a.send("<message id='offline' to='missing@localhost'><body>test</body></message>")
+    assert b'service-unavailable' in a.until(b'</message>')
+    a.send("<iq type='get' id='mam'><query xmlns='urn:xmpp:mam:2'/></iq>")
+    assert b'service-unavailable' in a.until(b'</iq>')
+    # Passwords must never be accepted on a plaintext stream.
+    raw=socket.create_connection(('127.0.0.1',55222),timeout=3)
+    raw.sendall(STREAM.encode());raw.recv(65536)
+    raw.sendall(b"<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='PLAIN'>eA==</auth>")
+    while True:
+        chunk=raw.recv(65536)
+        assert b'<success' not in chunk
+        if not chunk: break
+    raw.close()
+    print('PASS: TLS certificate validation, login, wrong-password rejection, independent account hash check, binding, ping, bidirectional chat, sender rewriting, offline error, unsupported MAM error, plaintext auth rejection')
+finally:
+    for c in clients:c.close()
+    p.send_signal(signal.SIGINT)
+    try:p.wait(timeout=5)
+    except subprocess.TimeoutExpired:p.kill();p.wait()

scripts/verify-local.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# All runtime checks below create isolated data and use loopback listeners.
+set -euo pipefail
+project_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
+cd -- "$project_dir"
+cargo fmt --check
+cargo test --locked
+cargo clippy --locked --all-targets -- -D warnings
+cargo build --release --locked
+python3 scripts/integration_xmpp.py
+python3 tests/sasl_network.py --spawn --hash sha1
+python3 tests/sasl_network.py --spawn --hash sha256
+python3 tests/android_sasl_network.py target/release/rust-xmpp
+python3 tests/logging_network.py target/release/rust-xmpp
+python3 tests/sasl_retry_network.py target/release/rust-xmpp
+python3 tests/sasl_web_network.py --binary target/release/rust-xmpp
+python3 tests/wss_interop.py --binary target/release/rust-xmpp
+python3 tests/http_proxy_paths_network.py --binary target/release/rust-xmpp
+python3 tests/http_altconnect_network.py --binary target/release/rust-xmpp
+python3 tests/sasl2_bind.py target/release/rust-xmpp
+python3 tests/sasl2_cached_inline.py target/release/rust-xmpp
+python3 tests/gajim_discovery_network.py target/release/rust-xmpp
+python3 tests/sasl2_resume.py target/release/rust-xmpp
+python3 tests/push_discovery.py target/release/rust-xmpp
+python3 tests/pep_client_network.py target/release/rust-xmpp
+python3 tests/pep_order_network.py target/release/rust-xmpp
+python3 tests/pep_notifications_network.py target/release/rust-xmpp
+python3 tests/preauth_network.py target/release/rust-xmpp
+python3 tests/muc_restart_network.py target/release/rust-xmpp
+python3 tests/resource_rebind.py target/release/rust-xmpp
+python3 scripts/integration_http.py
+python3 scripts/integration_http.py --custom-upload-path
+if [[ ${XMPP_LOAD_CHECK:-0} == 1 ]]; then
+  python3 scripts/load_smoke.py --clients 500 --accounts 100 --rounds 10 --message-rounds 10 --archive
+fi
+if [[ -n ${PROSODY_REFERENCE_SOURCE:-} ]]; then
+  RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN:-1.99.1} scripts/prosody_compat/verify.sh \
+    "$PROSODY_REFERENCE_SOURCE" "${PROSODY_REFERENCE_LUA:-lua5.4}"
+  PROSODY_SOURCE="$PROSODY_REFERENCE_SOURCE" PROSODY_LUA="${PROSODY_REFERENCE_LUA:-lua5.4}" scripts/native-federation.sh
+fi