- Offline CLI `upload-migrate OLD_CONFIG NEW_CONFIG --component HOST
[--execute]`: dry run по умолчанию, local ↔ S3 и S3 ↔ S3 без удаления source. - Последовательное копирование с проверкой размера и полным SHA-256 readback destination; конфликтующий объект не перезаписывается. Временный диск — максимум один переносимый файл дополнительно к постоянному destination. - Durable migration journal позволяет продолжить прерванную операцию с теми же конфигурациями. Незавершённый или повреждённый journal блокирует startup; backend identity меняется только после проверки всех файлов. - Native `uploads`/`upload_stats`, slots и JWT сохраняются; поддерживаются native локальные файлы без Rust completion sidecar. Миграция требует остановки RustXMPP и Prosody, использующих общий `data_path`. - Multipart intent сохраняется до создания удалённого upload. Recovery находит потерянный UploadId через paginated listing и очищает только точный key принадлежащего slot. При сохранных metadata cleanup не зависит от lifecycle; lifecycle остаётся дополнительной защитой, а не гарантией. - Перед completion сохраняется durable marker. Неопределённый результат completion при отсутствии подтверждённого объекта помещает только этот slot в quarantine: PUT 409, retention не удаляет его, maintenance повторяет сверку. Startup продолжается; сохраняется single-use семантика. - Добавлены network regressions миграции и расширены проверки multipart recovery, потерянных ответов, отмены запросов и конфликтов destination. Настройка, команды миграции, recovery и ограничения отката: [docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). Команды, результаты и границы подтверждения, включая qualification реального object store и 10 GiB: [docs/VALIDATION-0.7.9.md](docs/VALIDATION-0.7.9.md). Итог: 265 тестов, нагрузка 500 TLS клиентов и native reference — успешно. Реальные 10 GiB через MinIO (описанный test adapter), 1280parts и полный SHA-256 пройдены; подробности — [VALIDATION-0.7.9](docs/VALIDATION-0.7.9.md).
This commit is contained in:
wt committed
1 parent
c260d7bab8
commit
b30fc524d8
41 files changed
+5936
-152
No files matched your search
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Opt-in real XEP-0363 HTTPS large-file regression (default: 10 GiB).
|
||||
|
||||
Example: python3 tests/upload_large_network.py --binary /tmp/rust-xmpp
|
||||
Writes the actual complete payload through TLS to the server's local file
|
||||
stream storage, then reads every byte back through verified TLS. No source
|
||||
file, sparse allocation, external server, or body-sized Python buffer is used.
|
||||
Not part of the default local check because it needs >10 GiB free disk.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from _wire_client import Client, BIN
|
||||
|
||||
NS = 'urn:xmpp:http:upload:0'
|
||||
CHUNK = bytes(range(256)) * 4096 # one deterministic 1 MiB buffer
|
||||
|
||||
|
||||
def free_port():
|
||||
with socket.socket() as probe:
|
||||
probe.bind(('127.0.0.1', 0))
|
||||
return probe.getsockname()[1]
|
||||
|
||||
|
||||
def proc_status_for_child(pid):
|
||||
# /proc can expose host PIDs while Popen.pid is in a nested PID namespace.
|
||||
parent = str(Path('/proc/self').resolve().name)
|
||||
for entry in Path('/proc').iterdir():
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
fields = dict(line.split(':', 1) for line in (entry/'status').read_text().splitlines() if ':' in line)
|
||||
namespace_pids = fields.get('NSpid', fields.get('Pid', '')).split()
|
||||
if fields.get('PPid', '').strip() == parent and namespace_pids[-1:] == [str(pid)]:
|
||||
return entry/'status'
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def rss(status_path):
|
||||
if status_path is None:
|
||||
return 0
|
||||
try:
|
||||
for line in status_path.read_text().splitlines():
|
||||
if line.startswith('VmRSS:'):
|
||||
return int(line.split()[1]) * 1024
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def progress(phase, count, size, start, last):
|
||||
now = time.monotonic()
|
||||
if now - last >= 5 or count == size:
|
||||
print(f'{phase}: {count}/{size} bytes ({count / max(now-start, .001) / 2**20:.1f} MiB/s)', flush=True)
|
||||
return now
|
||||
return last
|
||||
|
||||
|
||||
def run(args, report):
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-large-', dir=args.temp_dir) as tmp:
|
||||
root = Path(tmp)
|
||||
report['free_disk_before_bytes'] = shutil.disk_usage(root).free
|
||||
assert report['free_disk_before_bytes'] >= args.size + 256 * 2**20, 'Insufficient free disk for real upload'
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
|
||||
'-keyout', str(root/'key.pem'), '-out', str(root/'cert.pem'),
|
||||
'-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'],
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
port, https = free_port(), free_port()
|
||||
while https == port:
|
||||
https = free_port()
|
||||
cert = {'key': str(root/'key.pem'), 'certificate': str(root/'cert.pem')}
|
||||
config = {'global': {'data_path': str(root/'data'), 'interfaces': ['127.0.0.1'],
|
||||
'c2s_ports': [port], 'http_ports': [], 'https_ports': [https], 's2s_ports': [],
|
||||
'modules_enabled': ['tls', 'saslauth', 'disco', 'ping'], 'ssl': cert},
|
||||
'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'ssl': cert}},
|
||||
'upload.localhost': {'component': 'http_file_share', 'options': {
|
||||
'http_host': 'localhost', 'http_file_share_backend': 'local', 'http_file_share_base_url': f'https://localhost:{https}/file_share',
|
||||
'http_file_share_size_limit': args.size,
|
||||
'http_file_share_daily_quota': args.size,
|
||||
'http_file_share_global_quota': args.size,
|
||||
'http_file_share_expires_after': 'never'}}}}
|
||||
config_path = root/'config.json'
|
||||
config_path.write_text(json.dumps(config))
|
||||
subprocess.run([str(args.binary), 'create-user', str(config_path), 'alice', '--password-stdin'],
|
||||
input='large-file-secret\n', text=True, check=True, capture_output=True)
|
||||
with (root/'server.log').open('w+') as log:
|
||||
server = subprocess.Popen([str(args.binary), 'serve', str(config_path), '--experimental'], stdout=log, stderr=log)
|
||||
stop = threading.Event()
|
||||
report['max_rss_bytes'] = {'server': 0, 'client': 0}
|
||||
server_status = proc_status_for_child(server.pid)
|
||||
report['rss_proc_status_paths'] = {'server': str(server_status), 'client': '/proc/self/status'}
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
for role, status_path in [('server', server_status), ('client', Path('/proc/self/status'))]:
|
||||
report['max_rss_bytes'][role] = max(report['max_rss_bytes'][role], rss(status_path))
|
||||
stop.wait(.05)
|
||||
monitor = threading.Thread(target=sample, daemon=True)
|
||||
monitor.start()
|
||||
client = None
|
||||
try:
|
||||
assert server_status is not None, 'cannot resolve actual server /proc status for RSS sampling'
|
||||
for _ in range(200):
|
||||
assert server.poll() is None, 'server exited during startup'
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', port), .1):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(.05)
|
||||
else:
|
||||
raise AssertionError('server startup timed out')
|
||||
client = Client(port, root/'cert.pem', 'alice', 'large-file-secret')
|
||||
client.send(f"<iq type='get' to='upload.localhost' id='large-slot'><request xmlns='{NS}' filename='large.bin' size='{args.size}' content-type='application/octet-stream'/></iq>")
|
||||
slot = client.iq('large-slot').find(f'{{{NS}}}slot')
|
||||
assert slot is not None
|
||||
put = slot.find(f'{{{NS}}}put')
|
||||
get = slot.find(f'{{{NS}}}get')
|
||||
put_url, get_url = urlsplit(put.get('url')), urlsplit(get.get('url'))
|
||||
assert put_url.scheme == get_url.scheme == 'https'
|
||||
assert put_url.hostname == get_url.hostname == 'localhost'
|
||||
headers = {h.get('name'): h.text for h in put.findall(f'{{{NS}}}header')}
|
||||
context = ssl.create_default_context(cafile=str(root/'cert.pem'))
|
||||
def connect(url):
|
||||
return http.client.HTTPSConnection(url.hostname, url.port, context=context, timeout=120)
|
||||
def path(url):
|
||||
return url.path + ('?' + url.query if url.query else '')
|
||||
start = last = time.monotonic()
|
||||
digest = hashlib.sha256()
|
||||
sent = 0
|
||||
conn = connect(put_url)
|
||||
try:
|
||||
conn.putrequest('PUT', path(put_url))
|
||||
for name, value in {**headers, 'Content-Length': str(args.size), 'Content-Type': 'application/octet-stream'}.items():
|
||||
conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
while sent < args.size:
|
||||
piece = memoryview(CHUNK)[:min(len(CHUNK), args.size - sent)]
|
||||
conn.send(piece)
|
||||
digest.update(piece)
|
||||
sent += len(piece)
|
||||
last = progress('PUT', sent, args.size, start, last)
|
||||
response = conn.getresponse()
|
||||
assert response.status == 201, (response.status, response.read(4096))
|
||||
response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
report.update(put_bytes=sent, put_seconds=time.monotonic()-start, sha256=digest.hexdigest())
|
||||
# A real complete on-disk file, including allocated blocks, is required.
|
||||
stored = list((root/'data').glob('*/http_file_share/*.bin'))
|
||||
assert len(stored) == 1, stored
|
||||
stat = stored[0].stat()
|
||||
assert stat.st_size == args.size
|
||||
assert stat.st_blocks * 512 >= args.size, 'file unexpectedly sparse'
|
||||
report['stored_bytes'] = stat.st_size
|
||||
report['allocated_bytes'] = stat.st_blocks * 512
|
||||
report['free_disk_after_put_bytes'] = shutil.disk_usage(root).free
|
||||
conn = connect(get_url)
|
||||
try:
|
||||
conn.request('HEAD', path(get_url))
|
||||
response = conn.getresponse()
|
||||
assert response.status == 200 and int(response.getheader('Content-Length')) == args.size
|
||||
assert response.read() == b''
|
||||
finally:
|
||||
conn.close()
|
||||
report['head_status'] = 200
|
||||
start = last = time.monotonic()
|
||||
received, reads = 0, 0
|
||||
downloaded = hashlib.sha256()
|
||||
conn = connect(get_url)
|
||||
try:
|
||||
conn.request('GET', path(get_url))
|
||||
response = conn.getresponse()
|
||||
assert response.status == 200 and int(response.getheader('Content-Length')) == args.size
|
||||
while True:
|
||||
piece = response.read(len(CHUNK))
|
||||
reads += 1
|
||||
if not piece:
|
||||
break
|
||||
downloaded.update(piece)
|
||||
received += len(piece)
|
||||
last = progress('GET', received, args.size, start, last)
|
||||
assert received == args.size and downloaded.digest() == digest.digest()
|
||||
finally:
|
||||
conn.close()
|
||||
report.update(get_bytes=received, get_read_calls=reads, get_seconds=time.monotonic()-start,
|
||||
get_sha256=downloaded.hexdigest())
|
||||
# Exercise a range crossing 4 GiB when the selected size permits it.
|
||||
offset = min(2**32 - 7, args.size - 32)
|
||||
conn = connect(get_url)
|
||||
try:
|
||||
conn.request('GET', path(get_url), headers={'Range': f'bytes={offset}-{offset+31}'})
|
||||
response = conn.getresponse()
|
||||
assert response.status == 206
|
||||
assert response.getheader('Content-Range') == f'bytes {offset}-{offset+31}/{args.size}'
|
||||
assert response.read(33) == bytes((offset+i) % 256 for i in range(32))
|
||||
finally:
|
||||
conn.close()
|
||||
report.update(range_status=206, range_offset=offset)
|
||||
conn = connect(put_url)
|
||||
try:
|
||||
# No second large body: single-use check must reject at headers.
|
||||
conn.putrequest('PUT', path(put_url))
|
||||
for name, value in {**headers, 'Content-Length': str(args.size)}.items():
|
||||
conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
response = conn.getresponse()
|
||||
assert response.status == 409, response.status
|
||||
response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
report['reuse_status'] = 409
|
||||
client.send(f"<iq type='get' to='upload.localhost' id='over-quota'><request xmlns='{NS}' filename='extra.bin' size='1'/></iq>")
|
||||
rejected = client.iq('over-quota', error=True)
|
||||
assert rejected.find('.//{urn:ietf:params:xml:ns:xmpp-stanzas}not-acceptable') is not None, ET.tostring(rejected)
|
||||
report['over_quota_condition'] = 'not-acceptable'
|
||||
assert 0 < report['max_rss_bytes']['client'] < 256 * 2**20, report['max_rss_bytes']
|
||||
assert 0 < report['max_rss_bytes']['server'] < 512 * 2**20, report['max_rss_bytes']
|
||||
report['status'] = 'passed'
|
||||
except BaseException:
|
||||
log.flush()
|
||||
log.seek(0)
|
||||
print(log.read()[-20000:], flush=True)
|
||||
raise
|
||||
finally:
|
||||
if client:
|
||||
client.close()
|
||||
server.terminate()
|
||||
try:
|
||||
server.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
server.kill()
|
||||
server.wait()
|
||||
stop.set()
|
||||
monitor.join(timeout=2)
|
||||
report['temporary_files_cleaned'] = not root.exists()
|
||||
report['free_disk_after_cleanup_bytes'] = shutil.disk_usage(args.temp_dir or tempfile.gettempdir()).free
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--binary', type=Path, default=BIN)
|
||||
parser.add_argument('--size', type=int, default=10 * 2**30, help='actual network bytes; default 10 GiB')
|
||||
parser.add_argument('--report', type=Path, default=Path('/tmp/xmpp078-large.json'))
|
||||
parser.add_argument('--temp-dir', type=Path)
|
||||
args = parser.parse_args()
|
||||
assert args.size >= 32
|
||||
args.binary = args.binary.resolve()
|
||||
report = {'status': 'failed', 'requested_bytes': args.size, 'python_chunk_bytes': len(CHUNK),
|
||||
'transport': 'verified TLS HTTPS', 'storage': 'local streaming file', 'rss_sample_interval_seconds': .05}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
run(args, report)
|
||||
except BaseException as error:
|
||||
report['error'] = repr(error)
|
||||
raise
|
||||
finally:
|
||||
report['total_seconds'] = time.monotonic() - started
|
||||
args.report.write_text(json.dumps(report, indent=2) + '\n')
|
||||
print(json.dumps(report, indent=2), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,263 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline upload-migrate regression using the independent signed S3 fixture.
|
||||
|
||||
Usage: python3 tests/upload_migration_network.py target/release/rust-xmpp
|
||||
Exercises real XMPP slots and HTTP bodies; no cloud service or large disk is needed.
|
||||
The fixture validates SigV4 independently (see upload_s3_network.py).
|
||||
"""
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from _wire_client import Client
|
||||
from upload_s3_network import (ACCESS, PART, REGION, SECRET, TOKEN, Store,
|
||||
free_port, request, slot, start, stop)
|
||||
|
||||
HOST = 'upload.localhost'
|
||||
|
||||
|
||||
class MigrationStore(Store):
|
||||
"""Fail a chosen completion after earlier objects have really been copied."""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.completions = 0
|
||||
self.fail_completion = None
|
||||
|
||||
def verify(self, handler, payload):
|
||||
super().verify(handler, payload)
|
||||
query = parse_qs(urlsplit(handler.path).query, keep_blank_values=True)
|
||||
if handler.command == 'POST' and 'uploadId' in query:
|
||||
self.completions += 1
|
||||
if self.completions == self.fail_completion:
|
||||
self.fail = 'complete-error'
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def checked(command, success=True):
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=90)
|
||||
output = result.stdout + result.stderr
|
||||
assert (result.returncode == 0) == success, (command, result.returncode, output)
|
||||
assert all(value not in output for value in (ACCESS, SECRET, TOKEN)), 'credentials leaked by CLI'
|
||||
return output
|
||||
|
||||
|
||||
def migrate(binary, old, new, execute=False, success=True):
|
||||
command = [binary, 'upload-migrate', str(old), str(new), '--component', HOST]
|
||||
if execute:
|
||||
command.append('--execute')
|
||||
return checked(command, success)
|
||||
|
||||
|
||||
def source_files(directory):
|
||||
return {str(path.relative_to(directory)): sha(path.read_bytes())
|
||||
for path in directory.rglob('*.bin')}
|
||||
|
||||
|
||||
def run(binary):
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-upload-migration-') as directory, \
|
||||
MigrationStore() as first, MigrationStore() as second:
|
||||
root = Path(directory)
|
||||
stores = (first, second)
|
||||
workers = [threading.Thread(target=store.serve_forever, daemon=True) for store in stores]
|
||||
for worker in workers:
|
||||
worker.start()
|
||||
checked(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
|
||||
'-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem'),
|
||||
'-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'])
|
||||
c2s, http_port = free_port(), free_port()
|
||||
while c2s == http_port:
|
||||
http_port = free_port()
|
||||
cert = {'key': str(root / 'key.pem'), 'certificate': str(root / 'cert.pem')}
|
||||
options = {'http_host': '127.0.0.1', 'http_file_share_backend': 'local',
|
||||
'http_file_share_base_url': f'http://127.0.0.1:{http_port}/files',
|
||||
'http_paths': {'file_share': '/files'},
|
||||
'http_file_share_size_limit': PART + 123,
|
||||
'http_file_share_daily_quota': PART * 8,
|
||||
'http_file_share_global_quota': PART * 8,
|
||||
'http_file_share_expires_after': False}
|
||||
base = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'],
|
||||
'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [http_port],
|
||||
'https_ports': [], 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']},
|
||||
'hosts': {'localhost': {'options': {'authentication': 'internal_hashed',
|
||||
'password_hash': 'SHA-256', 'ssl': cert}},
|
||||
HOST: {'component': 'http_file_share', 'options': options}}}
|
||||
configs = []
|
||||
for name, store in [('local', None), ('s3-first', first), ('s3-second', second)]:
|
||||
config = copy.deepcopy(base)
|
||||
if store:
|
||||
component = config['hosts'][HOST]['options']
|
||||
component['http_file_share_backend'] = 's3'
|
||||
component['http_file_share_s3'] = {
|
||||
'endpoint': f'http://127.0.0.1:{store.server_address[1]}/api',
|
||||
'bucket': 'fixture-bucket', 'region': REGION, 'access_key': ACCESS,
|
||||
'secret_key': SECRET, 'session_token': TOKEN, 'key_prefix': 'space +'}
|
||||
path = root / f'{name}.json'
|
||||
path.write_text(json.dumps(config))
|
||||
configs.append(path)
|
||||
local, remote_a, remote_b = configs
|
||||
result = subprocess.run([binary, 'create-user', str(local), 'alice', '--password-stdin'],
|
||||
input='secret\n', text=True, capture_output=True, timeout=30)
|
||||
assert result.returncode == 0, result.stderr
|
||||
component_path = root / 'data' / 'upload%2elocalhost'
|
||||
identity_path = component_path / 'rust_upload_backend.json'
|
||||
metadata_path = component_path / 'rust_uploads.json'
|
||||
archive_path = component_path / 'uploads.list'
|
||||
process = client = None
|
||||
uploads = []
|
||||
with (root / 'server.log').open('w+') as log:
|
||||
try:
|
||||
process = start(binary, local, c2s, log)
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
for index, payload in enumerate((b'A' * PART + b'B' * 123, b'migration\x00hash\xff', b'')):
|
||||
url, headers = slot(client, HOST, len(payload), f'migrate-{index}')
|
||||
assert request(url, 'PUT', payload, headers)[0] == 201
|
||||
status, head, body = request(url, 'HEAD')
|
||||
assert status == 200 and not body
|
||||
head = {key.lower(): value for key, value in head.items()}
|
||||
uploads.append((url, headers, payload, head['etag']))
|
||||
# Both dry-run and execute need the same exclusive data lock.
|
||||
migrate(binary, local, remote_a, success=False)
|
||||
migrate(binary, local, remote_a, execute=True, success=False)
|
||||
assert not first.events, 'live-server lock denial reached S3'
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
original_files = source_files(component_path / 'http_file_share')
|
||||
assert len(original_files) == len(uploads)
|
||||
original_records = json.loads(metadata_path.read_text())
|
||||
original_archive = archive_path.read_bytes()
|
||||
|
||||
def verify_serving(config, preserve_metadata=True):
|
||||
nonlocal process
|
||||
process = start(binary, config, c2s, log)
|
||||
for url, headers, payload, etag in uploads:
|
||||
status, _, data = request(url)
|
||||
assert status == 200 and sha(data) == sha(payload), 'original URL body/hash changed'
|
||||
status, head, body = request(url, 'HEAD')
|
||||
head = {key.lower(): value for key, value in head.items()}
|
||||
assert status == 200 and body == b''
|
||||
assert int(head['content-length']) == len(payload) and head['etag'] == etag
|
||||
assert request(url, headers={'If-None-Match': etag})[0] == 304
|
||||
assert request(url, 'PUT', payload, headers)[0] == 409, 'original JWT became reusable'
|
||||
stop(process); process = None
|
||||
if preserve_metadata:
|
||||
records = json.loads(metadata_path.read_text())
|
||||
stable = lambda rows: {key: {field: value for field, value in record.items() if field != 'xml'}
|
||||
for key, record in rows.items()}
|
||||
assert stable(records) == stable(original_records), 'slot/JWT metadata changed'
|
||||
|
||||
# Invalid destination configuration must not mutate source state.
|
||||
invalid = json.loads(remote_a.read_text())
|
||||
del invalid['hosts'][HOST]['options']['http_file_share_s3']['bucket']
|
||||
bad = root / 'invalid.json'
|
||||
bad.write_text(json.dumps(invalid))
|
||||
identity = identity_path.read_bytes()
|
||||
migrate(binary, local, bad, execute=True, success=False)
|
||||
assert identity_path.read_bytes() == identity and not first.objects
|
||||
assert source_files(component_path / 'http_file_share') == original_files
|
||||
|
||||
# Default is a plan only: no writes, no identity/metadata cutover.
|
||||
migrate(binary, local, remote_a)
|
||||
assert not first.objects and not first.uploads
|
||||
assert not any(stage in ('init', 'part', 'complete', 'abort', 'delete')
|
||||
for stage, _, _ in first.events), 'dry-run wrote to destination'
|
||||
assert identity_path.read_bytes() == identity
|
||||
assert json.loads(metadata_path.read_text()) == original_records
|
||||
|
||||
# One object is durable before a later copy fails. Neither the
|
||||
# failed command nor a server restart may cut over early.
|
||||
first.fail_completion = first.completions + 2
|
||||
migrate(binary, local, remote_a, execute=True, success=False)
|
||||
assert first.objects, 'failure injection did not leave a resumable copied object'
|
||||
assert identity_path.read_bytes() == identity, 'incomplete migration changed active backend'
|
||||
assert json.loads(metadata_path.read_text()) == original_records
|
||||
assert source_files(component_path / 'http_file_share') == original_files
|
||||
# A pending journal fences both configurations until resume.
|
||||
for blocked in (local, remote_a):
|
||||
rejected = subprocess.Popen([binary, 'serve', str(blocked), '--experimental'],
|
||||
stdout=log, stderr=log)
|
||||
try:
|
||||
assert rejected.wait(timeout=15) != 0, 'pending migration started serving'
|
||||
finally:
|
||||
if rejected.poll() is None:
|
||||
rejected.kill(); rejected.wait(timeout=10)
|
||||
first.fail_completion = None
|
||||
completed_before_resume = first.completions
|
||||
migration_cache, migration_archive = metadata_path.read_bytes(), archive_path.read_bytes()
|
||||
migrate(binary, local, remote_a, execute=True)
|
||||
assert metadata_path.read_bytes() == migration_cache and archive_path.read_bytes() == migration_archive
|
||||
assert first.completions - completed_before_resume < len(uploads), 'resume recopied verified objects'
|
||||
assert source_files(component_path / 'http_file_share') == original_files
|
||||
verify_serving(remote_a)
|
||||
assert sorted(sha(data) for data in first.objects.values()) == sorted(sha(row[2]) for row in uploads)
|
||||
|
||||
# S3 -> S3 and S3 -> local preserve remote source objects, even
|
||||
# though final local files already exist from the first backend.
|
||||
first_objects = dict(first.objects)
|
||||
for old, new, destination in ((remote_a, remote_b, second), (remote_b, local, None)):
|
||||
second_objects = dict(second.objects)
|
||||
before = identity_path.read_bytes()
|
||||
destination_objects = dict(destination.objects) if destination else None
|
||||
migrate(binary, old, new)
|
||||
assert identity_path.read_bytes() == before
|
||||
if destination:
|
||||
assert destination.objects == destination_objects
|
||||
migration_cache, migration_archive = metadata_path.read_bytes(), archive_path.read_bytes()
|
||||
migrate(binary, old, new, execute=True)
|
||||
assert metadata_path.read_bytes() == migration_cache and archive_path.read_bytes() == migration_archive
|
||||
verify_serving(new)
|
||||
assert first.objects == first_objects, 'S3 source objects were deleted/changed'
|
||||
if destination is None:
|
||||
assert second.objects == second_objects, 'second S3 source objects were deleted/changed'
|
||||
assert sorted(sha(data) for data in second.objects.values()) == sorted(sha(row[2]) for row in uploads)
|
||||
assert source_files(component_path / 'http_file_share') == original_files
|
||||
# A native final local file is authoritative even when its Rust
|
||||
# cache completion flag is false. Unfinished multipart intent
|
||||
# still rejects the operation without modifying the backend.
|
||||
native = copy.deepcopy(original_records)
|
||||
native_id = next(iter(native))
|
||||
native[native_id]['multipart_intent'] = True
|
||||
metadata_path.write_text(json.dumps(native))
|
||||
before = identity_path.read_bytes()
|
||||
migrate(binary, local, remote_a, execute=True, success=False)
|
||||
assert identity_path.read_bytes() == before
|
||||
native[native_id] = copy.deepcopy(original_records[native_id])
|
||||
native[native_id]['complete'] = False
|
||||
metadata_path.write_text(json.dumps(native))
|
||||
native_bytes = metadata_path.read_bytes()
|
||||
native_archive = archive_path.read_bytes()
|
||||
migrate(binary, local, remote_a, execute=True)
|
||||
assert metadata_path.read_bytes() == native_bytes, 'migration rewrote native completion cache'
|
||||
assert archive_path.read_bytes() == native_archive
|
||||
verify_serving(remote_a, preserve_metadata=False)
|
||||
assert all(not store.errors for store in stores), [store.errors for store in stores]
|
||||
print('PASS upload migration: local -> S3 -> S3 -> local, hashes, original URLs/JWT/HEAD, lock, dry-run, fail-closed resume')
|
||||
except BaseException:
|
||||
log.flush(); log.seek(0)
|
||||
output = log.read()
|
||||
for credential in (ACCESS, SECRET, TOKEN):
|
||||
output = output.replace(credential, '[REDACTED]')
|
||||
print('SERVER LOG:\n' + output)
|
||||
print('FIXTURE ERRORS:', [store.errors for store in stores])
|
||||
raise
|
||||
finally:
|
||||
if client is not None:
|
||||
client.close()
|
||||
if process is not None:
|
||||
stop(process)
|
||||
for store in stores:
|
||||
store.shutdown()
|
||||
for worker in workers:
|
||||
worker.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(str(Path(sys.argv[1] if len(sys.argv) > 1 else 'target/release/rust-xmpp').resolve()))
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Opt-in genuine MinIO integration; no S3 fixture is used.
|
||||
|
||||
MINIO_BINARY=/path/to/minio python3 tests/upload_minio_network.py --binary target/release/rust-xmpp
|
||||
Default sends an actual 10 GiB HTTPS upload through Rust into isolated MinIO,
|
||||
then verifies every byte through HTTPS, multipart ETag, ranges, restart and crash
|
||||
cleanup. Requires openssl, a genuine MinIO executable, and size + 512 MiB free.
|
||||
--size permits a small smoke check; only sizes >5 GiB qualify the large-object test.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import quote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from _wire_client import Client
|
||||
from upload_large_network import CHUNK, free_port, proc_status_for_child, rss, progress
|
||||
from upload_s3_network import slot, start, stop, wait_for
|
||||
|
||||
ACCESS = 'isolated-minio-test-access'
|
||||
SECRET = 'isolated-minio-test-secret-not-production'
|
||||
BUCKET = 'xmpp-integration'
|
||||
PART = 8 * 2**20
|
||||
|
||||
|
||||
def signed(port, method, path, query='', body=b''):
|
||||
"""Small independent SigV4 administrative client, never stores object bodies."""
|
||||
date = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())
|
||||
scope = date[:8] + '/us-east-1/s3/aws4_request'
|
||||
payload = hashlib.sha256(body).hexdigest()
|
||||
headers = {'host': f'127.0.0.1:{port}', 'x-amz-content-sha256': payload, 'x-amz-date': date}
|
||||
names = ';'.join(sorted(headers))
|
||||
canonical = '\n'.join((method, path, query, ''.join(f'{k}:{headers[k]}\n' for k in sorted(headers)), names, payload))
|
||||
key = ('AWS4' + SECRET).encode()
|
||||
for value in (date[:8], 'us-east-1', 's3', 'aws4_request'):
|
||||
key = hmac.new(key, value.encode(), hashlib.sha256).digest()
|
||||
value = '\n'.join(('AWS4-HMAC-SHA256', date, scope, hashlib.sha256(canonical.encode()).hexdigest()))
|
||||
signature = hmac.new(key, value.encode(), hashlib.sha256).hexdigest()
|
||||
headers['Authorization'] = f'AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={names}, Signature={signature}'
|
||||
conn = http.client.HTTPConnection('127.0.0.1', port, timeout=30)
|
||||
try:
|
||||
conn.request(method, path + ('?' + query if query else ''), body, headers)
|
||||
response = conn.getresponse()
|
||||
return response.status, dict(response.getheaders()), response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def uploads(port):
|
||||
status, _, body = signed(port, 'GET', '/' + BUCKET, 'uploads=')
|
||||
assert status == 200, (status, body[:1024])
|
||||
return [n for n in ET.fromstring(body).iter() if n.tag.endswith('}Upload')]
|
||||
|
||||
|
||||
def has_uploaded_part(port):
|
||||
for upload in uploads(port):
|
||||
values = {node.tag.rsplit('}', 1)[-1]: node.text for node in upload}
|
||||
path = '/' + BUCKET + '/' + quote(values['Key'], safe='/')
|
||||
status, _, body = signed(port, 'GET', path, 'uploadId=' + quote(values['UploadId'], safe='-_.~'))
|
||||
if status == 200 and any(node.tag.endswith('}Part') for node in ET.fromstring(body).iter()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def run(args, report):
|
||||
try:
|
||||
version = subprocess.check_output([str(args.minio), '--version'], text=True, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise RuntimeError('MinIO preflight failed before any upload: ' + error.output[-4000:]) from error
|
||||
assert 'version ' in version.lower() and 'MinIO, Inc.' in version, 'supplied executable does not identify as MinIO'
|
||||
binary_hash = hashlib.sha256()
|
||||
with args.minio.open('rb') as stream:
|
||||
for piece in iter(lambda: stream.read(2**20), b''): binary_hash.update(piece)
|
||||
rust_hash = hashlib.sha256()
|
||||
with args.binary.open('rb') as stream:
|
||||
for piece in iter(lambda: stream.read(2**20), b''): rust_hash.update(piece)
|
||||
rust_version = subprocess.check_output([str(args.binary), '--version'], text=True).strip()
|
||||
report.update(minio_version=version.strip(), minio_binary_sha256=binary_hash.hexdigest(), binary=str(args.binary), binary_version=rust_version, binary_sha256=rust_hash.hexdigest(), size=args.size, qualifies_over_5gib=args.size > 5 * 2**30)
|
||||
if args.minio_compat_patch:
|
||||
patch = args.minio_compat_patch.read_bytes()
|
||||
report['minio_test_only_compatibility_patch'] = {'sha256': hashlib.sha256(patch).hexdigest(), 'diff': patch.decode(), 'limitation': 'Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream.'}
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-minio-', dir=args.temp_dir) as directory:
|
||||
root = Path(directory)
|
||||
assert shutil.disk_usage(root).free > args.size + 512 * 2**20, 'insufficient free disk'
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-keyout', str(root/'key.pem'), '-out', str(root/'cert.pem'), '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
c2s, https, minio_port, console = [free_port() for _ in range(4)]
|
||||
cert = {'key': str(root/'key.pem'), 'certificate': str(root/'cert.pem')}
|
||||
options = {'http_host': 'localhost', 'http_file_share_backend': 's3', 'http_file_share_s3': {'endpoint': f'http://127.0.0.1:{minio_port}', 'bucket': BUCKET, 'region': 'us-east-1', 'access_key': ACCESS, 'secret_key': SECRET, 'key_prefix': 'integration'}, 'http_file_share_base_url': f'https://localhost:{https}/file_share', 'http_file_share_size_limit': args.size, 'http_file_share_daily_quota': args.size * 3, 'http_file_share_global_quota': args.size * 3, 'http_file_share_expires_after': False}
|
||||
config = {'global': {'data_path': str(root/'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [], 'https_ports': [https], 'ssl': cert, 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'ssl': cert}}, 'upload.localhost': {'component': 'http_file_share', 'options': options}}}
|
||||
config_path = root/'config.json'
|
||||
config_path.write_text(json.dumps(config))
|
||||
subprocess.run([str(args.binary), 'create-user', str(config_path), 'alice', '--password-stdin'], input='integration-secret\n', text=True, check=True, capture_output=True)
|
||||
minio = server = client = None
|
||||
done = threading.Event()
|
||||
report['max_rss_bytes'] = {'server': 0, 'client': 0, 'minio': 0}
|
||||
paths = {'client': Path('/proc/self/status')}
|
||||
def sample():
|
||||
while not done.wait(.05):
|
||||
for role, path in list(paths.items()):
|
||||
report['max_rss_bytes'][role] = max(report['max_rss_bytes'][role], rss(path))
|
||||
monitor = threading.Thread(target=sample, daemon=True)
|
||||
context = ssl.create_default_context(cafile=str(root/'cert.pem'))
|
||||
def request(url, method='GET', headers=None):
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120)
|
||||
connection.request(method, parsed.path, headers=headers or {})
|
||||
return connection, connection.getresponse()
|
||||
def check_range(url, offset):
|
||||
conn, response = request(url, headers={'Range': f'bytes={offset}-{offset+31}'})
|
||||
try:
|
||||
assert response.status == 206
|
||||
assert response.getheader('Content-Range') == f'bytes {offset}-{offset+31}/{args.size}'
|
||||
assert response.read(33) == bytes((offset+i) % 256 for i in range(32))
|
||||
finally:
|
||||
conn.close()
|
||||
with (root/'minio.log').open('w+') as minio_log, (root/'server.log').open('w+') as log:
|
||||
try:
|
||||
env = dict(os.environ, MINIO_ROOT_USER=ACCESS, MINIO_ROOT_PASSWORD=SECRET, MINIO_BROWSER='off', GOMAXPROCS='4', GOMEMLIMIT='1GiB')
|
||||
minio = subprocess.Popen([str(args.minio), 'server', str(root/'minio-data'), '--address', f'127.0.0.1:{minio_port}', '--console-address', f'127.0.0.1:{console}'], env=env, stdout=minio_log, stderr=minio_log)
|
||||
paths['minio'] = proc_status_for_child(minio.pid)
|
||||
for _ in range(200):
|
||||
assert minio.poll() is None, 'MinIO exited at startup'
|
||||
try:
|
||||
status, _, _ = signed(minio_port, 'PUT', '/' + BUCKET)
|
||||
if status in (200, 409): break
|
||||
except OSError: pass
|
||||
time.sleep(.05)
|
||||
else: raise AssertionError('MinIO startup timeout')
|
||||
server = start(str(args.binary), config_path, c2s, log)
|
||||
paths['server'] = proc_status_for_child(server.pid)
|
||||
assert all(paths.values()), 'cannot resolve actual child process RSS paths'
|
||||
monitor.start()
|
||||
client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret')
|
||||
url, headers = slot(client, 'upload.localhost', args.size, 'real-minio')
|
||||
parsed = urlsplit(url)
|
||||
assert parsed.scheme == 'https' and parsed.hostname == 'localhost'
|
||||
conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120)
|
||||
digest = hashlib.sha256(); sent = 0; began = last = time.monotonic()
|
||||
try:
|
||||
conn.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(args.size)}.items(): conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
while sent < args.size:
|
||||
piece = memoryview(CHUNK)[:min(len(CHUNK), args.size-sent)]
|
||||
conn.send(piece); digest.update(piece); sent += len(piece)
|
||||
last = progress('MINIO PUT', sent, args.size, began, last)
|
||||
response = conn.getresponse()
|
||||
assert response.status == 201, (response.status, response.read(4096))
|
||||
response.read()
|
||||
finally: conn.close()
|
||||
report.update(put_bytes=sent, put_seconds=time.monotonic()-began, sha256=digest.hexdigest())
|
||||
# MinIO's own HEAD proves the object is multipart, independent of Rust metadata.
|
||||
object_id = parsed.path.split('/')[-2]
|
||||
status, remote, body = signed(minio_port, 'HEAD', '/' + BUCKET + '/integration/' + object_id)
|
||||
assert status == 200, (status, body)
|
||||
remote = {k.lower(): v for k, v in remote.items()}
|
||||
assert int(remote['content-length']) == args.size
|
||||
etag = remote['etag'].strip('"')
|
||||
assert '-' in etag and int(etag.rsplit('-', 1)[1]) == (args.size + PART - 1)//PART, etag
|
||||
allocated = sum(path.stat().st_blocks * 512 for path in (root/'minio-data').rglob('*') if path.is_file())
|
||||
assert allocated >= args.size, 'MinIO payload storage unexpectedly sparse'
|
||||
report.update(minio_etag=etag, multipart_parts=int(etag.rsplit('-', 1)[1]), minio_allocated_bytes=allocated, free_disk_after_put_bytes=shutil.disk_usage(root).free)
|
||||
conn, response = request(url, 'HEAD')
|
||||
assert response.status == 200 and int(response.getheader('Content-Length')) == args.size
|
||||
response.read(); conn.close()
|
||||
received = 0; downloaded = hashlib.sha256(); began = last = time.monotonic()
|
||||
conn, response = request(url)
|
||||
try:
|
||||
assert response.status == 200
|
||||
while True:
|
||||
piece = response.read(len(CHUNK))
|
||||
if not piece: break
|
||||
downloaded.update(piece); received += len(piece)
|
||||
last = progress('MINIO GET', received, args.size, began, last)
|
||||
finally: conn.close()
|
||||
assert received == args.size and downloaded.digest() == digest.digest()
|
||||
report.update(get_bytes=received, get_sha256=downloaded.hexdigest(), get_seconds=time.monotonic()-began)
|
||||
offsets = sorted(set([0, min(2**32-7, args.size-32), min(5*2**30-7, args.size-32), args.size-32]))
|
||||
for offset in offsets: check_range(url, offset)
|
||||
report['range_offsets'] = offsets
|
||||
client.close(); client = None
|
||||
stop(server); server = None
|
||||
server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid)
|
||||
check_range(url, args.size-32)
|
||||
report['restart_range_ok'] = True
|
||||
replay = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=30)
|
||||
try:
|
||||
replay.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(args.size)}.items(): replay.putheader(name, value)
|
||||
replay.endheaders()
|
||||
response = replay.getresponse()
|
||||
assert response.status == 409, 'single-use upload state did not survive restart'
|
||||
response.read()
|
||||
finally: replay.close()
|
||||
report['restart_reuse_status'] = 409
|
||||
client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret')
|
||||
crash_size = max(PART*2, min(args.size, PART*3))
|
||||
# The configured maximum must also permit the crash probe in small smoke mode.
|
||||
crash_size = min(crash_size, args.size)
|
||||
assert crash_size > PART, 'size must exceed 8 MiB for crash multipart test'
|
||||
crash_url, crash_headers = slot(client, 'upload.localhost', crash_size, 'crash-minio')
|
||||
crash = urlsplit(crash_url)
|
||||
conn = http.client.HTTPSConnection(crash.hostname, crash.port, context=context, timeout=30)
|
||||
conn.putrequest('PUT', crash.path)
|
||||
for name, value in {**crash_headers, 'Content-Length': str(crash_size)}.items(): conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
for _ in range(PART//len(CHUNK)): conn.send(CHUNK)
|
||||
wait_for(lambda: has_uploaded_part(minio_port), 'MinIO never stored crash multipart part')
|
||||
crash_id = crash.path.split('/')[-2]
|
||||
metadata = root/'data'/'upload%2elocalhost'/'rust_uploads.json'
|
||||
wait_for(lambda: json.loads(metadata.read_text())[crash_id].get('upload_id'), 'multipart id not durable')
|
||||
server.kill(); server.wait(timeout=10); server = None
|
||||
conn.close(); client.close(); client = None
|
||||
assert uploads(minio_port), 'no real abandoned multipart at crash'
|
||||
server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid)
|
||||
wait_for(lambda: not uploads(minio_port), 'startup did not abort genuine MinIO multipart')
|
||||
check_range(url, args.size-32)
|
||||
report['crash_multipart_cleanup_ok'] = True
|
||||
assert 0 < report['max_rss_bytes']['server'] < args.server_rss_mib * 2**20, report['max_rss_bytes']
|
||||
assert 0 < report['max_rss_bytes']['client'] < 256 * 2**20, report['max_rss_bytes']
|
||||
assert 0 < report['max_rss_bytes']['minio'] < args.minio_rss_mib * 2**20, report['max_rss_bytes']
|
||||
report['ok'] = True
|
||||
except BaseException:
|
||||
for name, stream in [('RUST', log), ('MINIO', minio_log)]:
|
||||
stream.flush(); stream.seek(0)
|
||||
output = stream.read().replace(SECRET, '[REDACTED]').replace(ACCESS, '[REDACTED]')
|
||||
print(name + ' LOG:\n' + output[-12000:])
|
||||
raise
|
||||
finally:
|
||||
done.set()
|
||||
if monitor.is_alive(): monitor.join(timeout=2)
|
||||
if client is not None: client.close()
|
||||
if server is not None: stop(server)
|
||||
if minio is not None:
|
||||
minio.terminate()
|
||||
try: minio.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired: minio.kill(); minio.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--binary', type=Path, default=Path('target/release/rust-xmpp'))
|
||||
parser.add_argument('--minio', type=Path, default=os.environ.get('MINIO_BINARY'))
|
||||
parser.add_argument('--minio-compat-patch', type=Path, help='record any test-only MinIO environment adapter diff in the report')
|
||||
parser.add_argument('--size', type=int, default=10 * 2**30)
|
||||
parser.add_argument('--temp-dir', type=Path)
|
||||
parser.add_argument('--server-rss-mib', type=int, default=512)
|
||||
parser.add_argument('--minio-rss-mib', type=int, default=2048)
|
||||
parser.add_argument('--report', type=Path)
|
||||
args = parser.parse_args()
|
||||
parser.error('--minio or MINIO_BINARY must name a genuine MinIO executable') if args.minio is None else None
|
||||
args.binary, args.minio = args.binary.resolve(), args.minio.resolve()
|
||||
assert args.size > PART and args.size >= 32
|
||||
report = {'ok': False}
|
||||
try: run(args, report)
|
||||
except Exception as error:
|
||||
report['error'] = str(error)
|
||||
raise
|
||||
finally:
|
||||
result = json.dumps(report, indent=2)
|
||||
print(result, flush=True)
|
||||
if args.report: args.report.write_text(result + '\n')
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real TLS XMPP/HTTP multi-component upload isolation; no external services."""
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
import copy
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
from _wire_client import Client, BIN
|
||||
|
||||
UPLOAD = 'urn:xmpp:http:upload:0'
|
||||
DISCO = 'http://jabber.org/protocol/disco#info'
|
||||
|
||||
|
||||
def free_port():
|
||||
with socket.socket() as probe:
|
||||
probe.bind(('127.0.0.1', 0))
|
||||
return probe.getsockname()[1]
|
||||
|
||||
|
||||
def request(port, host, path, method='GET', body=None, authorization=None, extra_headers=None):
|
||||
connection = http.client.HTTPConnection('127.0.0.1', port, timeout=5)
|
||||
headers = {'Host': host, 'Connection': 'close'}
|
||||
if authorization:
|
||||
headers['Authorization'] = authorization
|
||||
headers.update(extra_headers or {})
|
||||
try:
|
||||
connection.request(method, path, body, headers)
|
||||
response = connection.getresponse()
|
||||
return response.status, response.read()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def running(binary, config, root):
|
||||
path = root / 'config.json'
|
||||
path.write_text(json.dumps(config))
|
||||
with (root / 'server.log').open('w+') as log:
|
||||
process = subprocess.Popen([binary, 'serve', str(path), '--experimental'], stdout=log, stderr=log)
|
||||
try:
|
||||
for _ in range(100):
|
||||
assert process.poll() is None, 'server exited during startup'
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', config['global']['http_ports'][0]), .1):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(.05)
|
||||
else:
|
||||
raise AssertionError('HTTP listener startup timed out')
|
||||
yield
|
||||
except BaseException:
|
||||
log.flush()
|
||||
log.seek(0)
|
||||
print(log.read())
|
||||
raise
|
||||
finally:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
|
||||
|
||||
def slot(client, component, identifier, size=5, error=False):
|
||||
client.send(f"<iq type='get' id='{identifier}' to='{component}'><request xmlns='{UPLOAD}' filename='hello.txt' size='{size}' content-type='text/plain'/></iq>")
|
||||
result = client.iq(identifier, error=error)
|
||||
if error:
|
||||
return result
|
||||
put = result.find(f'{{{UPLOAD}}}slot/{{{UPLOAD}}}put')
|
||||
return urlsplit(put.get('url')), put.find(f'{{{UPLOAD}}}header').text
|
||||
|
||||
|
||||
def exercise(binary, root, shared):
|
||||
root.mkdir()
|
||||
cert, key = root / 'cert.pem', root / 'key.pem'
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True)
|
||||
port, c2s = free_port(), free_port()
|
||||
hosts = ['files.example.test', 'files.example.test'] if shared else ['a.example.test', 'b.example.test']
|
||||
mounts = ['/files/a', '/files/b'] if shared else ['/files', '/files']
|
||||
components = ['upload-a.localhost', 'upload-b.localhost']
|
||||
config = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [c2s], 'http_ports': [port], 'https_ports': [], 's2s_ports': [], 'modules_enabled': ['tls', 'saslauth', 'disco', 'websocket', 'bosh', 'http_altconnect'], 'consider_bosh_secure': True, 'consider_websocket_secure': True, 'ssl': {'certificate': str(cert), 'key': str(key)}}, 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed'}}}}
|
||||
for index, component in enumerate(components):
|
||||
config['hosts'][component] = {'component': 'http_file_share', 'options': {'http_host': hosts[index], 'http_paths': {'file_share': mounts[index]}, 'http_file_share_size_limit': 8 * (index + 1), 'http_file_share_daily_quota': 5, 'http_file_share_global_quota': 5, 'http_file_share_secret': f'component-secret-{index}'}}
|
||||
path = root / 'config.json'
|
||||
path.write_text(json.dumps(config))
|
||||
subprocess.run([binary, 'create-user', str(path), 'alice', '--password-stdin'], input='secret\n', text=True, check=True, capture_output=True)
|
||||
slots = []
|
||||
with running(binary, config, root):
|
||||
client = Client(c2s, cert, 'alice', 'secret')
|
||||
try:
|
||||
client.send(f"<iq type='get' id='items' to='localhost'><query xmlns='http://jabber.org/protocol/disco#items'/></iq>")
|
||||
items = {node.get('jid') for node in client.iq('items').iter()}
|
||||
assert set(components) <= items, items
|
||||
for index, component in enumerate(components):
|
||||
client.send(f"<iq type='get' id='info-{index}' to='{component}'><query xmlns='{DISCO}'/></iq>")
|
||||
info = client.iq(f'info-{index}')
|
||||
assert any(node.get('var') == UPLOAD for node in info.iter())
|
||||
fields = {node.get('var'): node.find('{jabber:x:data}value').text for node in info.iter('{jabber:x:data}field')}
|
||||
assert fields['max-file-size'] == str(8 * (index + 1)), fields
|
||||
url, token = slot(client, component, f'slot-{index}')
|
||||
assert url.hostname == hosts[index] and url.path.startswith(mounts[index] + '/'), url
|
||||
slots.append((url, token))
|
||||
slot(client, component, f'quota-{index}', size=1, error=True)
|
||||
# A JWT issued by component A must not authorize B's valid slot.
|
||||
assert request(port, hosts[1], slots[1][0].path, 'PUT', b'hello', slots[0][1])[0] == 403
|
||||
for index, (url, token) in enumerate(slots):
|
||||
assert request(port, hosts[index], url.path, 'PUT', b'hello', token)[0] == 201
|
||||
assert request(port, hosts[index].upper() + f':{port}', url.path) == (200, b'hello')
|
||||
assert request(port, hosts[index], url.path, 'PUT', b'hello', token)[0] == 409
|
||||
assert request(port, 'unknown.example.test', url.path)[0] == 404
|
||||
other = 1 - index
|
||||
wrong_path = mounts[other] + url.path[len(mounts[index]):]
|
||||
assert request(port, hosts[other], wrong_path)[0] == 404
|
||||
# Transport and discovery middleware continue working around uploads.
|
||||
assert request(port, 'localhost', '/.well-known/host-meta')[0] == 200
|
||||
assert request(port, 'localhost', '/xmpp-websocket', extra_headers={'Connection': 'Upgrade', 'Upgrade': 'websocket', 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==', 'Sec-WebSocket-Protocol': 'xmpp'})[0] == 101
|
||||
assert request(port, 'localhost', '/http-bind', 'POST', b"<body xmlns='http://jabber.org/protocol/httpbind' to='localhost' rid='1' wait='1' hold='1' ver='1.6'/>")[0] == 200
|
||||
finally:
|
||||
client.close()
|
||||
with running(binary, config, root):
|
||||
for index, (url, _) in enumerate(slots):
|
||||
assert request(port, hosts[index], url.path) == (200, b'hello')
|
||||
log_text = (root / "server.log").read_text()
|
||||
assert '"route"="upload"' in log_text, "custom upload route missing from HTTP diagnostics"
|
||||
assert all(token not in log_text for _, token in slots), "upload credentials leaked to logs"
|
||||
if shared:
|
||||
for second_path in (mounts[0], mounts[0] + '/nested', '/'):
|
||||
ambiguous = copy.deepcopy(config)
|
||||
ambiguous['hosts'][components[1]]['options']['http_paths']['file_share'] = second_path
|
||||
path.write_text(json.dumps(ambiguous))
|
||||
result = subprocess.run([binary, 'serve', str(path), '--experimental'], capture_output=True, text=True, timeout=8)
|
||||
assert result.returncode != 0 and 'ambiguous HTTP upload components' in result.stderr, result.stderr
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--binary', default=str(BIN))
|
||||
args = parser.parse_args()
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-upload-multi-') as temporary:
|
||||
for shared in (False, True):
|
||||
exercise(str(Path(args.binary).resolve()), Path(temporary) / str(shared), shared)
|
||||
print('PASS: multiple upload hosts/shared-host paths, discovery, quotas, JWT isolation, persistence, transport composition and conflict validation')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real XMPP/HTTP upload integration against an independent S3 protocol fixture.
|
||||
|
||||
Usage: python3 tests/upload_s3_network.py target/release/rust-xmpp
|
||||
No external service or credentials are used. This validates the S3 HTTP contract
|
||||
and independently recomputes SigV4; it is not an AWS or MinIO qualification test.
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, quote, unquote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from _wire_client import Client
|
||||
|
||||
NS = 'urn:xmpp:http:upload:0'
|
||||
ACCESS = 'fixture-access-not-production'
|
||||
SECRET = 'fixture-secret-not-production'
|
||||
TOKEN = 'fixture-session-not-production'
|
||||
REGION = 'fixture-region-1'
|
||||
PART = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def free_port():
|
||||
with socket.socket() as stream:
|
||||
stream.bind(('127.0.0.1', 0))
|
||||
return stream.getsockname()[1]
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class Store(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(('127.0.0.1', 0), Handler)
|
||||
self.objects = {}
|
||||
self.uploads = {}
|
||||
self.events = []
|
||||
self.errors = []
|
||||
self.fail = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def verify(self, handler, payload):
|
||||
authorization = handler.headers['Authorization']
|
||||
algorithm, fields = authorization.split(' ', 1)
|
||||
assert algorithm == 'AWS4-HMAC-SHA256', 'wrong signature algorithm'
|
||||
fields = dict(field.strip().split('=', 1) for field in fields.split(','))
|
||||
credential = fields['Credential'].split('/')
|
||||
assert credential[0] == ACCESS and credential[2:] == [REGION, 's3', 'aws4_request'], 'wrong credential scope'
|
||||
date = handler.headers['x-amz-date']
|
||||
assert credential[1] == date[:8], 'date mismatch'
|
||||
assert handler.headers['x-amz-security-token'] == TOKEN, 'session token missing'
|
||||
payload_hash = digest(payload)
|
||||
assert handler.headers['x-amz-content-sha256'] == payload_hash, 'payload checksum mismatch'
|
||||
names = fields['SignedHeaders'].split(';')
|
||||
assert names == sorted(names), 'signed headers not sorted'
|
||||
assert {'host', 'x-amz-date', 'x-amz-content-sha256', 'x-amz-security-token'} <= set(names), 'unsigned required header'
|
||||
headers = ''.join(f'{name}:{" ".join(handler.headers[name].split())}\n' for name in names)
|
||||
parsed = urlsplit(handler.path)
|
||||
# AWS URI encoding preserves slash separators, but escapes all other
|
||||
# non-unreserved bytes. Keys here include spaces and plus signs.
|
||||
pairs = []
|
||||
for key, values in parse_qs(parsed.query, keep_blank_values=True).items():
|
||||
pairs.extend((quote(key, safe='-_.~'), quote(value, safe='-_.~')) for value in values)
|
||||
query = '&'.join(f'{key}={value}' for key, value in sorted(pairs))
|
||||
canonical = '\n'.join((handler.command, parsed.path, query, headers, ';'.join(names), payload_hash))
|
||||
scope = '/'.join(credential[1:])
|
||||
string_to_sign = '\n'.join((algorithm, date, scope, digest(canonical.encode())))
|
||||
signing_key = ('AWS4' + SECRET).encode()
|
||||
for value in (credential[1], REGION, 's3', 'aws4_request'):
|
||||
signing_key = hmac.new(signing_key, value.encode(), hashlib.sha256).digest()
|
||||
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
assert hmac.compare_digest(signature, fields['Signature']), 'invalid SigV4 signature'
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def reply(self, status, payload=b'', **headers):
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Length', str(len(payload)))
|
||||
for name, value in headers.items():
|
||||
self.send_header(name.replace('_', '-'), value)
|
||||
self.end_headers()
|
||||
if self.command != 'HEAD':
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self): self.dispatch()
|
||||
def do_HEAD(self): self.dispatch()
|
||||
def do_POST(self): self.dispatch()
|
||||
def do_PUT(self): self.dispatch()
|
||||
def do_DELETE(self): self.dispatch()
|
||||
|
||||
def dispatch(self):
|
||||
try:
|
||||
payload = self.rfile.read(int(self.headers.get('Content-Length', '0')))
|
||||
self.server.verify(self, payload)
|
||||
parsed = urlsplit(self.path)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if self.command == 'GET' and 'uploads' in query:
|
||||
assert parsed.path == '/api/fixture-bucket', 'list must address bucket root'
|
||||
assert query.get('encoding-type') == ['url'], 'list must encode arbitrary XML keys'
|
||||
prefix = query.get('prefix', [''])[0]
|
||||
with self.server.lock:
|
||||
self.server.events.append(('list', prefix, len(payload)))
|
||||
entries = sorted((unquote(upload['key'].removeprefix('/api/fixture-bucket/')), uid)
|
||||
for uid, upload in self.server.uploads.items()
|
||||
if unquote(upload['key'].removeprefix('/api/fixture-bucket/')).startswith(prefix))
|
||||
marker = (query.get('key-marker', [''])[0], query.get('upload-id-marker', [''])[0])
|
||||
entries = [entry for entry in entries if entry > marker]
|
||||
# Small pages force both marker fields to be exercised.
|
||||
page, remaining = entries[:2], entries[2:]
|
||||
root = ET.Element('ListMultipartUploadsResult')
|
||||
for key, uid in page:
|
||||
upload = ET.SubElement(root, 'Upload')
|
||||
ET.SubElement(upload, 'Key').text = quote(key, safe='-_.~')
|
||||
ET.SubElement(upload, 'UploadId').text = uid
|
||||
ET.SubElement(root, 'IsTruncated').text = 'true' if remaining else 'false'
|
||||
if remaining:
|
||||
ET.SubElement(root, 'NextKeyMarker').text = quote(page[-1][0], safe='-_.~')
|
||||
ET.SubElement(root, 'NextUploadIdMarker').text = page[-1][1]
|
||||
return self.reply(200, ET.tostring(root))
|
||||
assert parsed.path.startswith('/api/fixture-bucket/space%20%2B/'), 'bucket/prefix escaping mismatch'
|
||||
|
||||
key = parsed.path
|
||||
with self.server.lock:
|
||||
stage = ('init' if 'uploads' in query else 'part' if self.command == 'PUT'
|
||||
else 'complete' if self.command == 'POST' else 'abort' if 'uploadId' in query
|
||||
else self.command.lower())
|
||||
self.server.events.append((stage, key, len(payload)))
|
||||
if stage == 'complete':
|
||||
assert self.headers.get('If-None-Match') == '*', 'completion must prevent overwrite'
|
||||
if self.server.fail == 'complete-malformed':
|
||||
self.server.fail = None
|
||||
return self.reply(200, b'<unterminated')
|
||||
if stage == 'complete' and self.server.fail == 'complete-error':
|
||||
self.server.fail = None
|
||||
return self.reply(200, b'<Error><Code>InternalError</Code></Error>')
|
||||
if self.server.fail == stage:
|
||||
self.server.fail = None
|
||||
return self.reply(503, b'<Error><Code>FixtureUnavailable</Code></Error>')
|
||||
if stage == 'init':
|
||||
uid = f'upload+{len(self.server.events)}/='
|
||||
self.server.uploads[uid] = {'key': key, 'parts': {}}
|
||||
return self.reply(200, f'<InitiateMultipartUploadResult><UploadId>{uid}</UploadId></InitiateMultipartUploadResult>'.encode())
|
||||
if stage == 'part':
|
||||
upload = self.server.uploads[query['uploadId'][0]]
|
||||
number = int(query['partNumber'][0])
|
||||
etag = '"' + hashlib.md5(payload).hexdigest() + '"'
|
||||
upload['parts'][number] = (payload, etag)
|
||||
return self.reply(200, ETag=etag)
|
||||
if stage == 'complete':
|
||||
upload = self.server.uploads[query['uploadId'][0]]
|
||||
root = ET.fromstring(payload)
|
||||
parts = list(root)
|
||||
result = []
|
||||
for index, node in enumerate(parts, 1):
|
||||
number = int(node.findtext('PartNumber'))
|
||||
assert number == index, 'noncontiguous multipart completion'
|
||||
data, etag = upload['parts'][number]
|
||||
assert node.findtext('ETag') == etag, 'ETag mismatch'
|
||||
if index < len(parts):
|
||||
assert len(data) >= 5 * 1024 * 1024, 'nonfinal S3 part below 5 MiB'
|
||||
result.append(data)
|
||||
self.server.objects[key] = b''.join(result)
|
||||
del self.server.uploads[query['uploadId'][0]]
|
||||
if self.server.fail == 'complete-ambiguous':
|
||||
self.server.fail = None
|
||||
# The remote object exists, but the completion response
|
||||
# gives the proxy no trustworthy success confirmation.
|
||||
return self.reply(200, b'<Error><Code>InternalError</Code></Error>')
|
||||
return self.reply(200, b'<CompleteMultipartUploadResult><ETag>"complete"</ETag></CompleteMultipartUploadResult>')
|
||||
if stage == 'abort':
|
||||
removed = self.server.uploads.pop(query['uploadId'][0], None)
|
||||
return self.reply(204 if removed is not None else 404)
|
||||
if self.command == 'DELETE':
|
||||
self.server.objects.pop(key, None)
|
||||
return self.reply(204)
|
||||
data = self.server.objects.get(key)
|
||||
if data is None:
|
||||
return self.reply(404)
|
||||
if self.headers.get('Range'):
|
||||
start, end = self.headers['Range'].removeprefix('bytes=').split('-')
|
||||
start, end = int(start), int(end)
|
||||
return self.reply(206, data[start:end + 1], Content_Range=f'bytes {start}-{end}/{len(data)}')
|
||||
return self.reply(200, data, Content_Type='application/octet-stream')
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# The crash recovery case deliberately kills the HTTP peer.
|
||||
return
|
||||
except BaseException as error:
|
||||
self.server.errors.append(f'{type(error).__name__}: {error}')
|
||||
self.reply(500, b'fixture validation failed')
|
||||
|
||||
|
||||
def request(url, method='GET', body=None, headers=None):
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20)
|
||||
try:
|
||||
try:
|
||||
connection.request(method, parsed.path, body=body, headers=headers or {})
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# Authorization, replay, and backend preflight errors may produce
|
||||
# a response before the large request body finishes transmitting.
|
||||
pass
|
||||
response = connection.getresponse()
|
||||
return response.status, dict(response.getheaders()), response.read()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def slot(client, host, size, ident, error=False):
|
||||
client.send(f"<iq type='get' id='{ident}' to='{host}'><request xmlns='{NS}' filename='test +.bin' size='{size}' content-type='application/octet-stream'/></iq>")
|
||||
response = client.iq(ident, error=error)
|
||||
if error:
|
||||
return response
|
||||
node = response.find(f'{{{NS}}}slot')
|
||||
assert node is not None, ET.tostring(response)
|
||||
put = node.find(f'{{{NS}}}put')
|
||||
headers = {header.get('name'): header.text for header in put}
|
||||
return put.get('url'), headers
|
||||
|
||||
|
||||
def start(binary, config, port, log):
|
||||
process = subprocess.Popen([binary, 'serve', str(config), '--experimental'], stdout=log, stderr=log)
|
||||
for _ in range(100):
|
||||
assert process.poll() is None, 'server exited during startup'
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', port), timeout=.1):
|
||||
return process
|
||||
except OSError:
|
||||
time.sleep(.05)
|
||||
raise AssertionError('server startup timeout')
|
||||
|
||||
|
||||
def stop(process):
|
||||
process.terminate()
|
||||
try:
|
||||
assert process.wait(timeout=10) == 0, 'server shutdown failed'
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise
|
||||
|
||||
|
||||
def wait_for(predicate, message):
|
||||
for _ in range(100):
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(.05)
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def run(binary):
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-s3-network-') as directory, Store() as store:
|
||||
root = Path(directory)
|
||||
worker = threading.Thread(target=store.serve_forever, daemon=True)
|
||||
worker.start()
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
|
||||
'-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem'), '-subj',
|
||||
'/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
c2s, http_port = free_port(), free_port()
|
||||
size = PART + 123
|
||||
options = {'http_host': '127.0.0.1', 'http_file_share_backend': 's3', 'http_file_share_s3': {
|
||||
'endpoint': f'http://127.0.0.1:{store.server_address[1]}/api',
|
||||
'bucket': 'fixture-bucket', 'region': REGION, 'access_key': ACCESS,
|
||||
'secret_key': SECRET, 'session_token': TOKEN, 'key_prefix': 'space +'},
|
||||
'http_file_share_base_url': f'http://127.0.0.1:{http_port}/s3',
|
||||
'http_paths': {'file_share': '/s3'}, 'http_file_share_size_limit': size,
|
||||
'http_file_share_daily_quota': size * 7, 'http_file_share_global_quota': size * 7,
|
||||
'http_file_share_expires_after': False}
|
||||
config_data = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'],
|
||||
'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [http_port], 'https_ports': [],
|
||||
'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': {
|
||||
'localhost': {'options': {'authentication': 'internal_hashed', 'password_hash': 'SHA-256',
|
||||
'ssl': {'key': str(root / 'key.pem'), 'certificate': str(root / 'cert.pem')}}},
|
||||
's3.localhost': {'component': 'http_file_share', 'options': options},
|
||||
'local.localhost': {'component': 'http_file_share', 'options': {'http_host': '127.0.0.1',
|
||||
'http_file_share_base_url': f'http://127.0.0.1:{http_port}/local',
|
||||
'http_paths': {'file_share': '/local'}}}}}
|
||||
config = root / 'config.json'
|
||||
config.write_text(json.dumps(config_data))
|
||||
subprocess.run([binary, 'create-user', str(config), 'alice', '--password-stdin'],
|
||||
input='secret\n', text=True, check=True, capture_output=True)
|
||||
process = None
|
||||
client = None
|
||||
with (root / 'server.log').open('w+') as log:
|
||||
try:
|
||||
process = start(binary, config, c2s, log)
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
data = b'A' * PART + b'B' * 123
|
||||
url, headers = slot(client, 's3.localhost', size, 'good')
|
||||
# Send one part and fence on the fixture before sending the
|
||||
# tail: the proxy must make progress without buffering the file.
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20)
|
||||
connection.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(size)}.items():
|
||||
connection.putheader(name, value)
|
||||
connection.endheaders()
|
||||
connection.send(data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' for stage, _, _ in store.events), 'upload buffered the entire client body')
|
||||
connection.send(data[PART:])
|
||||
response = connection.getresponse()
|
||||
assert response.status == 201, (response.status, response.read())
|
||||
response.read()
|
||||
connection.close()
|
||||
assert [length for stage, _, length in store.events if stage == 'part'] == [PART, 123]
|
||||
assert request(url)[2] == data
|
||||
status, head, body = request(url, 'HEAD')
|
||||
assert status == 200 and int(head.get('content-length', head.get('Content-Length', -1))) == size and body == b''
|
||||
etag = head.get('etag', head.get('ETag'))
|
||||
assert etag, 'missing ETag'
|
||||
count = len(store.events)
|
||||
assert request(url, headers={'If-None-Match': etag})[0] == 304
|
||||
assert request(url, headers={'If-None-Match': 'W/' + etag})[0] == 304
|
||||
assert len(store.events) == count, 'conditional hit reached S3'
|
||||
assert request(url, headers={'Range': 'bytes=1-2', 'If-Range': etag})[2] == b'AA'
|
||||
status, _, body = request(url, headers={'Range': 'bytes=1-2', 'If-Range': '"stale"'})
|
||||
assert status == 200 and body == data
|
||||
status, range_headers, body = request(url, headers={'Range': f'bytes={PART - 2}-{PART + 2}'})
|
||||
assert status == 206 and body == b'AABBB'
|
||||
assert range_headers.get('content-range', range_headers.get('Content-Range')) == f'bytes {PART - 2}-{PART + 2}/{size}'
|
||||
assert request(url, headers={'Range': 'bytes=-5'})[2] == b'BBBBB'
|
||||
assert request(url, headers={'Range': f'bytes={PART}-'})[2] == b'B' * 123
|
||||
count = len(store.events)
|
||||
assert request(url, headers={'Range': f'bytes={size}-'})[0] == 416
|
||||
assert len(store.events) == count, 'invalid range reached S3'
|
||||
for stage, method in (('get', 'GET'), ('head', 'HEAD')):
|
||||
store.fail = stage
|
||||
assert request(url, method)[0] >= 500
|
||||
assert request(url)[2] == data
|
||||
assert request(url, 'PUT', data, headers)[0] == 409, 'slot reused'
|
||||
local_url, local_headers = slot(client, 'local.localhost', 4, 'local')
|
||||
assert request(local_url, 'PUT', b'here', local_headers)[0] == 201
|
||||
assert request(local_url)[2] == b'here'
|
||||
empty_url, empty_headers = slot(client, 's3.localhost', 0, 'empty')
|
||||
assert request(empty_url, 'PUT', b'', empty_headers)[0] == 201
|
||||
assert request(empty_url)[2] == b''
|
||||
for stage in ('init', 'part', 'complete', 'complete-error'):
|
||||
bad_url, bad_headers = slot(client, 's3.localhost', size, 'fail-' + stage)
|
||||
store.fail = stage
|
||||
status, _, response = request(bad_url, 'PUT', data, bad_headers)
|
||||
assert status >= 500, (stage, status)
|
||||
assert SECRET.encode() not in response and TOKEN.encode() not in response
|
||||
assert request(bad_url)[0] in (404, 410), 'failed upload became readable'
|
||||
wait_for(lambda: not store.uploads, 'failed multipart was not aborted')
|
||||
assert request(local_url)[2] == b'here', 'S3 failure broke local backend'
|
||||
# Failed slots remain retryable and only commit on success.
|
||||
def retry_failed_slot():
|
||||
# Abort reconciliation briefly holds the same slot claim.
|
||||
retry_status = request(bad_url, 'PUT', data, bad_headers)[0]
|
||||
assert retry_status in (201, 409), (stage, retry_status)
|
||||
return retry_status == 201
|
||||
wait_for(retry_failed_slot, 'failed slot remained unavailable: ' + stage)
|
||||
ambiguous_url, ambiguous_headers = slot(client, 's3.localhost', size, 'ambiguous-complete')
|
||||
store.fail = 'complete-ambiguous'
|
||||
status, _, _ = request(ambiguous_url, 'PUT', data, ambiguous_headers)
|
||||
assert status >= 500, 'ambiguous completion reported success'
|
||||
wait_for(lambda: request(ambiguous_url)[0] == 200, 'committed object was not reconciled after ambiguous completion')
|
||||
assert request(ambiguous_url)[2] == data
|
||||
assert request(ambiguous_url, 'PUT', data, ambiguous_headers)[0] == 409, 'ambiguous completion allowed object overwrite'
|
||||
assert not store.uploads
|
||||
abort_url, abort_headers = slot(client, 's3.localhost', size, 'disconnect')
|
||||
parsed = urlsplit(abort_url)
|
||||
with socket.create_connection((parsed.hostname, parsed.port), timeout=10) as stream:
|
||||
head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n'
|
||||
head += ''.join(f'{key}: {value}\r\n' for key, value in abort_headers.items()) + '\r\n'
|
||||
stream.sendall(head.encode() + data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnect fixture did not receive first part')
|
||||
wait_for(lambda: any(stage == 'abort' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnected multipart was not aborted')
|
||||
wait_for(lambda: not store.uploads, 'disconnected multipart leaked')
|
||||
slot(client, 's3.localhost', size, 'quota', error=True)
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
options['http_file_share_daily_quota'] = size * 12
|
||||
options['http_file_share_global_quota'] = size * 12
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(url)[2] == data, 'S3 metadata did not survive restart'
|
||||
assert request(url, 'PUT', data, headers)[0] == 409, 'single-use state lost on restart'
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
crash_url, crash_headers = slot(client, 's3.localhost', size, 'crash')
|
||||
parsed = urlsplit(crash_url)
|
||||
crash_id = parsed.path.split('/')[-2]
|
||||
stream = socket.create_connection((parsed.hostname, parsed.port), timeout=10)
|
||||
head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n'
|
||||
head += ''.join(f'{key}: {value}\r\n' for key, value in crash_headers.items()) + '\r\n'
|
||||
stream.sendall(head.encode() + data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' and key.endswith('/' + crash_id) for stage, key, _ in store.events), 'crash fixture did not receive first part')
|
||||
metadata = root / 'data' / 's3%2elocalhost' / 'rust_uploads.json'
|
||||
cached = json.loads(metadata.read_text())
|
||||
assert cached[crash_id].get('upload_id'), 'multipart identifier was not durable before part upload'
|
||||
process.kill(); process.wait(timeout=10); process = None
|
||||
stream.close()
|
||||
client.close(); client = None
|
||||
assert store.uploads, 'crash did not leave a remote multipart to recover'
|
||||
# Model the earlier crash window: initiation reached S3, but its
|
||||
# UploadId never reached the local journal. Include multiple
|
||||
# owned uploads plus a foreign prefix match across list pages.
|
||||
cached = json.loads(metadata.read_text())
|
||||
assert cached[crash_id].get('multipart_intent'), 'intent not durable before initiation'
|
||||
cached[crash_id]['upload_id'] = None
|
||||
metadata.write_text(json.dumps(cached))
|
||||
owned_key = '/api/fixture-bucket/space%20%2B/' + crash_id
|
||||
foreign_uid = 'foreign-prefix-match'
|
||||
with store.lock:
|
||||
for uid in ('orphan-a', 'orphan-b', 'orphan-c'):
|
||||
store.uploads[uid] = {'key': owned_key, 'parts': {}}
|
||||
store.uploads[foreign_uid] = {'key': owned_key + '-foreign', 'parts': {}}
|
||||
process = start(binary, config, c2s, log)
|
||||
wait_for(lambda: set(store.uploads) == {foreign_uid}, 'startup did not abort only exact owned-key multiparts')
|
||||
assert not any(stage == 'abort' and key == owned_key + '-foreign' for stage, key, _ in store.events), 'recovery aborted foreign prefix match'
|
||||
with store.lock:
|
||||
del store.uploads[foreign_uid]
|
||||
wait_for(lambda: not json.loads(metadata.read_text())[crash_id].get('upload_id'), 'startup retained abandoned multipart identifier')
|
||||
assert request(url)[2] == data
|
||||
stop(process); process = None
|
||||
# Model a remote completion followed by a crash before the
|
||||
# local completion marker was persisted. Age exceeds the stale
|
||||
# reservation timeout but remains within retention.
|
||||
good_id = urlsplit(url).path.split('/')[-2]
|
||||
cached = json.loads(metadata.read_text())
|
||||
old = int(time.time()) - 700
|
||||
cached[good_id].update(complete=False, upload_id='completed-before-crash', created=old)
|
||||
metadata.write_text(json.dumps(cached))
|
||||
archive = metadata.parent / 'uploads.list'
|
||||
lines = archive.read_text().splitlines(keepends=True)
|
||||
changed = False
|
||||
for index, line in enumerate(lines):
|
||||
if good_id in line:
|
||||
line, count = re.subn(r'(\["when"\]=)[0-9.]+', lambda match: match[1] + str(old) + '.0', line)
|
||||
assert count == 1, 'failed to age authoritative upload timestamp'
|
||||
lines[index] = line
|
||||
changed = True
|
||||
assert changed
|
||||
archive.write_text(''.join(lines))
|
||||
archive.with_suffix('.lidx').unlink(missing_ok=True)
|
||||
options['http_file_share_expires_after'] = 3600
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
wait_for(lambda: json.loads(metadata.read_text()).get(good_id, {}).get('complete'), 'startup did not reconcile committed object before stale pruning')
|
||||
assert request(url)[2] == data, 'committed object was deleted as an old reservation'
|
||||
assert not json.loads(metadata.read_text())[good_id].get('upload_id')
|
||||
# Unknown completion + HEAD404 quarantines only its own slot.
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
uncertain_url, uncertain_headers = slot(client, 's3.localhost', 1, 'uncertain')
|
||||
uncertain_id = urlsplit(uncertain_url).path.split('/')[-2]
|
||||
store.fail = 'complete-malformed'
|
||||
assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] >= 500
|
||||
assert json.loads(metadata.read_text())[uncertain_id]['completion_attempted']
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] == 409
|
||||
retained = json.loads(metadata.read_text())[uncertain_id]
|
||||
assert retained['completion_attempted'] and not retained['complete']
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
independent_url, independent_headers = slot(client, 's3.localhost', 1, 'independent')
|
||||
assert request(independent_url, 'PUT', b'I', independent_headers)[0] == 201
|
||||
# A late commit resolves through the next maintenance pass;
|
||||
# requesting another slot also triggers that same pruning pass.
|
||||
with store.lock:
|
||||
store.objects['/api/fixture-bucket/space%20%2B/' + uncertain_id] = b'Q'
|
||||
slot(client, 's3.localhost', 1, 'trigger-reconciliation')
|
||||
wait_for(lambda: json.loads(metadata.read_text())[uncertain_id]['complete'], 'late completion was not reconciled')
|
||||
assert request(uncertain_url)[2] == b'Q'
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
# Force expiry without slowing this test with a wall-clock wait.
|
||||
options['http_file_share_expires_after'] = 0
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(url)[0] in (404, 410)
|
||||
wait_for(lambda: not store.objects, 'expired S3 objects were not deleted')
|
||||
assert not store.errors, store.errors
|
||||
log.flush(); log.seek(0)
|
||||
output = log.read()
|
||||
assert SECRET not in output and TOKEN not in output and ACCESS not in output, 'credentials leaked to logs'
|
||||
print('PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation')
|
||||
except BaseException:
|
||||
log.flush(); log.seek(0)
|
||||
output = log.read()
|
||||
for credential in (SECRET, TOKEN, ACCESS):
|
||||
output = output.replace(credential, '[REDACTED]')
|
||||
print('SERVER LOG:\n' + output)
|
||||
print('FIXTURE ERRORS:', store.errors)
|
||||
raise
|
||||
finally:
|
||||
if client is not None:
|
||||
client.close()
|
||||
if process is not None:
|
||||
stop(process)
|
||||
store.shutdown()
|
||||
worker.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(str(Path(sys.argv[1] if len(sys.argv) > 1 else 'target/release/rust-xmpp').resolve()))
|
||||
Reference in new issue
Block a user