- Offline CLI `upload-migrate OLD_CONFIG NEW_CONFIG --component HOST
[--execute]`: dry run по умолчанию, local ↔ S3 и S3 ↔ S3 без удаления source. - Последовательное копирование с проверкой размера и полным SHA-256 readback destination; конфликтующий объект не перезаписывается. Временный диск — максимум один переносимый файл дополнительно к постоянному destination. - Durable migration journal позволяет продолжить прерванную операцию с теми же конфигурациями. Незавершённый или повреждённый journal блокирует startup; backend identity меняется только после проверки всех файлов. - Native `uploads`/`upload_stats`, slots и JWT сохраняются; поддерживаются native локальные файлы без Rust completion sidecar. Миграция требует остановки RustXMPP и Prosody, использующих общий `data_path`. - Multipart intent сохраняется до создания удалённого upload. Recovery находит потерянный UploadId через paginated listing и очищает только точный key принадлежащего slot. При сохранных metadata cleanup не зависит от lifecycle; lifecycle остаётся дополнительной защитой, а не гарантией. - Перед completion сохраняется durable marker. Неопределённый результат completion при отсутствии подтверждённого объекта помещает только этот slot в quarantine: PUT 409, retention не удаляет его, maintenance повторяет сверку. Startup продолжается; сохраняется single-use семантика. - Добавлены network regressions миграции и расширены проверки multipart recovery, потерянных ответов, отмены запросов и конфликтов destination. Настройка, команды миграции, recovery и ограничения отката: [docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). Команды, результаты и границы подтверждения, включая qualification реального object store и 10 GiB: [docs/VALIDATION-0.7.9.md](docs/VALIDATION-0.7.9.md). Итог: 265 тестов, нагрузка 500 TLS клиентов и native reference — успешно. Реальные 10 GiB через MinIO (описанный test adapter), 1280parts и полный SHA-256 пройдены; подробности — [VALIDATION-0.7.9](docs/VALIDATION-0.7.9.md).
This commit is contained in:
wt committed
1 parent
c260d7bab8
commit
b30fc524d8
41 files changed
+5936
-152
No files matched your search
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Opt-in genuine MinIO integration; no S3 fixture is used.
|
||||
|
||||
MINIO_BINARY=/path/to/minio python3 tests/upload_minio_network.py --binary target/release/rust-xmpp
|
||||
Default sends an actual 10 GiB HTTPS upload through Rust into isolated MinIO,
|
||||
then verifies every byte through HTTPS, multipart ETag, ranges, restart and crash
|
||||
cleanup. Requires openssl, a genuine MinIO executable, and size + 512 MiB free.
|
||||
--size permits a small smoke check; only sizes >5 GiB qualify the large-object test.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import quote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from _wire_client import Client
|
||||
from upload_large_network import CHUNK, free_port, proc_status_for_child, rss, progress
|
||||
from upload_s3_network import slot, start, stop, wait_for
|
||||
|
||||
ACCESS = 'isolated-minio-test-access'
|
||||
SECRET = 'isolated-minio-test-secret-not-production'
|
||||
BUCKET = 'xmpp-integration'
|
||||
PART = 8 * 2**20
|
||||
|
||||
|
||||
def signed(port, method, path, query='', body=b''):
|
||||
"""Small independent SigV4 administrative client, never stores object bodies."""
|
||||
date = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())
|
||||
scope = date[:8] + '/us-east-1/s3/aws4_request'
|
||||
payload = hashlib.sha256(body).hexdigest()
|
||||
headers = {'host': f'127.0.0.1:{port}', 'x-amz-content-sha256': payload, 'x-amz-date': date}
|
||||
names = ';'.join(sorted(headers))
|
||||
canonical = '\n'.join((method, path, query, ''.join(f'{k}:{headers[k]}\n' for k in sorted(headers)), names, payload))
|
||||
key = ('AWS4' + SECRET).encode()
|
||||
for value in (date[:8], 'us-east-1', 's3', 'aws4_request'):
|
||||
key = hmac.new(key, value.encode(), hashlib.sha256).digest()
|
||||
value = '\n'.join(('AWS4-HMAC-SHA256', date, scope, hashlib.sha256(canonical.encode()).hexdigest()))
|
||||
signature = hmac.new(key, value.encode(), hashlib.sha256).hexdigest()
|
||||
headers['Authorization'] = f'AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={names}, Signature={signature}'
|
||||
conn = http.client.HTTPConnection('127.0.0.1', port, timeout=30)
|
||||
try:
|
||||
conn.request(method, path + ('?' + query if query else ''), body, headers)
|
||||
response = conn.getresponse()
|
||||
return response.status, dict(response.getheaders()), response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def uploads(port):
|
||||
status, _, body = signed(port, 'GET', '/' + BUCKET, 'uploads=')
|
||||
assert status == 200, (status, body[:1024])
|
||||
return [n for n in ET.fromstring(body).iter() if n.tag.endswith('}Upload')]
|
||||
|
||||
|
||||
def has_uploaded_part(port):
|
||||
for upload in uploads(port):
|
||||
values = {node.tag.rsplit('}', 1)[-1]: node.text for node in upload}
|
||||
path = '/' + BUCKET + '/' + quote(values['Key'], safe='/')
|
||||
status, _, body = signed(port, 'GET', path, 'uploadId=' + quote(values['UploadId'], safe='-_.~'))
|
||||
if status == 200 and any(node.tag.endswith('}Part') for node in ET.fromstring(body).iter()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def run(args, report):
|
||||
try:
|
||||
version = subprocess.check_output([str(args.minio), '--version'], text=True, stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise RuntimeError('MinIO preflight failed before any upload: ' + error.output[-4000:]) from error
|
||||
assert 'version ' in version.lower() and 'MinIO, Inc.' in version, 'supplied executable does not identify as MinIO'
|
||||
binary_hash = hashlib.sha256()
|
||||
with args.minio.open('rb') as stream:
|
||||
for piece in iter(lambda: stream.read(2**20), b''): binary_hash.update(piece)
|
||||
rust_hash = hashlib.sha256()
|
||||
with args.binary.open('rb') as stream:
|
||||
for piece in iter(lambda: stream.read(2**20), b''): rust_hash.update(piece)
|
||||
rust_version = subprocess.check_output([str(args.binary), '--version'], text=True).strip()
|
||||
report.update(minio_version=version.strip(), minio_binary_sha256=binary_hash.hexdigest(), binary=str(args.binary), binary_version=rust_version, binary_sha256=rust_hash.hexdigest(), size=args.size, qualifies_over_5gib=args.size > 5 * 2**30)
|
||||
if args.minio_compat_patch:
|
||||
patch = args.minio_compat_patch.read_bytes()
|
||||
report['minio_test_only_compatibility_patch'] = {'sha256': hashlib.sha256(patch).hexdigest(), 'diff': patch.decode(), 'limitation': 'Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream.'}
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-minio-', dir=args.temp_dir) as directory:
|
||||
root = Path(directory)
|
||||
assert shutil.disk_usage(root).free > args.size + 512 * 2**20, 'insufficient free disk'
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-keyout', str(root/'key.pem'), '-out', str(root/'cert.pem'), '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
c2s, https, minio_port, console = [free_port() for _ in range(4)]
|
||||
cert = {'key': str(root/'key.pem'), 'certificate': str(root/'cert.pem')}
|
||||
options = {'http_host': 'localhost', 'http_file_share_backend': 's3', 'http_file_share_s3': {'endpoint': f'http://127.0.0.1:{minio_port}', 'bucket': BUCKET, 'region': 'us-east-1', 'access_key': ACCESS, 'secret_key': SECRET, 'key_prefix': 'integration'}, 'http_file_share_base_url': f'https://localhost:{https}/file_share', 'http_file_share_size_limit': args.size, 'http_file_share_daily_quota': args.size * 3, 'http_file_share_global_quota': args.size * 3, 'http_file_share_expires_after': False}
|
||||
config = {'global': {'data_path': str(root/'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [], 'https_ports': [https], 'ssl': cert, 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'ssl': cert}}, 'upload.localhost': {'component': 'http_file_share', 'options': options}}}
|
||||
config_path = root/'config.json'
|
||||
config_path.write_text(json.dumps(config))
|
||||
subprocess.run([str(args.binary), 'create-user', str(config_path), 'alice', '--password-stdin'], input='integration-secret\n', text=True, check=True, capture_output=True)
|
||||
minio = server = client = None
|
||||
done = threading.Event()
|
||||
report['max_rss_bytes'] = {'server': 0, 'client': 0, 'minio': 0}
|
||||
paths = {'client': Path('/proc/self/status')}
|
||||
def sample():
|
||||
while not done.wait(.05):
|
||||
for role, path in list(paths.items()):
|
||||
report['max_rss_bytes'][role] = max(report['max_rss_bytes'][role], rss(path))
|
||||
monitor = threading.Thread(target=sample, daemon=True)
|
||||
context = ssl.create_default_context(cafile=str(root/'cert.pem'))
|
||||
def request(url, method='GET', headers=None):
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120)
|
||||
connection.request(method, parsed.path, headers=headers or {})
|
||||
return connection, connection.getresponse()
|
||||
def check_range(url, offset):
|
||||
conn, response = request(url, headers={'Range': f'bytes={offset}-{offset+31}'})
|
||||
try:
|
||||
assert response.status == 206
|
||||
assert response.getheader('Content-Range') == f'bytes {offset}-{offset+31}/{args.size}'
|
||||
assert response.read(33) == bytes((offset+i) % 256 for i in range(32))
|
||||
finally:
|
||||
conn.close()
|
||||
with (root/'minio.log').open('w+') as minio_log, (root/'server.log').open('w+') as log:
|
||||
try:
|
||||
env = dict(os.environ, MINIO_ROOT_USER=ACCESS, MINIO_ROOT_PASSWORD=SECRET, MINIO_BROWSER='off', GOMAXPROCS='4', GOMEMLIMIT='1GiB')
|
||||
minio = subprocess.Popen([str(args.minio), 'server', str(root/'minio-data'), '--address', f'127.0.0.1:{minio_port}', '--console-address', f'127.0.0.1:{console}'], env=env, stdout=minio_log, stderr=minio_log)
|
||||
paths['minio'] = proc_status_for_child(minio.pid)
|
||||
for _ in range(200):
|
||||
assert minio.poll() is None, 'MinIO exited at startup'
|
||||
try:
|
||||
status, _, _ = signed(minio_port, 'PUT', '/' + BUCKET)
|
||||
if status in (200, 409): break
|
||||
except OSError: pass
|
||||
time.sleep(.05)
|
||||
else: raise AssertionError('MinIO startup timeout')
|
||||
server = start(str(args.binary), config_path, c2s, log)
|
||||
paths['server'] = proc_status_for_child(server.pid)
|
||||
assert all(paths.values()), 'cannot resolve actual child process RSS paths'
|
||||
monitor.start()
|
||||
client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret')
|
||||
url, headers = slot(client, 'upload.localhost', args.size, 'real-minio')
|
||||
parsed = urlsplit(url)
|
||||
assert parsed.scheme == 'https' and parsed.hostname == 'localhost'
|
||||
conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120)
|
||||
digest = hashlib.sha256(); sent = 0; began = last = time.monotonic()
|
||||
try:
|
||||
conn.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(args.size)}.items(): conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
while sent < args.size:
|
||||
piece = memoryview(CHUNK)[:min(len(CHUNK), args.size-sent)]
|
||||
conn.send(piece); digest.update(piece); sent += len(piece)
|
||||
last = progress('MINIO PUT', sent, args.size, began, last)
|
||||
response = conn.getresponse()
|
||||
assert response.status == 201, (response.status, response.read(4096))
|
||||
response.read()
|
||||
finally: conn.close()
|
||||
report.update(put_bytes=sent, put_seconds=time.monotonic()-began, sha256=digest.hexdigest())
|
||||
# MinIO's own HEAD proves the object is multipart, independent of Rust metadata.
|
||||
object_id = parsed.path.split('/')[-2]
|
||||
status, remote, body = signed(minio_port, 'HEAD', '/' + BUCKET + '/integration/' + object_id)
|
||||
assert status == 200, (status, body)
|
||||
remote = {k.lower(): v for k, v in remote.items()}
|
||||
assert int(remote['content-length']) == args.size
|
||||
etag = remote['etag'].strip('"')
|
||||
assert '-' in etag and int(etag.rsplit('-', 1)[1]) == (args.size + PART - 1)//PART, etag
|
||||
allocated = sum(path.stat().st_blocks * 512 for path in (root/'minio-data').rglob('*') if path.is_file())
|
||||
assert allocated >= args.size, 'MinIO payload storage unexpectedly sparse'
|
||||
report.update(minio_etag=etag, multipart_parts=int(etag.rsplit('-', 1)[1]), minio_allocated_bytes=allocated, free_disk_after_put_bytes=shutil.disk_usage(root).free)
|
||||
conn, response = request(url, 'HEAD')
|
||||
assert response.status == 200 and int(response.getheader('Content-Length')) == args.size
|
||||
response.read(); conn.close()
|
||||
received = 0; downloaded = hashlib.sha256(); began = last = time.monotonic()
|
||||
conn, response = request(url)
|
||||
try:
|
||||
assert response.status == 200
|
||||
while True:
|
||||
piece = response.read(len(CHUNK))
|
||||
if not piece: break
|
||||
downloaded.update(piece); received += len(piece)
|
||||
last = progress('MINIO GET', received, args.size, began, last)
|
||||
finally: conn.close()
|
||||
assert received == args.size and downloaded.digest() == digest.digest()
|
||||
report.update(get_bytes=received, get_sha256=downloaded.hexdigest(), get_seconds=time.monotonic()-began)
|
||||
offsets = sorted(set([0, min(2**32-7, args.size-32), min(5*2**30-7, args.size-32), args.size-32]))
|
||||
for offset in offsets: check_range(url, offset)
|
||||
report['range_offsets'] = offsets
|
||||
client.close(); client = None
|
||||
stop(server); server = None
|
||||
server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid)
|
||||
check_range(url, args.size-32)
|
||||
report['restart_range_ok'] = True
|
||||
replay = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=30)
|
||||
try:
|
||||
replay.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(args.size)}.items(): replay.putheader(name, value)
|
||||
replay.endheaders()
|
||||
response = replay.getresponse()
|
||||
assert response.status == 409, 'single-use upload state did not survive restart'
|
||||
response.read()
|
||||
finally: replay.close()
|
||||
report['restart_reuse_status'] = 409
|
||||
client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret')
|
||||
crash_size = max(PART*2, min(args.size, PART*3))
|
||||
# The configured maximum must also permit the crash probe in small smoke mode.
|
||||
crash_size = min(crash_size, args.size)
|
||||
assert crash_size > PART, 'size must exceed 8 MiB for crash multipart test'
|
||||
crash_url, crash_headers = slot(client, 'upload.localhost', crash_size, 'crash-minio')
|
||||
crash = urlsplit(crash_url)
|
||||
conn = http.client.HTTPSConnection(crash.hostname, crash.port, context=context, timeout=30)
|
||||
conn.putrequest('PUT', crash.path)
|
||||
for name, value in {**crash_headers, 'Content-Length': str(crash_size)}.items(): conn.putheader(name, value)
|
||||
conn.endheaders()
|
||||
for _ in range(PART//len(CHUNK)): conn.send(CHUNK)
|
||||
wait_for(lambda: has_uploaded_part(minio_port), 'MinIO never stored crash multipart part')
|
||||
crash_id = crash.path.split('/')[-2]
|
||||
metadata = root/'data'/'upload%2elocalhost'/'rust_uploads.json'
|
||||
wait_for(lambda: json.loads(metadata.read_text())[crash_id].get('upload_id'), 'multipart id not durable')
|
||||
server.kill(); server.wait(timeout=10); server = None
|
||||
conn.close(); client.close(); client = None
|
||||
assert uploads(minio_port), 'no real abandoned multipart at crash'
|
||||
server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid)
|
||||
wait_for(lambda: not uploads(minio_port), 'startup did not abort genuine MinIO multipart')
|
||||
check_range(url, args.size-32)
|
||||
report['crash_multipart_cleanup_ok'] = True
|
||||
assert 0 < report['max_rss_bytes']['server'] < args.server_rss_mib * 2**20, report['max_rss_bytes']
|
||||
assert 0 < report['max_rss_bytes']['client'] < 256 * 2**20, report['max_rss_bytes']
|
||||
assert 0 < report['max_rss_bytes']['minio'] < args.minio_rss_mib * 2**20, report['max_rss_bytes']
|
||||
report['ok'] = True
|
||||
except BaseException:
|
||||
for name, stream in [('RUST', log), ('MINIO', minio_log)]:
|
||||
stream.flush(); stream.seek(0)
|
||||
output = stream.read().replace(SECRET, '[REDACTED]').replace(ACCESS, '[REDACTED]')
|
||||
print(name + ' LOG:\n' + output[-12000:])
|
||||
raise
|
||||
finally:
|
||||
done.set()
|
||||
if monitor.is_alive(): monitor.join(timeout=2)
|
||||
if client is not None: client.close()
|
||||
if server is not None: stop(server)
|
||||
if minio is not None:
|
||||
minio.terminate()
|
||||
try: minio.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired: minio.kill(); minio.wait()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--binary', type=Path, default=Path('target/release/rust-xmpp'))
|
||||
parser.add_argument('--minio', type=Path, default=os.environ.get('MINIO_BINARY'))
|
||||
parser.add_argument('--minio-compat-patch', type=Path, help='record any test-only MinIO environment adapter diff in the report')
|
||||
parser.add_argument('--size', type=int, default=10 * 2**30)
|
||||
parser.add_argument('--temp-dir', type=Path)
|
||||
parser.add_argument('--server-rss-mib', type=int, default=512)
|
||||
parser.add_argument('--minio-rss-mib', type=int, default=2048)
|
||||
parser.add_argument('--report', type=Path)
|
||||
args = parser.parse_args()
|
||||
parser.error('--minio or MINIO_BINARY must name a genuine MinIO executable') if args.minio is None else None
|
||||
args.binary, args.minio = args.binary.resolve(), args.minio.resolve()
|
||||
assert args.size > PART and args.size >= 32
|
||||
report = {'ok': False}
|
||||
try: run(args, report)
|
||||
except Exception as error:
|
||||
report['error'] = str(error)
|
||||
raise
|
||||
finally:
|
||||
result = json.dumps(report, indent=2)
|
||||
print(result, flush=True)
|
||||
if args.report: args.report.write_text(result + '\n')
|
||||
Reference in new issue
Block a user