- Offline CLI `upload-migrate OLD_CONFIG NEW_CONFIG --component HOST
[--execute]`: dry run по умолчанию, local ↔ S3 и S3 ↔ S3 без удаления source. - Последовательное копирование с проверкой размера и полным SHA-256 readback destination; конфликтующий объект не перезаписывается. Временный диск — максимум один переносимый файл дополнительно к постоянному destination. - Durable migration journal позволяет продолжить прерванную операцию с теми же конфигурациями. Незавершённый или повреждённый journal блокирует startup; backend identity меняется только после проверки всех файлов. - Native `uploads`/`upload_stats`, slots и JWT сохраняются; поддерживаются native локальные файлы без Rust completion sidecar. Миграция требует остановки RustXMPP и Prosody, использующих общий `data_path`. - Multipart intent сохраняется до создания удалённого upload. Recovery находит потерянный UploadId через paginated listing и очищает только точный key принадлежащего slot. При сохранных metadata cleanup не зависит от lifecycle; lifecycle остаётся дополнительной защитой, а не гарантией. - Перед completion сохраняется durable marker. Неопределённый результат completion при отсутствии подтверждённого объекта помещает только этот slot в quarantine: PUT 409, retention не удаляет его, maintenance повторяет сверку. Startup продолжается; сохраняется single-use семантика. - Добавлены network regressions миграции и расширены проверки multipart recovery, потерянных ответов, отмены запросов и конфликтов destination. Настройка, команды миграции, recovery и ограничения отката: [docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). Команды, результаты и границы подтверждения, включая qualification реального object store и 10 GiB: [docs/VALIDATION-0.7.9.md](docs/VALIDATION-0.7.9.md). Итог: 265 тестов, нагрузка 500 TLS клиентов и native reference — успешно. Реальные 10 GiB через MinIO (описанный test adapter), 1280parts и полный SHA-256 пройдены; подробности — [VALIDATION-0.7.9](docs/VALIDATION-0.7.9.md).
This commit is contained in:
wt committed
1 parent
c260d7bab8
commit
b30fc524d8
41 files changed
+5936
-152
No files matched your search
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real XMPP/HTTP upload integration against an independent S3 protocol fixture.
|
||||
|
||||
Usage: python3 tests/upload_s3_network.py target/release/rust-xmpp
|
||||
No external service or credentials are used. This validates the S3 HTTP contract
|
||||
and independently recomputes SigV4; it is not an AWS or MinIO qualification test.
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, quote, unquote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from _wire_client import Client
|
||||
|
||||
NS = 'urn:xmpp:http:upload:0'
|
||||
ACCESS = 'fixture-access-not-production'
|
||||
SECRET = 'fixture-secret-not-production'
|
||||
TOKEN = 'fixture-session-not-production'
|
||||
REGION = 'fixture-region-1'
|
||||
PART = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def free_port():
|
||||
with socket.socket() as stream:
|
||||
stream.bind(('127.0.0.1', 0))
|
||||
return stream.getsockname()[1]
|
||||
|
||||
|
||||
def digest(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
class Store(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(('127.0.0.1', 0), Handler)
|
||||
self.objects = {}
|
||||
self.uploads = {}
|
||||
self.events = []
|
||||
self.errors = []
|
||||
self.fail = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def verify(self, handler, payload):
|
||||
authorization = handler.headers['Authorization']
|
||||
algorithm, fields = authorization.split(' ', 1)
|
||||
assert algorithm == 'AWS4-HMAC-SHA256', 'wrong signature algorithm'
|
||||
fields = dict(field.strip().split('=', 1) for field in fields.split(','))
|
||||
credential = fields['Credential'].split('/')
|
||||
assert credential[0] == ACCESS and credential[2:] == [REGION, 's3', 'aws4_request'], 'wrong credential scope'
|
||||
date = handler.headers['x-amz-date']
|
||||
assert credential[1] == date[:8], 'date mismatch'
|
||||
assert handler.headers['x-amz-security-token'] == TOKEN, 'session token missing'
|
||||
payload_hash = digest(payload)
|
||||
assert handler.headers['x-amz-content-sha256'] == payload_hash, 'payload checksum mismatch'
|
||||
names = fields['SignedHeaders'].split(';')
|
||||
assert names == sorted(names), 'signed headers not sorted'
|
||||
assert {'host', 'x-amz-date', 'x-amz-content-sha256', 'x-amz-security-token'} <= set(names), 'unsigned required header'
|
||||
headers = ''.join(f'{name}:{" ".join(handler.headers[name].split())}\n' for name in names)
|
||||
parsed = urlsplit(handler.path)
|
||||
# AWS URI encoding preserves slash separators, but escapes all other
|
||||
# non-unreserved bytes. Keys here include spaces and plus signs.
|
||||
pairs = []
|
||||
for key, values in parse_qs(parsed.query, keep_blank_values=True).items():
|
||||
pairs.extend((quote(key, safe='-_.~'), quote(value, safe='-_.~')) for value in values)
|
||||
query = '&'.join(f'{key}={value}' for key, value in sorted(pairs))
|
||||
canonical = '\n'.join((handler.command, parsed.path, query, headers, ';'.join(names), payload_hash))
|
||||
scope = '/'.join(credential[1:])
|
||||
string_to_sign = '\n'.join((algorithm, date, scope, digest(canonical.encode())))
|
||||
signing_key = ('AWS4' + SECRET).encode()
|
||||
for value in (credential[1], REGION, 's3', 'aws4_request'):
|
||||
signing_key = hmac.new(signing_key, value.encode(), hashlib.sha256).digest()
|
||||
signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
assert hmac.compare_digest(signature, fields['Signature']), 'invalid SigV4 signature'
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def reply(self, status, payload=b'', **headers):
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Length', str(len(payload)))
|
||||
for name, value in headers.items():
|
||||
self.send_header(name.replace('_', '-'), value)
|
||||
self.end_headers()
|
||||
if self.command != 'HEAD':
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self): self.dispatch()
|
||||
def do_HEAD(self): self.dispatch()
|
||||
def do_POST(self): self.dispatch()
|
||||
def do_PUT(self): self.dispatch()
|
||||
def do_DELETE(self): self.dispatch()
|
||||
|
||||
def dispatch(self):
|
||||
try:
|
||||
payload = self.rfile.read(int(self.headers.get('Content-Length', '0')))
|
||||
self.server.verify(self, payload)
|
||||
parsed = urlsplit(self.path)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if self.command == 'GET' and 'uploads' in query:
|
||||
assert parsed.path == '/api/fixture-bucket', 'list must address bucket root'
|
||||
assert query.get('encoding-type') == ['url'], 'list must encode arbitrary XML keys'
|
||||
prefix = query.get('prefix', [''])[0]
|
||||
with self.server.lock:
|
||||
self.server.events.append(('list', prefix, len(payload)))
|
||||
entries = sorted((unquote(upload['key'].removeprefix('/api/fixture-bucket/')), uid)
|
||||
for uid, upload in self.server.uploads.items()
|
||||
if unquote(upload['key'].removeprefix('/api/fixture-bucket/')).startswith(prefix))
|
||||
marker = (query.get('key-marker', [''])[0], query.get('upload-id-marker', [''])[0])
|
||||
entries = [entry for entry in entries if entry > marker]
|
||||
# Small pages force both marker fields to be exercised.
|
||||
page, remaining = entries[:2], entries[2:]
|
||||
root = ET.Element('ListMultipartUploadsResult')
|
||||
for key, uid in page:
|
||||
upload = ET.SubElement(root, 'Upload')
|
||||
ET.SubElement(upload, 'Key').text = quote(key, safe='-_.~')
|
||||
ET.SubElement(upload, 'UploadId').text = uid
|
||||
ET.SubElement(root, 'IsTruncated').text = 'true' if remaining else 'false'
|
||||
if remaining:
|
||||
ET.SubElement(root, 'NextKeyMarker').text = quote(page[-1][0], safe='-_.~')
|
||||
ET.SubElement(root, 'NextUploadIdMarker').text = page[-1][1]
|
||||
return self.reply(200, ET.tostring(root))
|
||||
assert parsed.path.startswith('/api/fixture-bucket/space%20%2B/'), 'bucket/prefix escaping mismatch'
|
||||
|
||||
key = parsed.path
|
||||
with self.server.lock:
|
||||
stage = ('init' if 'uploads' in query else 'part' if self.command == 'PUT'
|
||||
else 'complete' if self.command == 'POST' else 'abort' if 'uploadId' in query
|
||||
else self.command.lower())
|
||||
self.server.events.append((stage, key, len(payload)))
|
||||
if stage == 'complete':
|
||||
assert self.headers.get('If-None-Match') == '*', 'completion must prevent overwrite'
|
||||
if self.server.fail == 'complete-malformed':
|
||||
self.server.fail = None
|
||||
return self.reply(200, b'<unterminated')
|
||||
if stage == 'complete' and self.server.fail == 'complete-error':
|
||||
self.server.fail = None
|
||||
return self.reply(200, b'<Error><Code>InternalError</Code></Error>')
|
||||
if self.server.fail == stage:
|
||||
self.server.fail = None
|
||||
return self.reply(503, b'<Error><Code>FixtureUnavailable</Code></Error>')
|
||||
if stage == 'init':
|
||||
uid = f'upload+{len(self.server.events)}/='
|
||||
self.server.uploads[uid] = {'key': key, 'parts': {}}
|
||||
return self.reply(200, f'<InitiateMultipartUploadResult><UploadId>{uid}</UploadId></InitiateMultipartUploadResult>'.encode())
|
||||
if stage == 'part':
|
||||
upload = self.server.uploads[query['uploadId'][0]]
|
||||
number = int(query['partNumber'][0])
|
||||
etag = '"' + hashlib.md5(payload).hexdigest() + '"'
|
||||
upload['parts'][number] = (payload, etag)
|
||||
return self.reply(200, ETag=etag)
|
||||
if stage == 'complete':
|
||||
upload = self.server.uploads[query['uploadId'][0]]
|
||||
root = ET.fromstring(payload)
|
||||
parts = list(root)
|
||||
result = []
|
||||
for index, node in enumerate(parts, 1):
|
||||
number = int(node.findtext('PartNumber'))
|
||||
assert number == index, 'noncontiguous multipart completion'
|
||||
data, etag = upload['parts'][number]
|
||||
assert node.findtext('ETag') == etag, 'ETag mismatch'
|
||||
if index < len(parts):
|
||||
assert len(data) >= 5 * 1024 * 1024, 'nonfinal S3 part below 5 MiB'
|
||||
result.append(data)
|
||||
self.server.objects[key] = b''.join(result)
|
||||
del self.server.uploads[query['uploadId'][0]]
|
||||
if self.server.fail == 'complete-ambiguous':
|
||||
self.server.fail = None
|
||||
# The remote object exists, but the completion response
|
||||
# gives the proxy no trustworthy success confirmation.
|
||||
return self.reply(200, b'<Error><Code>InternalError</Code></Error>')
|
||||
return self.reply(200, b'<CompleteMultipartUploadResult><ETag>"complete"</ETag></CompleteMultipartUploadResult>')
|
||||
if stage == 'abort':
|
||||
removed = self.server.uploads.pop(query['uploadId'][0], None)
|
||||
return self.reply(204 if removed is not None else 404)
|
||||
if self.command == 'DELETE':
|
||||
self.server.objects.pop(key, None)
|
||||
return self.reply(204)
|
||||
data = self.server.objects.get(key)
|
||||
if data is None:
|
||||
return self.reply(404)
|
||||
if self.headers.get('Range'):
|
||||
start, end = self.headers['Range'].removeprefix('bytes=').split('-')
|
||||
start, end = int(start), int(end)
|
||||
return self.reply(206, data[start:end + 1], Content_Range=f'bytes {start}-{end}/{len(data)}')
|
||||
return self.reply(200, data, Content_Type='application/octet-stream')
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# The crash recovery case deliberately kills the HTTP peer.
|
||||
return
|
||||
except BaseException as error:
|
||||
self.server.errors.append(f'{type(error).__name__}: {error}')
|
||||
self.reply(500, b'fixture validation failed')
|
||||
|
||||
|
||||
def request(url, method='GET', body=None, headers=None):
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20)
|
||||
try:
|
||||
try:
|
||||
connection.request(method, parsed.path, body=body, headers=headers or {})
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
# Authorization, replay, and backend preflight errors may produce
|
||||
# a response before the large request body finishes transmitting.
|
||||
pass
|
||||
response = connection.getresponse()
|
||||
return response.status, dict(response.getheaders()), response.read()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def slot(client, host, size, ident, error=False):
|
||||
client.send(f"<iq type='get' id='{ident}' to='{host}'><request xmlns='{NS}' filename='test +.bin' size='{size}' content-type='application/octet-stream'/></iq>")
|
||||
response = client.iq(ident, error=error)
|
||||
if error:
|
||||
return response
|
||||
node = response.find(f'{{{NS}}}slot')
|
||||
assert node is not None, ET.tostring(response)
|
||||
put = node.find(f'{{{NS}}}put')
|
||||
headers = {header.get('name'): header.text for header in put}
|
||||
return put.get('url'), headers
|
||||
|
||||
|
||||
def start(binary, config, port, log):
|
||||
process = subprocess.Popen([binary, 'serve', str(config), '--experimental'], stdout=log, stderr=log)
|
||||
for _ in range(100):
|
||||
assert process.poll() is None, 'server exited during startup'
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', port), timeout=.1):
|
||||
return process
|
||||
except OSError:
|
||||
time.sleep(.05)
|
||||
raise AssertionError('server startup timeout')
|
||||
|
||||
|
||||
def stop(process):
|
||||
process.terminate()
|
||||
try:
|
||||
assert process.wait(timeout=10) == 0, 'server shutdown failed'
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise
|
||||
|
||||
|
||||
def wait_for(predicate, message):
|
||||
for _ in range(100):
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(.05)
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def run(binary):
|
||||
with tempfile.TemporaryDirectory(prefix='xmpp-s3-network-') as directory, Store() as store:
|
||||
root = Path(directory)
|
||||
worker = threading.Thread(target=store.serve_forever, daemon=True)
|
||||
worker.start()
|
||||
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
|
||||
'-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem'), '-subj',
|
||||
'/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
c2s, http_port = free_port(), free_port()
|
||||
size = PART + 123
|
||||
options = {'http_host': '127.0.0.1', 'http_file_share_backend': 's3', 'http_file_share_s3': {
|
||||
'endpoint': f'http://127.0.0.1:{store.server_address[1]}/api',
|
||||
'bucket': 'fixture-bucket', 'region': REGION, 'access_key': ACCESS,
|
||||
'secret_key': SECRET, 'session_token': TOKEN, 'key_prefix': 'space +'},
|
||||
'http_file_share_base_url': f'http://127.0.0.1:{http_port}/s3',
|
||||
'http_paths': {'file_share': '/s3'}, 'http_file_share_size_limit': size,
|
||||
'http_file_share_daily_quota': size * 7, 'http_file_share_global_quota': size * 7,
|
||||
'http_file_share_expires_after': False}
|
||||
config_data = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'],
|
||||
'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [http_port], 'https_ports': [],
|
||||
'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': {
|
||||
'localhost': {'options': {'authentication': 'internal_hashed', 'password_hash': 'SHA-256',
|
||||
'ssl': {'key': str(root / 'key.pem'), 'certificate': str(root / 'cert.pem')}}},
|
||||
's3.localhost': {'component': 'http_file_share', 'options': options},
|
||||
'local.localhost': {'component': 'http_file_share', 'options': {'http_host': '127.0.0.1',
|
||||
'http_file_share_base_url': f'http://127.0.0.1:{http_port}/local',
|
||||
'http_paths': {'file_share': '/local'}}}}}
|
||||
config = root / 'config.json'
|
||||
config.write_text(json.dumps(config_data))
|
||||
subprocess.run([binary, 'create-user', str(config), 'alice', '--password-stdin'],
|
||||
input='secret\n', text=True, check=True, capture_output=True)
|
||||
process = None
|
||||
client = None
|
||||
with (root / 'server.log').open('w+') as log:
|
||||
try:
|
||||
process = start(binary, config, c2s, log)
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
data = b'A' * PART + b'B' * 123
|
||||
url, headers = slot(client, 's3.localhost', size, 'good')
|
||||
# Send one part and fence on the fixture before sending the
|
||||
# tail: the proxy must make progress without buffering the file.
|
||||
parsed = urlsplit(url)
|
||||
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20)
|
||||
connection.putrequest('PUT', parsed.path)
|
||||
for name, value in {**headers, 'Content-Length': str(size)}.items():
|
||||
connection.putheader(name, value)
|
||||
connection.endheaders()
|
||||
connection.send(data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' for stage, _, _ in store.events), 'upload buffered the entire client body')
|
||||
connection.send(data[PART:])
|
||||
response = connection.getresponse()
|
||||
assert response.status == 201, (response.status, response.read())
|
||||
response.read()
|
||||
connection.close()
|
||||
assert [length for stage, _, length in store.events if stage == 'part'] == [PART, 123]
|
||||
assert request(url)[2] == data
|
||||
status, head, body = request(url, 'HEAD')
|
||||
assert status == 200 and int(head.get('content-length', head.get('Content-Length', -1))) == size and body == b''
|
||||
etag = head.get('etag', head.get('ETag'))
|
||||
assert etag, 'missing ETag'
|
||||
count = len(store.events)
|
||||
assert request(url, headers={'If-None-Match': etag})[0] == 304
|
||||
assert request(url, headers={'If-None-Match': 'W/' + etag})[0] == 304
|
||||
assert len(store.events) == count, 'conditional hit reached S3'
|
||||
assert request(url, headers={'Range': 'bytes=1-2', 'If-Range': etag})[2] == b'AA'
|
||||
status, _, body = request(url, headers={'Range': 'bytes=1-2', 'If-Range': '"stale"'})
|
||||
assert status == 200 and body == data
|
||||
status, range_headers, body = request(url, headers={'Range': f'bytes={PART - 2}-{PART + 2}'})
|
||||
assert status == 206 and body == b'AABBB'
|
||||
assert range_headers.get('content-range', range_headers.get('Content-Range')) == f'bytes {PART - 2}-{PART + 2}/{size}'
|
||||
assert request(url, headers={'Range': 'bytes=-5'})[2] == b'BBBBB'
|
||||
assert request(url, headers={'Range': f'bytes={PART}-'})[2] == b'B' * 123
|
||||
count = len(store.events)
|
||||
assert request(url, headers={'Range': f'bytes={size}-'})[0] == 416
|
||||
assert len(store.events) == count, 'invalid range reached S3'
|
||||
for stage, method in (('get', 'GET'), ('head', 'HEAD')):
|
||||
store.fail = stage
|
||||
assert request(url, method)[0] >= 500
|
||||
assert request(url)[2] == data
|
||||
assert request(url, 'PUT', data, headers)[0] == 409, 'slot reused'
|
||||
local_url, local_headers = slot(client, 'local.localhost', 4, 'local')
|
||||
assert request(local_url, 'PUT', b'here', local_headers)[0] == 201
|
||||
assert request(local_url)[2] == b'here'
|
||||
empty_url, empty_headers = slot(client, 's3.localhost', 0, 'empty')
|
||||
assert request(empty_url, 'PUT', b'', empty_headers)[0] == 201
|
||||
assert request(empty_url)[2] == b''
|
||||
for stage in ('init', 'part', 'complete', 'complete-error'):
|
||||
bad_url, bad_headers = slot(client, 's3.localhost', size, 'fail-' + stage)
|
||||
store.fail = stage
|
||||
status, _, response = request(bad_url, 'PUT', data, bad_headers)
|
||||
assert status >= 500, (stage, status)
|
||||
assert SECRET.encode() not in response and TOKEN.encode() not in response
|
||||
assert request(bad_url)[0] in (404, 410), 'failed upload became readable'
|
||||
wait_for(lambda: not store.uploads, 'failed multipart was not aborted')
|
||||
assert request(local_url)[2] == b'here', 'S3 failure broke local backend'
|
||||
# Failed slots remain retryable and only commit on success.
|
||||
def retry_failed_slot():
|
||||
# Abort reconciliation briefly holds the same slot claim.
|
||||
retry_status = request(bad_url, 'PUT', data, bad_headers)[0]
|
||||
assert retry_status in (201, 409), (stage, retry_status)
|
||||
return retry_status == 201
|
||||
wait_for(retry_failed_slot, 'failed slot remained unavailable: ' + stage)
|
||||
ambiguous_url, ambiguous_headers = slot(client, 's3.localhost', size, 'ambiguous-complete')
|
||||
store.fail = 'complete-ambiguous'
|
||||
status, _, _ = request(ambiguous_url, 'PUT', data, ambiguous_headers)
|
||||
assert status >= 500, 'ambiguous completion reported success'
|
||||
wait_for(lambda: request(ambiguous_url)[0] == 200, 'committed object was not reconciled after ambiguous completion')
|
||||
assert request(ambiguous_url)[2] == data
|
||||
assert request(ambiguous_url, 'PUT', data, ambiguous_headers)[0] == 409, 'ambiguous completion allowed object overwrite'
|
||||
assert not store.uploads
|
||||
abort_url, abort_headers = slot(client, 's3.localhost', size, 'disconnect')
|
||||
parsed = urlsplit(abort_url)
|
||||
with socket.create_connection((parsed.hostname, parsed.port), timeout=10) as stream:
|
||||
head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n'
|
||||
head += ''.join(f'{key}: {value}\r\n' for key, value in abort_headers.items()) + '\r\n'
|
||||
stream.sendall(head.encode() + data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnect fixture did not receive first part')
|
||||
wait_for(lambda: any(stage == 'abort' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnected multipart was not aborted')
|
||||
wait_for(lambda: not store.uploads, 'disconnected multipart leaked')
|
||||
slot(client, 's3.localhost', size, 'quota', error=True)
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
options['http_file_share_daily_quota'] = size * 12
|
||||
options['http_file_share_global_quota'] = size * 12
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(url)[2] == data, 'S3 metadata did not survive restart'
|
||||
assert request(url, 'PUT', data, headers)[0] == 409, 'single-use state lost on restart'
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
crash_url, crash_headers = slot(client, 's3.localhost', size, 'crash')
|
||||
parsed = urlsplit(crash_url)
|
||||
crash_id = parsed.path.split('/')[-2]
|
||||
stream = socket.create_connection((parsed.hostname, parsed.port), timeout=10)
|
||||
head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n'
|
||||
head += ''.join(f'{key}: {value}\r\n' for key, value in crash_headers.items()) + '\r\n'
|
||||
stream.sendall(head.encode() + data[:PART])
|
||||
wait_for(lambda: any(stage == 'part' and key.endswith('/' + crash_id) for stage, key, _ in store.events), 'crash fixture did not receive first part')
|
||||
metadata = root / 'data' / 's3%2elocalhost' / 'rust_uploads.json'
|
||||
cached = json.loads(metadata.read_text())
|
||||
assert cached[crash_id].get('upload_id'), 'multipart identifier was not durable before part upload'
|
||||
process.kill(); process.wait(timeout=10); process = None
|
||||
stream.close()
|
||||
client.close(); client = None
|
||||
assert store.uploads, 'crash did not leave a remote multipart to recover'
|
||||
# Model the earlier crash window: initiation reached S3, but its
|
||||
# UploadId never reached the local journal. Include multiple
|
||||
# owned uploads plus a foreign prefix match across list pages.
|
||||
cached = json.loads(metadata.read_text())
|
||||
assert cached[crash_id].get('multipart_intent'), 'intent not durable before initiation'
|
||||
cached[crash_id]['upload_id'] = None
|
||||
metadata.write_text(json.dumps(cached))
|
||||
owned_key = '/api/fixture-bucket/space%20%2B/' + crash_id
|
||||
foreign_uid = 'foreign-prefix-match'
|
||||
with store.lock:
|
||||
for uid in ('orphan-a', 'orphan-b', 'orphan-c'):
|
||||
store.uploads[uid] = {'key': owned_key, 'parts': {}}
|
||||
store.uploads[foreign_uid] = {'key': owned_key + '-foreign', 'parts': {}}
|
||||
process = start(binary, config, c2s, log)
|
||||
wait_for(lambda: set(store.uploads) == {foreign_uid}, 'startup did not abort only exact owned-key multiparts')
|
||||
assert not any(stage == 'abort' and key == owned_key + '-foreign' for stage, key, _ in store.events), 'recovery aborted foreign prefix match'
|
||||
with store.lock:
|
||||
del store.uploads[foreign_uid]
|
||||
wait_for(lambda: not json.loads(metadata.read_text())[crash_id].get('upload_id'), 'startup retained abandoned multipart identifier')
|
||||
assert request(url)[2] == data
|
||||
stop(process); process = None
|
||||
# Model a remote completion followed by a crash before the
|
||||
# local completion marker was persisted. Age exceeds the stale
|
||||
# reservation timeout but remains within retention.
|
||||
good_id = urlsplit(url).path.split('/')[-2]
|
||||
cached = json.loads(metadata.read_text())
|
||||
old = int(time.time()) - 700
|
||||
cached[good_id].update(complete=False, upload_id='completed-before-crash', created=old)
|
||||
metadata.write_text(json.dumps(cached))
|
||||
archive = metadata.parent / 'uploads.list'
|
||||
lines = archive.read_text().splitlines(keepends=True)
|
||||
changed = False
|
||||
for index, line in enumerate(lines):
|
||||
if good_id in line:
|
||||
line, count = re.subn(r'(\["when"\]=)[0-9.]+', lambda match: match[1] + str(old) + '.0', line)
|
||||
assert count == 1, 'failed to age authoritative upload timestamp'
|
||||
lines[index] = line
|
||||
changed = True
|
||||
assert changed
|
||||
archive.write_text(''.join(lines))
|
||||
archive.with_suffix('.lidx').unlink(missing_ok=True)
|
||||
options['http_file_share_expires_after'] = 3600
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
wait_for(lambda: json.loads(metadata.read_text()).get(good_id, {}).get('complete'), 'startup did not reconcile committed object before stale pruning')
|
||||
assert request(url)[2] == data, 'committed object was deleted as an old reservation'
|
||||
assert not json.loads(metadata.read_text())[good_id].get('upload_id')
|
||||
# Unknown completion + HEAD404 quarantines only its own slot.
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
uncertain_url, uncertain_headers = slot(client, 's3.localhost', 1, 'uncertain')
|
||||
uncertain_id = urlsplit(uncertain_url).path.split('/')[-2]
|
||||
store.fail = 'complete-malformed'
|
||||
assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] >= 500
|
||||
assert json.loads(metadata.read_text())[uncertain_id]['completion_attempted']
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] == 409
|
||||
retained = json.loads(metadata.read_text())[uncertain_id]
|
||||
assert retained['completion_attempted'] and not retained['complete']
|
||||
client = Client(c2s, root / 'cert.pem', 'alice', 'secret')
|
||||
independent_url, independent_headers = slot(client, 's3.localhost', 1, 'independent')
|
||||
assert request(independent_url, 'PUT', b'I', independent_headers)[0] == 201
|
||||
# A late commit resolves through the next maintenance pass;
|
||||
# requesting another slot also triggers that same pruning pass.
|
||||
with store.lock:
|
||||
store.objects['/api/fixture-bucket/space%20%2B/' + uncertain_id] = b'Q'
|
||||
slot(client, 's3.localhost', 1, 'trigger-reconciliation')
|
||||
wait_for(lambda: json.loads(metadata.read_text())[uncertain_id]['complete'], 'late completion was not reconciled')
|
||||
assert request(uncertain_url)[2] == b'Q'
|
||||
client.close(); client = None
|
||||
stop(process); process = None
|
||||
# Force expiry without slowing this test with a wall-clock wait.
|
||||
options['http_file_share_expires_after'] = 0
|
||||
config.write_text(json.dumps(config_data))
|
||||
process = start(binary, config, c2s, log)
|
||||
assert request(url)[0] in (404, 410)
|
||||
wait_for(lambda: not store.objects, 'expired S3 objects were not deleted')
|
||||
assert not store.errors, store.errors
|
||||
log.flush(); log.seek(0)
|
||||
output = log.read()
|
||||
assert SECRET not in output and TOKEN not in output and ACCESS not in output, 'credentials leaked to logs'
|
||||
print('PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation')
|
||||
except BaseException:
|
||||
log.flush(); log.seek(0)
|
||||
output = log.read()
|
||||
for credential in (SECRET, TOKEN, ACCESS):
|
||||
output = output.replace(credential, '[REDACTED]')
|
||||
print('SERVER LOG:\n' + output)
|
||||
print('FIXTURE ERRORS:', store.errors)
|
||||
raise
|
||||
finally:
|
||||
if client is not None:
|
||||
client.close()
|
||||
if process is not None:
|
||||
stop(process)
|
||||
store.shutdown()
|
||||
worker.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(str(Path(sys.argv[1] if len(sys.argv) > 1 else 'target/release/rust-xmpp').resolve()))
|
||||
Reference in new issue
Block a user