From b30fc524d8fca470fe06d91788ceaf019770c9b3 Mon Sep 17 00:00:00 2001 From: wt Date: Tue, 20 Oct 2026 17:06:14 +0700 Subject: [PATCH] =?UTF-8?q?-=20Offline=20CLI=20`upload-migrate=20OLD=5FCON?= =?UTF-8?q?FIG=20NEW=5FCONFIG=20--component=20HOST=20=20=20[--execute]`:?= =?UTF-8?q?=20dry=20run=20=D0=BF=D0=BE=20=D1=83=D0=BC=D0=BE=D0=BB=D1=87?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8E,=20local=20=E2=86=94=20S3=20=D0=B8=20S?= =?UTF-8?q?3=20=E2=86=94=20S3=20=D0=B1=D0=B5=D0=B7=20=D1=83=D0=B4=D0=B0?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=20=20source.=20-=20=D0=9F?= =?UTF-8?q?=D0=BE=D1=81=D0=BB=D0=B5=D0=B4=D0=BE=D0=B2=D0=B0=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D0=BE=D0=B5=20=D0=BA=D0=BE=D0=BF=D0=B8=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D1=81=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=BE=D0=B9=20=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D0=BC=D0=B5=D1=80=D0=B0=20=D0=B8=20=D0=BF=D0=BE=D0=BB=D0=BD?= =?UTF-8?q?=D1=8B=D0=BC=20SHA-256=20=20=20readback=20=20=20destination;=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BD=D1=84=D0=BB=D0=B8=D0=BA=D1=82=D1=83=D1=8E?= =?UTF-8?q?=D1=89=D0=B8=D0=B9=20=D0=BE=D0=B1=D1=8A=D0=B5=D0=BA=D1=82=20?= =?UTF-8?q?=D0=BD=D0=B5=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B7=D0=B0=D0=BF=D0=B8?= =?UTF-8?q?=D1=81=D1=8B=D0=B2=D0=B0=D0=B5=D1=82=D1=81=D1=8F.=20=D0=92?= =?UTF-8?q?=D1=80=D0=B5=D0=BC=D0=B5=D0=BD=D0=BD=D1=8B=D0=B9=20=D0=B4=D0=B8?= =?UTF-8?q?=D1=81=D0=BA=20=20=20=E2=80=94=20=20=20=D0=BC=D0=B0=D0=BA=D1=81?= =?UTF-8?q?=D0=B8=D0=BC=D1=83=D0=BC=20=D0=BE=D0=B4=D0=B8=D0=BD=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=BD=D0=BE=D1=81=D0=B8=D0=BC=D1=8B=D0=B9=20?= =?UTF-8?q?=D1=84=D0=B0=D0=B9=D0=BB=20=D0=B4=D0=BE=D0=BF=D0=BE=D0=BB=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=20=D0=BA=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=BE=D1=8F=D0=BD=D0=BD=D0=BE=D0=BC=D1=83=20?= =?UTF-8?q?=20=20destination.=20-=20Durable=20migration=20journal=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B7=D0=B2=D0=BE=D0=BB=D1=8F=D0=B5=D1=82=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6=D0=B8=D1=82=D1=8C=20=D0=BF=D1=80?= =?UTF-8?q?=D0=B5=D1=80=D0=B2=D0=B0=D0=BD=D0=BD=D1=83=D1=8E=20=D0=BE=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B0=D1=86=D0=B8=D1=8E=20=D1=81=20=20=20=D1=82?= =?UTF-8?q?=D0=B5=D0=BC=D0=B8=20=20=20=D0=B6=D0=B5=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=84=D0=B8=D0=B3=D1=83=D1=80=D0=B0=D1=86=D0=B8=D1=8F=D0=BC?= =?UTF-8?q?=D0=B8.=20=D0=9D=D0=B5=D0=B7=D0=B0=D0=B2=D0=B5=D1=80=D1=88?= =?UTF-8?q?=D1=91=D0=BD=D0=BD=D1=8B=D0=B9=20=D0=B8=D0=BB=D0=B8=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2=D1=80=D0=B5=D0=B6=D0=B4=D1=91=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D0=B9=20journal=20=D0=B1=D0=BB=D0=BE=D0=BA=D0=B8=D1=80=D1=83?= =?UTF-8?q?=D0=B5=D1=82=20=20=20startup;=20=20=20backend=20identity=20?= =?UTF-8?q?=D0=BC=D0=B5=D0=BD=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20=D1=82=D0=BE?= =?UTF-8?q?=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5=20?= =?UTF-8?q?=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B8=20=D0=B2=D1=81?= =?UTF-8?q?=D0=B5=D1=85=20=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2.=20-=20Nati?= =?UTF-8?q?ve=20`uploads`/`upload=5Fstats`,=20slots=20=D0=B8=20JWT=20?= =?UTF-8?q?=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D1=8F=D1=8E=D1=82=D1=81?= =?UTF-8?q?=D1=8F;=20=20=20=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=B8=D0=B2=D0=B0=D1=8E=D1=82=D1=81=D1=8F=20=20=20native=20?= =?UTF-8?q?=D0=BB=D0=BE=D0=BA=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=B5=20=D1=84?= =?UTF-8?q?=D0=B0=D0=B9=D0=BB=D1=8B=20=D0=B1=D0=B5=D0=B7=20Rust=20completi?= =?UTF-8?q?on=20sidecar.=20=D0=9C=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8?= =?UTF-8?q?=D1=8F=20=D1=82=D1=80=D0=B5=D0=B1=D1=83=D0=B5=D1=82=20=20=20?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BA=D0=B8=20RustXM?= =?UTF-8?q?PP=20=D0=B8=20Prosody,=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D0=B7=D1=83=D1=8E=D1=89=D0=B8=D1=85=20=D0=BE=D0=B1=D1=89=D0=B8?= =?UTF-8?q?=D0=B9=20`data=5Fpath`.=20-=20Multipart=20intent=20=D1=81=D0=BE?= =?UTF-8?q?=D1=85=D1=80=D0=B0=D0=BD=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20=D0=B4?= =?UTF-8?q?=D0=BE=20=D1=81=D0=BE=D0=B7=D0=B4=D0=B0=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D1=83=D0=B4=D0=B0=D0=BB=D1=91=D0=BD=D0=BD=D0=BE=D0=B3=D0=BE=20?= =?UTF-8?q?upload.=20Recovery=20=20=20=D0=BD=D0=B0=D1=85=D0=BE=D0=B4=D0=B8?= =?UTF-8?q?=D1=82=20=D0=BF=D0=BE=D1=82=D0=B5=D1=80=D1=8F=D0=BD=D0=BD=D1=8B?= =?UTF-8?q?=D0=B9=20UploadId=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20paginated?= =?UTF-8?q?=20listing=20=D0=B8=20=D0=BE=D1=87=D0=B8=D1=89=D0=B0=D0=B5?= =?UTF-8?q?=D1=82=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=20=20=D1=82?= =?UTF-8?q?=D0=BE=D1=87=D0=BD=D1=8B=D0=B9=20=20=20key=20=D0=BF=D1=80=D0=B8?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=D0=BB=D0=B5=D0=B6=D0=B0=D1=89=D0=B5=D0=B3?= =?UTF-8?q?=D0=BE=20slot.=20=D0=9F=D1=80=D0=B8=20=D1=81=D0=BE=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=BD=D1=8B=D1=85=20metadata=20cleanup=20=D0=BD?= =?UTF-8?q?=D0=B5=20=D0=B7=D0=B0=D0=B2=D0=B8=D1=81=D0=B8=D1=82=20=D0=BE?= =?UTF-8?q?=D1=82=20=20=20lifecycle;=20lifecycle=20=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D1=91=D1=82=D1=81=D1=8F=20=D0=B4=D0=BE=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D0=BD=D0=B8=D1=82=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=D0=B9=20=D0=B7?= =?UTF-8?q?=D0=B0=D1=89=D0=B8=D1=82=D0=BE=D0=B9,=20=D0=B0=20=D0=BD=D0=B5?= =?UTF-8?q?=20=D0=B3=D0=B0=D1=80=D0=B0=D0=BD=D1=82=D0=B8=D0=B5=D0=B9.=20-?= =?UTF-8?q?=20=D0=9F=D0=B5=D1=80=D0=B5=D0=B4=20completion=20=D1=81=D0=BE?= =?UTF-8?q?=D1=85=D1=80=D0=B0=D0=BD=D1=8F=D0=B5=D1=82=D1=81=D1=8F=20durabl?= =?UTF-8?q?e=20marker.=20=D0=9D=D0=B5=D0=BE=D0=BF=D1=80=D0=B5=D0=B4=D0=B5?= =?UTF-8?q?=D0=BB=D1=91=D0=BD=D0=BD=D1=8B=D0=B9=20=D1=80=D0=B5=D0=B7=D1=83?= =?UTF-8?q?=D0=BB=D1=8C=D1=82=D0=B0=D1=82=20=20=20completion=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B8=20=D0=BE=D1=82=D1=81=D1=83=D1=82=D1=81=D1=82=D0=B2?= =?UTF-8?q?=D0=B8=D0=B8=20=D0=BF=D0=BE=D0=B4=D1=82=D0=B2=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=B4=D1=91=D0=BD=D0=BD=D0=BE=D0=B3=D0=BE=20=D0=BE=D0=B1=D1=8A?= =?UTF-8?q?=D0=B5=D0=BA=D1=82=D0=B0=20=D0=BF=D0=BE=D0=BC=D0=B5=D1=89=D0=B0?= =?UTF-8?q?=D0=B5=D1=82=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20=D1=8D?= =?UTF-8?q?=D1=82=D0=BE=D1=82=20=20=20slot=20=D0=B2=20quarantine:=20PUT=20?= =?UTF-8?q?409,=20retention=20=D0=BD=D0=B5=20=D1=83=D0=B4=D0=B0=D0=BB?= =?UTF-8?q?=D1=8F=D0=B5=D1=82=20=D0=B5=D0=B3=D0=BE,=20maintenance=20=20=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D1=8F=D0=B5=D1=82=20=20=20?= =?UTF-8?q?=D1=81=D0=B2=D0=B5=D1=80=D0=BA=D1=83.=20Startup=20=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B4=D0=BE=D0=BB=D0=B6=D0=B0=D0=B5=D1=82=D1=81=D1=8F;?= =?UTF-8?q?=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D1=8F=D0=B5=D1=82=D1=81?= =?UTF-8?q?=D1=8F=20single-use=20=D1=81=D0=B5=D0=BC=D0=B0=D0=BD=D1=82?= =?UTF-8?q?=D0=B8=D0=BA=D0=B0.=20-=20=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D1=8B=20network=20regressions=20=D0=BC=D0=B8=D0=B3?= =?UTF-8?q?=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20=D0=B8=20=D1=80=D0=B0=D1=81?= =?UTF-8?q?=D1=88=D0=B8=D1=80=D0=B5=D0=BD=D1=8B=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=BA=D0=B8=20multipart=20=20=20recovery,=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=82=D0=B5=D1=80=D1=8F=D0=BD=D0=BD=D1=8B=D1=85=20=D0=BE?= =?UTF-8?q?=D1=82=D0=B2=D0=B5=D1=82=D0=BE=D0=B2,=20=D0=BE=D1=82=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D1=8B=20=D0=B7=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=B8=20=D0=BA=D0=BE=D0=BD=D1=84=D0=BB=D0=B8=D0=BA?= =?UTF-8?q?=D1=82=D0=BE=D0=B2=20=20=20destination.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Настройка, команды миграции, recovery и ограничения отката: [docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). Команды, результаты и границы подтверждения, включая qualification реального object store и 10 GiB: [docs/VALIDATION-0.7.9.md](docs/VALIDATION-0.7.9.md). Итог: 265 тестов, нагрузка 500 TLS клиентов и native reference — успешно. Реальные 10 GiB через MinIO (описанный test adapter), 1280parts и полный SHA-256 пройдены; подробности — [VALIDATION-0.7.9](docs/VALIDATION-0.7.9.md). --- CHANGELOG-0.7.8.md | 30 + CHANGELOG-0.7.9.md | 33 + Cargo.lock | 19 +- Cargo.toml | 3 +- README.md | 11 +- docs/COMPATIBILITY.md | 10 +- docs/HTTP-UPLOAD.md | 251 ++++++ docs/SOURCE-SHA256.txt | 21 +- docs/VALIDATION-0.7.7.md | 31 + docs/VALIDATION-0.7.8.md | 33 + docs/VALIDATION-0.7.9.md | 59 ++ docs/VALIDATION.md | 74 +- docs/validation-logs-0.7.8/large.json | 35 + docs/validation-logs-0.7.8/large.txt | 52 ++ docs/validation-logs-0.7.8/multi.txt | 1 + docs/validation-logs-0.7.8/s3.txt | 1 + docs/validation-logs-0.7.8/verify.txt | 711 +++++++++++++++++ docs/validation-logs-0.7.9/migration.txt | 1 + .../minio-provenance.json | 12 + .../minio-sandbox-compat.patch | 21 + docs/validation-logs-0.7.9/minio.json | 51 ++ docs/validation-logs-0.7.9/minio.txt | 52 ++ docs/validation-logs-0.7.9/s3.txt | 1 + docs/validation-logs-0.7.9/verify.txt | 713 ++++++++++++++++++ docs/validation-results-0.7.7.json | 93 +++ docs/validation-results-0.7.8.json | 126 ++++ docs/validation-results.json | 122 ++- scripts/verify-local.sh | 7 + src/http_config.rs | 2 +- src/http_service.rs | 653 +++++++++++++++- src/http_transport.rs | 1 + src/lib.rs | 4 + src/main.rs | 15 +- src/object_store.rs | 605 +++++++++++++++ src/server.rs | 188 ++++- src/upload_migration.rs | 565 ++++++++++++++ tests/upload_large_network.py | 276 +++++++ tests/upload_migration_network.py | 263 +++++++ tests/upload_minio_network.py | 273 +++++++ tests/upload_multi_network.py | 154 ++++ tests/upload_s3_network.py | 515 +++++++++++++ 41 files changed, 5936 insertions(+), 152 deletions(-) create mode 100644 CHANGELOG-0.7.8.md create mode 100644 CHANGELOG-0.7.9.md create mode 100644 docs/HTTP-UPLOAD.md create mode 100644 docs/VALIDATION-0.7.7.md create mode 100644 docs/VALIDATION-0.7.8.md create mode 100644 docs/VALIDATION-0.7.9.md create mode 100644 docs/validation-logs-0.7.8/large.json create mode 100644 docs/validation-logs-0.7.8/large.txt create mode 100644 docs/validation-logs-0.7.8/multi.txt create mode 100644 docs/validation-logs-0.7.8/s3.txt create mode 100644 docs/validation-logs-0.7.8/verify.txt create mode 100644 docs/validation-logs-0.7.9/migration.txt create mode 100644 docs/validation-logs-0.7.9/minio-provenance.json create mode 100644 docs/validation-logs-0.7.9/minio-sandbox-compat.patch create mode 100644 docs/validation-logs-0.7.9/minio.json create mode 100644 docs/validation-logs-0.7.9/minio.txt create mode 100644 docs/validation-logs-0.7.9/s3.txt create mode 100644 docs/validation-logs-0.7.9/verify.txt create mode 100644 docs/validation-results-0.7.7.json create mode 100644 docs/validation-results-0.7.8.json create mode 100644 src/object_store.rs create mode 100644 src/upload_migration.rs create mode 100644 tests/upload_large_network.py create mode 100644 tests/upload_migration_network.py create mode 100644 tests/upload_minio_network.py create mode 100644 tests/upload_multi_network.py create mode 100644 tests/upload_s3_network.py diff --git a/CHANGELOG-0.7.8.md b/CHANGELOG-0.7.8.md new file mode 100644 index 0000000..1db6932 --- /dev/null +++ b/CHANGELOG-0.7.8.md @@ -0,0 +1,30 @@ +# 0.7.8 + +- Несколько XEP-0363 `http_file_share` components на общих HTTP listeners: + маршрутизация по Host и mount, отдельные discovery/IQ slots, квоты и tokens. +- Разные HTTP hosts могут использовать одинаковые mounts; один host — разные + непересекающиеся paths. Неоднозначные одинаковые/вложенные mounts отвергаются + при startup с именами components и конфликтующими путями. +- Upload dispatcher компонуется с WebSocket, BOSH и `http_altconnect` без + объединения конфликтующих fallback routers. +- Сохранён local backend; добавлен S3-compatible gateway с SigV4, path-style + addressing, HTTPS и loopback HTTP для изолированных тестов. Credentials + можно получать через поля `*_env`. +- Multipart передача частями 8 MiB, максимум восемь активных S3 uploads + на component, + проверка stream size, single-use PUT, abort при ошибке/отмене и recovery + сохранённых multipart records перед открытием listeners. +- Native XMPP metadata остаются локальными; backend identity guard запрещает + скрытую смену хранилища при существующих slots. Rotation S3 credentials + не меняет identity backend. +- Добавлены network regressions для нескольких components и независимый + локальный S3 protocol fixture с повторной проверкой SigV4. Fixture не + подтверждает совместимость с production AWS S3 или MinIO. + +Настройка, lifecycle `AbortIncompleteMultipartUpload`, recovery и ограничения: +[docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). + +Реальная передача 10 GiB по HTTPS с local backend проверена: полный PUT/GET, +совпадение SHA-256, Range за 4 GiB и выборочное измерение памяти. Полная +регрессия, нагрузка 500 TLS клиентов и native reference tests пройдены. +Результаты: [docs/VALIDATION-0.7.8.md](docs/VALIDATION-0.7.8.md). diff --git a/CHANGELOG-0.7.9.md b/CHANGELOG-0.7.9.md new file mode 100644 index 0000000..1b6965d --- /dev/null +++ b/CHANGELOG-0.7.9.md @@ -0,0 +1,33 @@ +# 0.7.9 + +- Offline CLI `upload-migrate OLD_CONFIG NEW_CONFIG --component HOST + [--execute]`: dry run по умолчанию, local ↔ S3 и S3 ↔ S3 без удаления source. +- Последовательное копирование с проверкой размера и полным SHA-256 readback + destination; конфликтующий объект не перезаписывается. Временный диск — + максимум один переносимый файл дополнительно к постоянному destination. +- Durable migration journal позволяет продолжить прерванную операцию с теми + же конфигурациями. Незавершённый или повреждённый journal блокирует startup; + backend identity меняется только после проверки всех файлов. +- Native `uploads`/`upload_stats`, slots и JWT сохраняются; поддерживаются + native локальные файлы без Rust completion sidecar. Миграция требует + остановки RustXMPP и Prosody, использующих общий `data_path`. +- Multipart intent сохраняется до создания удалённого upload. Recovery + находит потерянный UploadId через paginated listing и очищает только точный + key принадлежащего slot. При сохранных metadata cleanup не зависит от + lifecycle; lifecycle остаётся дополнительной защитой, а не гарантией. +- Перед completion сохраняется durable marker. Неопределённый результат + completion при отсутствии подтверждённого объекта помещает только этот + slot в quarantine: PUT 409, retention не удаляет его, maintenance повторяет + сверку. Startup продолжается; сохраняется single-use семантика. +- Добавлены network regressions миграции и расширены проверки multipart + recovery, потерянных ответов, отмены запросов и конфликтов destination. + +Настройка, команды миграции, recovery и ограничения отката: +[docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). + +Команды, результаты и границы подтверждения, включая qualification реального +object store и 10 GiB: [docs/VALIDATION-0.7.9.md](docs/VALIDATION-0.7.9.md). + +Итог: 265 тестов, нагрузка 500 TLS клиентов и native reference — успешно. +Реальные 10 GiB через MinIO (описанный test adapter), 1280parts и полный SHA-256 +пройдены; подробности — [VALIDATION-0.7.9](docs/VALIDATION-0.7.9.md). diff --git a/Cargo.lock b/Cargo.lock index 73faf58..b4a848f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -563,6 +563,7 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] @@ -1110,7 +1111,7 @@ dependencies = [ [[package]] name = "rust-xmpp" -version = "0.7.7" +version = "0.7.9" dependencies = [ "anyhow", "axum", @@ -1122,6 +1123,7 @@ dependencies = [ "hickory-resolver", "hmac", "http-body-util", + "hyper", "hyper-util", "mlua", "pbkdf2", @@ -1719,6 +1721,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.29.0" @@ -1821,6 +1829,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 3a8f4b0..20e943e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rust-xmpp" -version = "0.7.7" +version = "0.7.9" edition = "2024" description = "Experimental Rust XMPP server with Prosody-compatible storage" license = "MIT" @@ -34,6 +34,7 @@ axum = { version = "0.8", features = ["ws"] } futures-util = "0.3" tokio-util = { version = "0.7", features = ["io"] } http-body-util = "0.1" +hyper = { version = "1", features = ["client", "http1"] } hyper-util = { version = "0.1", features = ["server-auto", "tokio", "service"] } socket2 = "0.6" diff --git a/README.md b/README.md index e961aee..f54ed3c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Rust XMPP 0.7.7 +# Rust XMPP 0.7.9 Сервер на Rust, разрабатываемый для замены **Prosody 13.0 nightly 100, cafd74b9c5ea** с предоставленными модулями. Читает существующий Lua-конфиг, @@ -197,3 +197,12 @@ Cached SASL2 Bind2/SM requests допускают legacy fallback, XEP-0258/0077 объявляются для Gajim; policy/classifications схема может использоваться как каталог labels. Дубликаты одинаковых log destinations устранены. Подробности — [CHANGELOG-0.7.5.md](CHANGELOG-0.7.5.md). + +## HTTP Upload в 0.7.9 + +Несколько upload components с разными HTTP hosts или mounts и S3-compatible +gateway с multipart streaming описаны в [docs/HTTP-UPLOAD.md](docs/HTTP-UPLOAD.md). +Добавлены offline миграция с SHA-256/resume, точечная очистка multipart и +карантин неопределённого completion. Реальные 10 GiB через MinIO проверены с +описанной адаптацией тестовой среды. Изменения и границы подтверждения — +[CHANGELOG-0.7.9.md](CHANGELOG-0.7.9.md). diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index e80cc63..06ee2ac 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -8,7 +8,7 @@ | Область | Что реализовано | Существенные пределы | |---|---|---| | Конфигурация | Lua/JSON/YAML/YML/TOML, Include, глобальные и локальные области, наследование модулей | Lua VM ограничена; не все функции окружения конфигуратора Prosody доступны. Runtime обслуживает один VirtualHost и его компоненты | -| Хранилище | internal `.dat`, `.list`, бинарные строки, смешанные ключи Lua, атомарные записи, потоковые архивы, инвалидирование `.lidx` | SQL, external storage и произвольные драйверы не реализованы. Каталог блокируется между процессами Rust; Prosody в блокировке не участвует | +| Хранилище | internal `.dat`, `.list`, бинарные строки, смешанные ключи Lua, атомарные записи, потоковые архивы, инвалидирование `.lidx` | SQL, external storage остальных модулей и произвольные драйверы не реализованы; S3 uploads поддержаны. Каталог блокируется между процессами Rust; Prosody в блокировке не участвует | | Аккаунты, saslauth | internal_hashed SHA-1/SHA-256, SASLprep, PLAIN внутри TLS, SCRAM и PLUS с tls-exporter/tls-server-end-point, SSDP, SASL2 | Только internal_hashed; опциональный sasl2_bind2 (alias bind2) поддерживает inline bind/SM/CSI/Carbons. SASL2 inline SM resume и takeover живого транспорта реализованы; MAM metadata при Bind2 читает весь архив синхронно | | Сообщения и presence | локальная маршрутизация, приоритеты ресурсов, контакты/подписки/probes, offline, carbons, блокировки | Полная совместимость всех пограничных сценариев маршрутизации ещё не подтверждена | | MAM | личный и комнатный MAM2, prefs, формы, RSM, фильтры времени/контакта/ID, extended queries, metadata, stanza-id, expiry | Нет доказательства совпадения каждой ошибки и порядка результатов при всех комбинациях запроса | @@ -18,7 +18,7 @@ | SM/push | acknowledgements, resumable sessions, replay, ограниченные очереди, native регистрации push и ответы сервисов | Есть coalescing и watchdog первого push; полный цикл с настоящим APNs/FCM шлюзом не проверен. Состояние живых сессий не переживает перезапуск процесса | | CSI | active/inactive, ограниченный буфер, native importance rules, delay, flush при клиентской активности и grace period | Событие socket drain приближено завершением записи; необязательные расширенные XPath-фильтры поддержаны не полностью | | Федерация | входящие/исходящие STARTTLS, CA/DNS-ID, SASL EXTERNAL, DNS SRV, проверка домена каждого stanza, ограниченный пул соединений | Нет dialback/DANE/OtherName-only/SRV-ID/bidi/direct TLS | -| HTTP upload | слоты XEP-0363, streaming PUT, GET/HEAD/range/conditional, лимиты/квоты/expiry, native metadata, JWT slots | Один upload-компонент; внешнее объектное хранилище не реализовано. Фактический upload 10 GiB не прогонялся | +| HTTP upload | Несколько XEP-0363 components, Host/path isolation, local/S3 gateway, multipart/SigV4, streaming PUT, GET/HEAD/range/conditional, квоты/expiry, native metadata/JWT; offline local↔S3/S3↔S3 migration с SHA-256/resume; intent/exact-key recovery и quarantine; реальный 10 GiB PUT/GET через MinIO проверен | MinIO single-node с описанным netlink adapter тестовой среды; AWS/distributed deployment не проверены. Миграция требует остановки сервера и временного места под один файл; source сохраняется. Потерянные ownership metadata и неопределённый Complete требуют отдельного разбора; HTTP-UPLOAD.md | | WebSocket/BOSH | WS/WSS и HTTP/HTTPS, общая XMPP-сессия, origin rules, BOSH RID/retry/hold/reorder/restart; http_altconnect XRD/JRD discovery; http_paths, HTTP Host/fallback, внешний URL, доверенные IP/CIDR и forwarded headers для TLS reverse proxy | Один VirtualHost с компонентами; нет PROXYv1/v2, произвольных Lua HTTP-приложений и горячей перенастройки. См. HTTP-PROXY.md | | admin_shell | протокол admin socket, status/version/uptime, списки, основные операции аккаунтов | Нет произвольного Lua, горячей загрузки модулей и полного набора prosodyctl | | Остальные модули | disco, ping, time, version, uptime, register, TURN extdisco/HMAC, каталог seclabels | Нет встроенного TURN. Coturn работает отдельно. `pep_simple` пуст в предоставленном исходнике | @@ -29,8 +29,10 @@ на противоречащий этому комментарий в исходном файле. - Личная история настроена `never`, комнатная — на неделю. Очистка выполняется периодически; append не переписывает весь архив. -- Лимит отдельного файла — 10 GiB, общий — 100 GiB. Сетевой тест проверяет - потоковую загрузку меньшего файла; это не проверка фактической загрузки 10 GiB. +- Лимит отдельного файла — 10 GiB, общий — 100 GiB. В 0.7.8 фактические + HTTPS PUT/GET 10 GiB с local backend проверены с совпадением SHA-256, + Range за 4 GiB и выборочным RSS. В 0.7.9 дополнительно проверены 10 GiB через + MinIO single-node с описанным adapter тестовой среды; см. VALIDATION-0.7.9.md. - Структура `security_labels.default_policy/policies/classifications` отличается от native `mod_seclabels`. С0.7.1 её точная schema может использоваться как catalog metadata: выбранный default_policy и его classifications. ACL и diff --git a/docs/HTTP-UPLOAD.md b/docs/HTTP-UPLOAD.md new file mode 100644 index 0000000..b70d200 --- /dev/null +++ b/docs/HTTP-UPLOAD.md @@ -0,0 +1,251 @@ +# HTTP File Upload (0.7.9) + +XEP-0363 component `http_file_share` выдаёт slots через XMPP и обслуживает +HTTP PUT/GET/HEAD. Доступны локальные файлы и S3-compatible object storage. +В S3-режиме RustXMPP работает как gateway: клиент получает URL component, +а сервер подписывает запросы к bucket по AWS Signature Version 4. Это не +выдача клиенту S3 credentials или presigned bucket URLs. + +## Несколько components + +Каждый component сохраняет собственные XMPP JID, discovery, размер файла, +квоты, slots, JWT secret, backend и HTTP mount. HTTP-маршрут выбирается по +`Host` и `http_paths.file_share`. Можно использовать разные HTTP hosts с +одинаковым mount либо один host с непересекающимися mounts: + +```lua +-- Добавьте к существующей конфигурации; сохраните VirtualHost, TLS и модули. +Component "upload.example.org" "http_file_share" + http_host = "files.example.org"; + http_paths = { file_share = "/personal"; } + http_file_share_backend = "local"; + http_file_share_size_limit = 10737418240; -- 10 GiB + http_file_share_daily_quota = 21474836480; + http_file_share_global_quota = 107374182400; + http_file_share_expires_after = "7d"; + +Component "media.example.org" "http_file_share" + http_host = "files.example.org"; + http_paths = { file_share = "/media"; } + http_file_share_backend = "local"; + http_file_share_size_limit = 104857600; +``` + +Для одного effective HTTP host одинаковые или вложенные mounts отвергаются +при startup с сообщением `ambiguous HTTP upload components`, содержащим +имена обоих components и пути. Например, `/files` конфликтует с `/files` +и `/files/private`; `/files` и `/files-other` не конфликтуют. Mount `/` +пересекается со всеми mounts этого host. Hosts сравниваются без учёта +регистра; порт в HTTP Host не выбирает отдельный component. + +WebSocket, BOSH и `http_altconnect` используют общие HTTP listeners вместе +с uploads. Их маршруты обрабатываются соответствующим middleware. Выделяйте +uploads собственные mounts, чтобы действующие transport/discovery endpoints +не перекрывали публичные адреса загрузки. + +`http_external_url` задаёт публичную базу, сохраняя внутренний mount. +`http_file_share_base_url` явно переопределяет полный публичный upload prefix. +При reverse proxy настройте переписывание пути и передавайте правильный Host: +одного изменения рекламируемого URL недостаточно для настройки proxy. +Пример конфигурации proxy — [HTTP-PROXY.md](HTTP-PROXY.md). + +## S3-compatible backend + +```lua +Component "upload-s3.example.org" "http_file_share" + http_host = "files.example.org"; + http_paths = { file_share = "/objects"; } + http_external_url = "https://files.example.org"; + http_file_share_backend = "s3"; + http_file_share_size_limit = 10737418240; + http_file_share_daily_quota = 21474836480; + http_file_share_global_quota = 107374182400; + http_file_share_s3 = { + endpoint = "https://objects.example.org"; + bucket = "xmpp-uploads"; + region = "us-east-1"; + access_key_env = "RUSTXMPP_S3_ACCESS_KEY"; + secret_key_env = "RUSTXMPP_S3_SECRET_KEY"; + -- Только для временных credentials: + -- session_token_env = "RUSTXMPP_S3_SESSION_TOKEN"; + key_prefix = "upload-s3"; + } +``` + +Переменные должны присутствовать в окружении процесса сервера при startup, +например через защищённый EnvironmentFile службы. Их отсутствие вызывает +ошибку startup. Доступны и строковые поля `access_key`, `secret_key`, +`session_token`; соответствующее поле `*_env` имеет приоритет. Credentials +не включаются в сохранённую backend identity. + +Endpoint использует path-style bucket addressing. HTTPS требуется для +нелокальных S3 endpoints. HTTP допускается только для `localhost` или +loopback IP, например `http://127.0.0.1:9000` в изолированном тесте. +Production endpoint должен иметь сертификат, доверенный системным root CAs. +Не путайте HTTPS object-store endpoint с публичным HTTPS upload URL: +оба соединения настраиваются независимо. Для публичных клиентов используйте +HTTPS listener либо существующий TLS reverse proxy с HTTPS advertised URL. + +Bucket должен разрешать создание multipart upload, загрузку частей, завершение, +abort, `ListMultipartUploads` (`s3:ListBucketMultipartUploads`), HEAD/GET +и удаление объектов для retention. +Перед CreateMultipartUpload сервер сохраняет intent для принадлежащего ему +slot. Если ответ с UploadId потерян, recovery перечисляет multipart uploads +с pagination и прерывает только uploads с полным совпадением object key; +соседние keys с тем же prefix не затрагиваются. Сохранных native metadata, +подтверждающих владение slot, достаточно и для legacy records без intent или +UploadId. Cleanup такого slot не зависит от bucket lifecycle. + +Lifecycle `AbortIncompleteMultipartUpload` рекомендуется как дополнительная +защита, например при утрате metadata или внешних orphan uploads. Он не даёт +абсолютной гарантии очистки; scope, сроки и выполнение правил зависят от +object store. Не запускайте внешних writers с теми же slot keys: cleanup +предполагает исключительное владение ими. + +Gateway передаёт файл частями по 8 MiB и ограничивает число одновременно +активных S3 uploads восемью на component. Файл целиком не буферизуется в памяти и не +сохраняется во временный локальный файл. Служебная память, TLS/HTTP buffers +и буфер текущего входящего frame добавляются к памяти частей; ограничение +не является гарантией точного общего RSS. Ограничение размера применяется +и к chunked PUT, а квоты учитывают зарезервированные slots component. + +## Metadata, recovery и смена credentials + +XMPP upload metadata остаются в native хранилище `uploads` и `upload_stats` +в `data_path`. S3 заменяет хранение содержимого файлов, а не XMPP metadata. +Сохраняйте `data_path` вместе с bucket: один bucket не восстанавливает +владельцев, quotas и slots. В `rust_uploads.json` сохраняются служебное +состояние slots, completion, multipart intent и незавершённые UploadId; +файл не заменяет native архив. `rust_upload_backend.json` фиксирует backend identity. + +Перед открытием listeners сервер восстанавливает незавершённые S3 records: +прерывает принадлежащие slot multipart uploads, затем проверяет удалённый +объект и размер и учитывает уже завершённый commit. Ошибка обязательного +recovery, кроме quarantine completion, останавливает startup. Перед +CompleteMultipartUpload сохраняется `completion_attempted`. Если результат +completion неизвестен и после cleanup +HEAD ещё возвращает 404, только этот slot остаётся в quarantine: PUT возвращает +409, retention его не удаляет, остальные slots и startup продолжают работу. +Maintenance каждые 60 секунд повторяет сверку; подтверждённый объект +ожидаемого размера снимает quarantine и завершает recovery. Неопределённый +результат требует сверки оператором с object store, если объект так и не +появился. Корректный явный S3 completion error допускает повтор после +reconciliation; network/timeout или некорректный ответ не разрешают +перезапись возможного позднего commit. Ошибки обязательных listing/HEAD +по-прежнему останавливают startup. +PUT остаётся single-use; повторная запись завершённого slot возвращает conflict. + +При наличии slots нельзя незаметно переключить local/S3 backend, endpoint, +bucket или prefix: startup отвергает изменение backend identity. Добавление +S3 к существующим local slots также требует явной миграции через offline CLI, +описанный ниже; не удаляйте identity-файл для обхода проверки. + +Для rotation S3 credentials обновите переменные окружения/секреты службы +и перезапустите сервер, сохранив endpoint, bucket и prefix. Новая identity +credentials не содержит, поэтому их rotation не считается миграцией. +Сначала обеспечьте новым credentials доступ к существующим объектам и +multipart cleanup. Rotation `http_file_share_secret` — отдельная операция: +смена JWT signing secret делает ранее выданные PUT tokens недействительными. + +## Offline миграция local ↔ S3 и S3 ↔ S3 + +Остановите RustXMPP и Prosody, использующие этот `data_path`, и сохраните +резервную копию metadata и содержимого backend. Подготовьте две полные +конфигурации: `current.cfg.lua` описывает действующий source, `new.cfg.lua` — +destination. Обе должны содержать тот же VirtualHost, upload component, +`data_path`, HTTP/HTTPS ports, публичные маршруты и `http_file_share_secret`; +меняйте только параметры backend. Credentials обоих S3 хранилищ должны быть +доступны CLI; используйте отдельные переменные окружения для source и +destination. Для каждого component выполняется отдельная команда: + +```sh +# План: без копирования объектов, изменения journal или backend identity. +rust-xmpp upload-migrate current.cfg.lua new.cfg.lua --component upload.example.org + +# Копирование и проверка; при остановке повторите эту же команду. +rust-xmpp upload-migrate current.cfg.lua new.cfg.lua --component upload.example.org --execute + +# После сообщения Migration committed запускайте сервер с новой конфигурацией. +rust-xmpp serve new.cfg.lua --experimental +``` + +Команда захватывает exclusive lock `data_path`, исключающий параллельный +RustXMPP или другую миграцию; Prosody этот lock не использует. Source и +destination должны иметь разные backend identities. Поддерживаются local → +S3, S3 → local и S3 → другой endpoint, bucket или prefix. Перед миграцией +восстановите незавершённые uploads исходным сервером. Dry run проверяет +конфигурации и metadata и определяет объекты для копирования; полный SHA-256 +рассчитывается при `--execute`. + +Копируются завершённые файлы, включая native файлы без Rust completion +sidecar и файлы, опубликованные перед сбоем записи completion cache. +Отсутствующие незавершённые reservations остаются в metadata. Native +`uploads`, `upload_stats`, владельцы, slots и JWT не переписываются. +Source objects никогда не удаляются. Для каждого файла CLI проверяет размер, +считает SHA-256 source и повторно читает destination для сверки; ETag не +используется как checksum. Существующий destination с другим содержимым +вызывает отказ без перезаписи. + +Миграция обрабатывает файлы последовательно через приватный временный файл: +нужно свободное место размером максимум в один переносимый файл дополнительно +к постоянному local destination. Память ограничена buffers частей, файл +целиком в RAM не загружается. После сбоя временный spool удаляется и текущий +файл читается заново; это возобновление по объектам, не по byte offset. + +`rust_upload_migration.json` хранит identities, fingerprint metadata, +checksums и прогресс. Повторный запуск с теми же конфигурациями сверяет source +и уже скопированные файлы, восстанавливает multipart cleanup и продолжает +работу. Backend identity переключается только после проверки всех объектов. +Незавершённый или повреждённый journal блокирует startup до offline recovery, +даже если component убран из новой конфигурации. Не удаляйте journal для +обхода барьера и не редактируйте metadata во время миграции. + +Сохранённый source позволяет сверить данные и подготовить откат, но простая +подмена конфигурации после commit отвергается identity guard. Пока migration +не завершена, продолжите её с теми же параметрами. После commit выполняйте +обратную offline миграцию с текущей конфигурацией как source и прежней как +destination. Новые uploads могли появиться только в destination; откат к +старой резервной копии без их переноса потеряет эти изменения. Конфликтующие +объекты обратная миграция также не перезаписывает. Удаление прежнего source +после проверки результата — отдельное решение оператора. + +## Проверки и границы подтверждения + +`tests/upload_multi_network.py` проверяет настоящий binary через TLS XMPP +и HTTP: отдельные/shared hosts, discovery и IQ slots, квоты, изоляцию JWT, +PUT/GET и restart, а также композицию с WS/BOSH/host-meta и отказ при +неоднозначных mounts. + +`tests/upload_large_network.py` выполняет opt-in передачу полного файла по +проверяемому HTTPS с local backend и записывает измерения в JSON. Для default +10 GiB необходимо более 10 GiB свободного места. Это отдельная проверка от S3 +fixture, и её результат не подтверждает передачу 10 GiB в object store. + +`tests/upload_s3_network.py` использует независимый локальный HTTP fixture, +который повторно вычисляет SigV4 и реализует S3 protocol для проверки gateway, +multipart, чтения и обработки ошибок. Это не qualification на AWS S3, +MinIO или другом production object store. Их совместимость требует отдельного +запуска в целевой среде. + +`tests/upload_migration_network.py` проверяет offline миграцию с независимым +SigV4 fixture: local ↔ S3, S3 ↔ S3, dry run, прерывание/возобновление, +checksums, сохранность source и native metadata, блокировку startup. + +`tests/upload_minio_network.py` — opt-in запуск с настоящим MinIO executable. +По умолчанию он передаёт 10 GiB через HTTPS gateway в MinIO, проверяет полный +GET и SHA-256, multipart, Range, restart и crash cleanup. Малый `--size` +проверяет smoke path, но не подтверждает передачу большого объекта. + +Итоговые команды, результаты и измерения release validation публикуются в +[release validation report](VALIDATION-0.7.9.md). Предыдущий результат 10 GiB +с local backend, SHA-256, Range за 4 GiB и выборочным RSS опубликован в +[отчёте 0.7.8](VALIDATION-0.7.8.md); сам по себе он не подтверждает S3 backend. + + +В 0.7.9 фактические 10 GiB HTTPS PUT/GET через MinIO успешно пройдены: 1280 +multipart parts, SHA-256 совпал, Range/restart/crash cleanup проверены. +MinIO собран из upstream release с test-only fallback перечисления интерфейсов +на loopback при EPERM; S3/auth/storage code не менялся. Точный diff и hashes +в VALIDATION-0.7.9.md. Немодифицированный MinIO/AWS/distributed deployment этим +не сертифицируются. MINIO_BINARY перед verify-local.sh включает opt-in 10 GiB +прогон; для обычной среды используйте собственный MinIO executable. diff --git a/docs/SOURCE-SHA256.txt b/docs/SOURCE-SHA256.txt index b1f2f8d..2257c61 100644 --- a/docs/SOURCE-SHA256.txt +++ b/docs/SOURCE-SHA256.txt @@ -14,7 +14,7 @@ b1018e1b279c89f6cf14b4017e4f1b14e9165afef2f668cec6ac8bbe0c710d85 scripts/integr e08ac6136cce27cda2ddbd510da0f25750766ab0150a9f4b8f87c953276d16e9 scripts/prosody_compat/oracle.lua 2c7a2c946885205e9df38b97305073a8ac72f58ac8ff84fc6f9af320c35bbc96 scripts/prosody_compat/verify.sh 1b9ed063daf3e0320819b4fe0135c96dd27e49877044aa2f5093b8ab4fbda41d scripts/smoke.py -4d9702d73fab7c20f1e5b9798c0f53c6e1e77aab15074c1c8118df9ae5abd795 scripts/verify-local.sh +0b2f6209a36fe9783b83d43396da26862c89cad1da1483ae49a79800e31982c1 scripts/verify-local.sh a2e47f547ab9349f84ee6c4f8e51c147b7110b41a82c17cdb938027a84d82ef3 src/account_modules.rs 5637f816586f0613b83e14d0ed478bd539f325b70ccbfc9fdf893adbe494453d src/admin.rs 89d7e29d8f8daab614c9c836864b166fdaa324ca035c2d3d5f95639c24b2a9e1 src/archive.rs @@ -25,28 +25,30 @@ e15b079a420227ee59e3b7ac305783c51eb59d76a8fb5dd12760547b0f812107 src/auth.rs 1c8aa5cb816a32a0f9f24589ac00beba27544a83c3d9aad771523a1d1cbcd4c8 src/extras.rs dca8959effb123b94e1fbd42c561a79c721178c4b612a004db9f9fccb3b8950d src/federation.rs 46a59bc6689f0931670547e26b3dc4beae3aa210eee1aae832070925b377499c src/http_altconnect.rs -43ab37be4b6ad41e51c3918c8806d91650c1ee2b62977caade639920abf7aa44 src/http_config.rs +d561a6bd4db079cfb1ef3bb7ae75f23e6080c5d5705583774f6c5f1ca6ed4420 src/http_config.rs 3fa73ae129fe552bd7de7afca0617a1c18bf4d3b36a39023bfec036bca17ffc5 src/http_proxy.rs -9366d1aaa7e13d06cf7547cfb1e10461091ba876413927e0ad49df41c3661e6f src/http_service.rs -c4fd5046b55510f87a1c87e2b293500f0a01b76f22dacfa6d145e857e857fc09 src/http_transport.rs -9f27d509f7e58467c92e1f7ac9175deeb7347c61a45664bb35d17ed4ad0d0d0e src/lib.rs +f4f87f5c20cb0602d803e86a22c209756c3b1f22c532743ec7e0dbf5bb174a0e src/http_service.rs +c604a0a764ffac6bae0c9aa15c9ef7022d35cb1f8c9159eb66257114445621c5 src/http_transport.rs +c5b7bcc0d08e48fd3805d6571428bb83b7fdc6f6e23a65bf689f079ad87bc32a src/lib.rs a21e1119ea0271b63935e92eb106ab6076f0035e6d3a06c21a352f152b9b37db src/logging.rs 58ce2a9daa36ac26b39c8f263a16f4fa8cc4e57c91bbbf5b24a6aad3f1a326e0 src/lua.rs -85616b8aa8e6fc7ac9aeaa75f256463ee9c2527a590832b243481720caabbe42 src/main.rs +a264e2765a31c8c096462551aca229530cad356e39e338370d1a0bdd97860680 src/main.rs 71d5d185900b9cde190e10e2f28371cd5042d98b5900b4385d823f724234fced src/mam.rs 0f780d95523b41c4ae242bd4a1fb6b65ff57826dca6d23ac73a1a1459609ec0a src/muc.rs d8263eb28f5cdff362512a6b13217aa8f7574f4a3d54b7bcb6ba7930f8c353d8 src/muc_history.rs 4c151ffcdc36e4a3e680f859d4dc7ba16b0162d4f1fe6fb470df4c5e21197e5a src/muc_recovery.rs +3252884252158e1c4bd58922b97733f434c7d28baf059e63267632a3290f660e src/object_store.rs 7d7114a22ef3bf02f9caaf578f0a50d4f75b4dcd4b335b4cb8425d9716c97e71 src/pep_diagnostics.rs f09a6479b76c9c601a169b5f1f9cf1364e82df5ae27e4bab8d1b1428b5cad457 src/preauth.rs a7333b31a57780c57f2d30620cdfaeb7b1215808865960bb7e5b595d3ad98f7c src/protocol.rs 761e29d6431fac5c2a4a24bcfca1f393949a5c55585fd6b0dd7a4f6e9f94a10d src/pubsub.rs 913cd033f60a68f6c7cf9b3adbaf76db1c5c9bc88a0599196d373b16b64c0a1e src/push.rs bec477f2520af6a9d7ca54e43eb12158b53e3cf09057653e504dfdeab956885e src/sasl.rs -46b532fa98cfe8d5d6878ea7bdcc535132a678906caf5f84e7e8feb726048a25 src/server.rs +10f24a2e8e776c7aa2475913e15b01b0f4e57f7a9de92660bafeddd912101d26 src/server.rs 894f11389bfc3fddc3508396d106146261376999f7adcef5a049458f6b4028cf src/sessions.rs 71d7ce1036a091d8c27016ea7ce4cffedf904dfca9d1cc4891b7dc66a1e06a4e src/storage.rs 462b00edb2c1e02dd34524bafe4f82b5ef902028b44310ba31b085d31f4dc2aa src/tls_binding.rs +a78592aa79d8f01c22af1457d0c3f5922863ccd3e8ebb1aa05e043cc44579b8a src/upload_migration.rs f9b023565711e0217238b53fb678c5fb14849cdcd2efcd0fce249018274fad69 src/wire.rs b4eb7a230570dfde0fd0d301a6ace36559a5eae3e1402e42c76f014f61739c24 tests/_wire_client.py 45613ad6c53087a50c6c3a7969b30783578c334499bc6baceadb2d18b0b3ea4e tests/admin_socket.rs @@ -112,4 +114,9 @@ d8882b8aabd6ff603c81fbbc259eefa68772afb2b3eec2e003ac29efd65d146b tests/sasl_web 5c38afd5f77b5774552d12c461902ffaae21de11735560c6f0088e3dda8afa58 tests/storage_compat.rs 8724991fdf22471952cf1a0fac863abe837069c93d3c8d2fa751af4f6bbc3e38 tests/tls_binding.rs 5bea9b5546d3a6ae4fe78d17acbc3f6e6a1d5e1d13b32f2aa88d0e320892b7c5 tests/tls_plus_network.rs +720abdef4c9af261b0e6ac7becb0f9a73e879ba13daceabb14cf8e91be2e0e7d tests/upload_large_network.py +ba7811c8d8a22692059b2224c563023553aac63d275a56a93716fd94075bfa2f tests/upload_migration_network.py +775e14f8f08fa8a419941f7795379cc555bda79ae07373a713fab8f371d21985 tests/upload_minio_network.py +c00cd2b123e8299e5e71e55a776dc7f8ac6403dd749eab11bfcb18d41fa80934 tests/upload_multi_network.py +8ec91d74a0aaaf654b70e873e5c7f7fd28cd60131de6226aae7b0de83d917a8e tests/upload_s3_network.py e454204af26d5a356cb6aa838c3b46f4d13c552b5bd36d6b01bcd6f604821ff2 tests/wss_interop.py diff --git a/docs/VALIDATION-0.7.7.md b/docs/VALIDATION-0.7.7.md new file mode 100644 index 0000000..11cea68 --- /dev/null +++ b/docs/VALIDATION-0.7.7.md @@ -0,0 +1,31 @@ +# Проверка0.7.7 + +Linux x86_64, Rust1.99.1,2026-10-20. Итоговый лог: +validation-logs-0.7.7/verify.txt; структурированный итог — validation-results.json. + +| Проверка | Результат | +|---|---| +| cargo test --locked | 262 passed,0 failed,11 ignored | +| fmt / Clippy all-targets -D warnings / release | Успешно | +| Новый module unit | Link порядок/schemes, загруженные/disabled модули, flags, наследование custom paths/public prefix, неправильные типы и path collisions | +| http_altconnect wire | 10 изолированных конфигураций: настоящий HTTP и CA-verified HTTPS, XRD/JRD parity, MIME, CORS, HEAD/OPTIONS/405, query/JSONP, Host, custom mount/public prefix, HTTPS preference, HTTP fallback | +| Транспортная регрессия | WSS framing работает до и после отключения discovery; старые WS/BOSH/proxy paths/upload сценарии прошли | +| Остальные регрессии | SASL/PLUS/SSDP/Bind2/resume, PEP/push/preauth, logging/resource/CSI/MAM/MUC/offline, MUC restart — успешно | +| Native total | 8 module/storage tests +1 TLS/SASL EXTERNAL federation — успешно | +| Нагрузка | 100 аккаунтов,500 TLS клиентов,5000 ping,5000 сообщений,10000 MAM; failures=[] | + +Итоговый прогон выполнен после исправления проверки Host с портом вне диапазона, +которую обнаружил новый сетевой тест. Все тесты запускают отдельные loopback +listeners и временные данные. После прогона менялись только отчёты и упаковка. +Native reference suite прежних модулей пройдена; непосредственно http_altconnect +проверен Rust unit/network и сопоставлением с supplied Lua source, без отдельного +native differential oracle этого модуля. + +Команда: XMPP_LOAD_CHECK=1 bash scripts/verify-local.sh с native reference env +согласно scripts/prosody_compat/README.md. Точный скрипт включает новый wire test. + +Рабочие конфиги/DNS/proxy/certificates production не изменялись; оператор должен +добавить http_altconnect и опубликовать HTTPS well-known. Инструкция: +HTTP-ALTCONNECT.md. Native HTTP/WS fallback сохранён; для строгого XEP0156 нужны +HTTPS/WSS URLs. Настоящие Converse/Monal/Conversations UI и полный production +rollout не проверены. Общие ограничения сервера — COMPATIBILITY.md. diff --git a/docs/VALIDATION-0.7.8.md b/docs/VALIDATION-0.7.8.md new file mode 100644 index 0000000..253ed1e --- /dev/null +++ b/docs/VALIDATION-0.7.8.md @@ -0,0 +1,33 @@ +# Проверка 0.7.8 + +Linux x86_64, Rust 1.99.1, 2026-10-20. Полный лог: validation-logs-0.7.8/verify.txt. +Структурированный отчёт: validation-results.json; large.json содержит измерения реальной передачи. + +| Проверка | Результат | +|---|---| +| cargo test --locked | 262 passed, 0 failed, 11 ignored | +| fmt / Clippy all-targets -D warnings / release | Успешно | +| Несколько upload components | Hosts/paths, discovery, quota/JWT/object isolation, restart, WS/BOSH/host-meta composition, ambiguous mount rejection — успешно | +| S3 wire | Независимая проверка SigV4, multipart, ошибки/abort/retry, ambiguous completion, SIGKILL и startup recovery, range/conditional/expiry — успешно | +| Реальный HTTPS local upload | 10737418240 байт PUT и полный GET, SHA-256 совпал; реально выделены дисковые блоки | +| Большой файл | PUT 14.39 с; GET 69.22 с; Range за 4 GiB, HEAD, reuse409, quota — успешно | +| Выборочный RSS каждые 50 мс | Сервер 17.24 MiB; клиент 20.19 MiB | +| Native reference | 8 module/storage + 1 TLS/SASL EXTERNAL federation — успешно | +| Нагрузка | 100 аккаунтов, 500 TLS клиентов, 5000 ping, 5000 сообщений, 10000 MAM; failures=[] | + +Команда полного прогона: XMPP_LOAD_CHECK=1 bash scripts/verify-local.sh с native +reference env согласно scripts/prosody_compat/README.md. +Большой файл отдельно: python3 tests/upload_large_network.py --binary target/release/rust-xmpp. +Его 10 GiB временные данные удалены после проверки. Ошибка PID namespace в первом +измерителе RSS исправлена; итоговый прогон выполнен повторно с ненулевыми измерениями. +RSS — выборочный максимум, а не гарантия точного пика при любой нагрузке. + +S3 проверен на независимом локальном HTTP protocol fixture; AWS/MinIO и реальная +передача 10 GiB в object store не проверены. Сервер использует multipart 8 MiB, +до восьми активных uploads на component. Для остаточных multipart sessions в +окне аварии до сохранения UploadId требуется bucket lifecycle +AbortIncompleteMultipartUpload. Настройка и ограничения: HTTP-UPLOAD.md. + +Полная совместимость с Prosody и работа настоящих клиентов в production этим +прогоном не заявляются. Остальные ограничения: COMPATIBILITY.md. После финального +прогона исходники сервера не изменялись; менялись только отчёты и упаковка. diff --git a/docs/VALIDATION-0.7.9.md b/docs/VALIDATION-0.7.9.md new file mode 100644 index 0000000..25684dc --- /dev/null +++ b/docs/VALIDATION-0.7.9.md @@ -0,0 +1,59 @@ +# Проверка 0.7.9 + +Linux x86_64, Rust 1.99.1, 2026-10-20. Полный итоговый прогон: +validation-logs-0.7.9/verify.txt. Структурированный отчёт: validation-results.json. + +| Проверка | Результат | +|---|---| +| cargo test --locked | 265 passed, 0 failed, 11 ignored | +| fmt / Clippy all-targets -D warnings / release | Успешно | +| Upload migration | local → S3 A → S3 B → local, SHA-256, прежние URL/JWT/HEAD, native metadata/source, dry run, lock, прерывание/resume и startup barrier — успешно | +| S3 recovery | Intent до initiation, отсутствие UploadId, pagination и exact-key cleanup без удаления чужого prefix, ambiguous completion, quarantine и late commit — успешно | +| Конкурентный PUT | Регрессия: захват слота читает свежие completion/quarantine flags атомарно — успешно | +| Реальная передача через MinIO | 10,737,418,240 байт PUT и полный GET; SHA-256 совпал; объект: 1280 multipart parts | +| Время большого файла | PUT 48.453 с; GET 13.710 с | +| Выборочный RSS каждые 50 мс | Rust 101.14 MiB; клиент 23.21 MiB; MinIO 475.84 MiB | +| MinIO дополнительные проверки | Range за 4/5 GiB и хвост, restart/read/reuse409, SIGKILL после сохранённой части и abort при recovery — успешно | +| Native reference | 8 module/storage + 1 TLS/SASL EXTERNAL federation — успешно | +| Нагрузка | 100 аккаунтов, 500 TLS клиентов, 5000 ping,5000 сообщений, 10000 MAM; failures=[] | + +Итоговая сборка SHA-256: 24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c. MinIO report автоматически записывает +тот же hash. Большая передача повторена после последних изменений сервера. +Полный прогон включает исправление гонки устаревшего snapshot при захвате слота. +Отдельные первые прогоны нашли и помогли исправить нестабильный fingerprint +миграции и пропуск немедленного reconciliation под ещё активным claim. +Финальный результат не содержит этих ошибок. RSS — выборочный максимум, а не +гарантия точного пика для любой нагрузки. Временные 10 GiB объекты после проверки удалены. + +## Точная граница MinIO проверки + +Использован настоящий upstream MinIO RELEASE.2025-04-22T22-12-26Z, собранный +Go1.24.3 из официального GitHub archive. Немодифицированная сборка в этой среде +не запускается: net.Interfaces/netlinkrib запрещён с EPERM. Тестовая сборка +добавляет только fallback на 127.0.0.1 для этой ошибки перечисления интерфейсов. +S3, authentication, multipart и storage code не изменены. Это не qualification +немодифицированного MinIO в обычной production среде. + +Точный diff, hashes исходного архива, обеих сборок и adapter: +validation-logs-0.7.9/minio-sandbox-compat.patch и minio-provenance.json. +Полный результат и 1280-part ETag: validation-logs-0.7.9/minio.json. +Проверен single-node loopback с HTTP к MinIO и CA-verified HTTPS к gateway. +AWS account, distributed MinIO и production сеть не проверены. + +## Повторение + +Полный прогон: XMPP_LOAD_CHECK=1 bash scripts/verify-local.sh с native env +согласно scripts/prosody_compat/README.md. В этой среде дополнительно применены +CARGO_INCREMENTAL=0 CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0. +MinIO отдельно: python3 tests/upload_minio_network.py --binary target/release/rust-xmpp +--minio /path/to/minio --report /tmp/minio-report.json. При test adapter передайте +--minio-compat-patch /path/to/patch, чтобы он попал в результат. Можно задать +MINIO_BINARY перед verify-local.sh для opt-in передачи10 GiB; нужны место и MinIO executable. + +Инструкции migration/recovery: HTTP-UPLOAD.md. Source всегда сохраняется; +незавершённая миграция блокирует startup, неопределённый Complete quarantines +только свой slot. Потеря ownership metadata не позволяет безопасно удалить +произвольные чужие multiparts; bucket lifecycle остаётся дополнительной защитой. +Сетевой сбой с HEAD404 не выдаётся за доказанную неудачу удалённого commit. +Полная замена Prosody, настоящие production клиенты и DNSSEC этим не заявляются; +общие границы — COMPATIBILITY.md. diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 11cea68..25684dc 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -1,31 +1,59 @@ -# Проверка0.7.7 +# Проверка 0.7.9 -Linux x86_64, Rust1.99.1,2026-10-20. Итоговый лог: -validation-logs-0.7.7/verify.txt; структурированный итог — validation-results.json. +Linux x86_64, Rust 1.99.1, 2026-10-20. Полный итоговый прогон: +validation-logs-0.7.9/verify.txt. Структурированный отчёт: validation-results.json. | Проверка | Результат | |---|---| -| cargo test --locked | 262 passed,0 failed,11 ignored | +| cargo test --locked | 265 passed, 0 failed, 11 ignored | | fmt / Clippy all-targets -D warnings / release | Успешно | -| Новый module unit | Link порядок/schemes, загруженные/disabled модули, flags, наследование custom paths/public prefix, неправильные типы и path collisions | -| http_altconnect wire | 10 изолированных конфигураций: настоящий HTTP и CA-verified HTTPS, XRD/JRD parity, MIME, CORS, HEAD/OPTIONS/405, query/JSONP, Host, custom mount/public prefix, HTTPS preference, HTTP fallback | -| Транспортная регрессия | WSS framing работает до и после отключения discovery; старые WS/BOSH/proxy paths/upload сценарии прошли | -| Остальные регрессии | SASL/PLUS/SSDP/Bind2/resume, PEP/push/preauth, logging/resource/CSI/MAM/MUC/offline, MUC restart — успешно | -| Native total | 8 module/storage tests +1 TLS/SASL EXTERNAL federation — успешно | -| Нагрузка | 100 аккаунтов,500 TLS клиентов,5000 ping,5000 сообщений,10000 MAM; failures=[] | +| Upload migration | local → S3 A → S3 B → local, SHA-256, прежние URL/JWT/HEAD, native metadata/source, dry run, lock, прерывание/resume и startup barrier — успешно | +| S3 recovery | Intent до initiation, отсутствие UploadId, pagination и exact-key cleanup без удаления чужого prefix, ambiguous completion, quarantine и late commit — успешно | +| Конкурентный PUT | Регрессия: захват слота читает свежие completion/quarantine flags атомарно — успешно | +| Реальная передача через MinIO | 10,737,418,240 байт PUT и полный GET; SHA-256 совпал; объект: 1280 multipart parts | +| Время большого файла | PUT 48.453 с; GET 13.710 с | +| Выборочный RSS каждые 50 мс | Rust 101.14 MiB; клиент 23.21 MiB; MinIO 475.84 MiB | +| MinIO дополнительные проверки | Range за 4/5 GiB и хвост, restart/read/reuse409, SIGKILL после сохранённой части и abort при recovery — успешно | +| Native reference | 8 module/storage + 1 TLS/SASL EXTERNAL federation — успешно | +| Нагрузка | 100 аккаунтов, 500 TLS клиентов, 5000 ping,5000 сообщений, 10000 MAM; failures=[] | -Итоговый прогон выполнен после исправления проверки Host с портом вне диапазона, -которую обнаружил новый сетевой тест. Все тесты запускают отдельные loopback -listeners и временные данные. После прогона менялись только отчёты и упаковка. -Native reference suite прежних модулей пройдена; непосредственно http_altconnect -проверен Rust unit/network и сопоставлением с supplied Lua source, без отдельного -native differential oracle этого модуля. +Итоговая сборка SHA-256: 24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c. MinIO report автоматически записывает +тот же hash. Большая передача повторена после последних изменений сервера. +Полный прогон включает исправление гонки устаревшего snapshot при захвате слота. +Отдельные первые прогоны нашли и помогли исправить нестабильный fingerprint +миграции и пропуск немедленного reconciliation под ещё активным claim. +Финальный результат не содержит этих ошибок. RSS — выборочный максимум, а не +гарантия точного пика для любой нагрузки. Временные 10 GiB объекты после проверки удалены. -Команда: XMPP_LOAD_CHECK=1 bash scripts/verify-local.sh с native reference env -согласно scripts/prosody_compat/README.md. Точный скрипт включает новый wire test. +## Точная граница MinIO проверки -Рабочие конфиги/DNS/proxy/certificates production не изменялись; оператор должен -добавить http_altconnect и опубликовать HTTPS well-known. Инструкция: -HTTP-ALTCONNECT.md. Native HTTP/WS fallback сохранён; для строгого XEP0156 нужны -HTTPS/WSS URLs. Настоящие Converse/Monal/Conversations UI и полный production -rollout не проверены. Общие ограничения сервера — COMPATIBILITY.md. +Использован настоящий upstream MinIO RELEASE.2025-04-22T22-12-26Z, собранный +Go1.24.3 из официального GitHub archive. Немодифицированная сборка в этой среде +не запускается: net.Interfaces/netlinkrib запрещён с EPERM. Тестовая сборка +добавляет только fallback на 127.0.0.1 для этой ошибки перечисления интерфейсов. +S3, authentication, multipart и storage code не изменены. Это не qualification +немодифицированного MinIO в обычной production среде. + +Точный diff, hashes исходного архива, обеих сборок и adapter: +validation-logs-0.7.9/minio-sandbox-compat.patch и minio-provenance.json. +Полный результат и 1280-part ETag: validation-logs-0.7.9/minio.json. +Проверен single-node loopback с HTTP к MinIO и CA-verified HTTPS к gateway. +AWS account, distributed MinIO и production сеть не проверены. + +## Повторение + +Полный прогон: XMPP_LOAD_CHECK=1 bash scripts/verify-local.sh с native env +согласно scripts/prosody_compat/README.md. В этой среде дополнительно применены +CARGO_INCREMENTAL=0 CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0. +MinIO отдельно: python3 tests/upload_minio_network.py --binary target/release/rust-xmpp +--minio /path/to/minio --report /tmp/minio-report.json. При test adapter передайте +--minio-compat-patch /path/to/patch, чтобы он попал в результат. Можно задать +MINIO_BINARY перед verify-local.sh для opt-in передачи10 GiB; нужны место и MinIO executable. + +Инструкции migration/recovery: HTTP-UPLOAD.md. Source всегда сохраняется; +незавершённая миграция блокирует startup, неопределённый Complete quarantines +только свой slot. Потеря ownership metadata не позволяет безопасно удалить +произвольные чужие multiparts; bucket lifecycle остаётся дополнительной защитой. +Сетевой сбой с HEAD404 не выдаётся за доказанную неудачу удалённого commit. +Полная замена Prosody, настоящие production клиенты и DNSSEC этим не заявляются; +общие границы — COMPATIBILITY.md. diff --git a/docs/validation-logs-0.7.8/large.json b/docs/validation-logs-0.7.8/large.json new file mode 100644 index 0000000..0cbd305 --- /dev/null +++ b/docs/validation-logs-0.7.8/large.json @@ -0,0 +1,35 @@ +{ + "status": "passed", + "requested_bytes": 10737418240, + "python_chunk_bytes": 1048576, + "transport": "verified TLS HTTPS", + "storage": "local streaming file", + "rss_sample_interval_seconds": 0.05, + "free_disk_before_bytes": 14037065728, + "max_rss_bytes": { + "server": 18079744, + "client": 21168128 + }, + "rss_proc_status_paths": { + "server": "/proc/68012/status", + "client": "/proc/self/status" + }, + "put_bytes": 10737418240, + "put_seconds": 14.389516601999276, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "stored_bytes": 10737418240, + "allocated_bytes": 10737422336, + "free_disk_after_put_bytes": 2734645248, + "head_status": 200, + "get_bytes": 10737418240, + "get_read_calls": 10241, + "get_seconds": 69.22072244599985, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "range_status": 206, + "range_offset": 4294967289, + "reuse_status": 409, + "over_quota_condition": "not-acceptable", + "temporary_files_cleaned": true, + "free_disk_after_cleanup_bytes": 12774170624, + "total_seconds": 84.19309745400096 +} diff --git a/docs/validation-logs-0.7.8/large.txt b/docs/validation-logs-0.7.8/large.txt new file mode 100644 index 0000000..0303bbc --- /dev/null +++ b/docs/validation-logs-0.7.8/large.txt @@ -0,0 +1,52 @@ +PUT: 3862953984/10737418240 bytes (736.8 MiB/s) +PUT: 7720665088/10737418240 bytes (736.2 MiB/s) +PUT: 10737418240/10737418240 bytes (724.1 MiB/s) +GET: 753926144/10737418240 bytes (143.6 MiB/s) +GET: 1443889152/10737418240 bytes (137.5 MiB/s) +GET: 2130706432/10737418240 bytes (135.3 MiB/s) +GET: 2915041280/10737418240 bytes (138.8 MiB/s) +GET: 3630170112/10737418240 bytes (138.3 MiB/s) +GET: 4347396096/10737418240 bytes (138.1 MiB/s) +GET: 5118099456/10737418240 bytes (139.3 MiB/s) +GET: 5929697280/10737418240 bytes (141.3 MiB/s) +GET: 6700400640/10737418240 bytes (141.9 MiB/s) +GET: 7456423936/10737418240 bytes (142.1 MiB/s) +GET: 8232370176/10737418240 bytes (142.6 MiB/s) +GET: 9137291264/10737418240 bytes (145.1 MiB/s) +GET: 10012852224/10737418240 bytes (146.8 MiB/s) +GET: 10737418240/10737418240 bytes (147.9 MiB/s) +{ + "status": "passed", + "requested_bytes": 10737418240, + "python_chunk_bytes": 1048576, + "transport": "verified TLS HTTPS", + "storage": "local streaming file", + "rss_sample_interval_seconds": 0.05, + "free_disk_before_bytes": 14037065728, + "max_rss_bytes": { + "server": 18079744, + "client": 21168128 + }, + "rss_proc_status_paths": { + "server": "/proc/68012/status", + "client": "/proc/self/status" + }, + "put_bytes": 10737418240, + "put_seconds": 14.389516601999276, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "stored_bytes": 10737418240, + "allocated_bytes": 10737422336, + "free_disk_after_put_bytes": 2734645248, + "head_status": 200, + "get_bytes": 10737418240, + "get_read_calls": 10241, + "get_seconds": 69.22072244599985, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "range_status": 206, + "range_offset": 4294967289, + "reuse_status": 409, + "over_quota_condition": "not-acceptable", + "temporary_files_cleaned": true, + "free_disk_after_cleanup_bytes": 12774170624, + "total_seconds": 84.19309745400096 +} diff --git a/docs/validation-logs-0.7.8/multi.txt b/docs/validation-logs-0.7.8/multi.txt new file mode 100644 index 0000000..12b7662 --- /dev/null +++ b/docs/validation-logs-0.7.8/multi.txt @@ -0,0 +1 @@ +PASS: multiple upload hosts/shared-host paths, discovery, quotas, JWT isolation, persistence, transport composition and conflict validation diff --git a/docs/validation-logs-0.7.8/s3.txt b/docs/validation-logs-0.7.8/s3.txt new file mode 100644 index 0000000..68d17ed --- /dev/null +++ b/docs/validation-logs-0.7.8/s3.txt @@ -0,0 +1 @@ +PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation diff --git a/docs/validation-logs-0.7.8/verify.txt b/docs/validation-logs-0.7.8/verify.txt new file mode 100644 index 0000000..4953d94 --- /dev/null +++ b/docs/validation-logs-0.7.8/verify.txt @@ -0,0 +1,711 @@ + Compiling rust-xmpp v0.7.8 (/workspace/scratch/362577cdf2e0/recovered/rust-xmpp) + Finished `test` profile [unoptimized + debuginfo] target(s) in 19.55s + Running unittests src/lib.rs (target/debug/deps/rust_xmpp-35326025a84450cd) + +running 42 tests +test http_config::tests::literal_pchars_and_percent_encoding_are_supported ... ok +test http_config::tests::external_prefix_and_standard_port ... ok +test http_config::tests::invalid_paths_types_urls_and_conflicts_are_errors ... ok +test http_config::tests::mounts_inherit_per_app_and_expand_virtual_host ... ok +test http_config::tests::routes_hosts_and_default_without_overriding_known_hosts ... ok +test http_proxy::tests::config_validation_and_network_boundaries ... ok +test http_proxy::tests::empty_native_lua_table_disables_proxy_trust ... ok +test http_proxy::tests::forwarded_tls_termination_cannot_upgrade_a_plaintext_chain_hop ... ok +test http_proxy::tests::forwarded_mode_stops_at_first_untrusted_hop ... ok +test federation::tests::starttls_must_be_an_immediate_stream_feature ... ok +test federation::tests::federation_domains_are_dns_names_and_case_insensitive ... ok +test federation::tests::external_must_be_in_the_advertised_mechanisms ... ok +test http_proxy::tests::legacy_chain_uses_rightmost_untrusted_and_tls_is_preserved ... ok +test http_proxy::tests::untrusted_socket_ignores_spoofed_headers ... ok +test http_proxy::tests::malformed_forwarded_and_unknown_identifiers_are_ignored ... ok +test federation::tests::stream_restarts_compare_canonical_domains ... ok +test http_transport::tests::bosh_processing_capacity_is_bounded ... ok +test http_transport::tests::configured_mount_host_origin_and_secure_dispatch ... ok +test http_transport::tests::independent_framing ... ok +test logging::tests::console_aliases_share_one_sink_with_combined_levels ... ok +test http_transport::tests::websocket_preserves_inherited_namespaces_and_spaced_declarations ... ok +test sasl::tests::reject_malformed_and_downgrade ... ok +test server::authorization_tests::http_interfaces_follow_secure_override_wildcards_and_global_fallback ... ok +test server::authorization_tests::pep_diagnostics_count_only_first_item_devices_without_logging_ids ... ok +test server::authorization_tests::pep_diagnostics_exclude_payloads_and_identifiers ... ok +test sasl::tests::rfc5802_sha1_vector ... ok +test sessions::push_timeout_tests::first_important_push_shortens_extended_hibernation_once ... ok +test sessions::tests::acknowledgement_wraps ... ok +test sessions::tests::graceful_disconnect_is_not_resumable ... ok +test sessions::tests::resume_replays_only_unacked_and_checks_owner ... ok +test wire::tests::arbitrary_prefix_and_inherited_namespaces ... ok +test wire::tests::cdata_and_comments_frame_without_counting_brackets ... ok +test wire::tests::declarations_rejected ... ok +test wire::tests::fragmented_and_pipelined ... ok +test wire::tests::invalid_namespace_and_declarations_rejected ... ok +test http_service::tests::upload_internal_mount_is_independent_of_advertised_proxy_prefix ... ok +test http_service::tests::abandoned_slots_release_quota_without_pruning_existing_or_active_files ... ok +test sasl::tests::sha256_independent_python_vector ... ok +test sasl::tests::plus_channel_binding_and_ssdp_are_authenticated ... ok +test server::authorization_tests::upload_requires_registered_local_account_or_explicit_remote_access ... ok +test sasl::tests::wrong_proof_nonce_binding_and_duplicates ... ok +test admin::tests::real_admin_xml_protocol_over_duplex ... ok + +test result: ok. 42 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.57s + + Running unittests src/main.rs (target/debug/deps/rust_xmpp-d70be398a259de34) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/admin_socket.rs (target/debug/deps/admin_socket-ac25be69d4e5c9d7) + +running 1 test +test prosody_wire_commands_password_prompt_and_safe_errors ... ignored, requires AF_UNIX socket permission; run on target host with --ignored + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/archive_mam.rs (target/debug/deps/archive_mam-e17f5bdb2236639c) + +running 14 tests +test archive_larger_than_128mib_streams ... ignored, large-file scalability regression; run explicitly +test mam_preferences_persist ... ok +test corrupt_archive_reports_server_error_and_namespace_stripping_is_precise ... ok +test append_is_incremental_and_recovers_torn_tail ... ok +test archive_roundtrip_preserves_default_namespace_resets ... ok +test legacy_records_binary_cache_and_roundtrip ... ok +test real_archive_sid_and_sender_privacy ... ok +test muc_archive_filters_stanza_kind_instead_of_sender_jid ... ok +test smart_enable_and_message_selection ... ok +test equal_cursors_and_normalized_query_jids ... ok +test mam_paging_filters_and_forbidden ... ok +test zero_page_dates_and_invalid_forms ... ok +test extended_ids_ranges_flip_and_metadata ... ok +test namespaces_expiry_and_concurrent_append ... ok + +test result: ok. 13 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/auth.rs (target/debug/deps/auth-413b6a1e33e69a8b) + +running 3 tests +test traversal_and_malicious_account_records_rejected ... ok +test create_and_read_hashed_accounts ... ok +test authentication_prepares_names_without_relaxing_account_management ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.02s + + Running tests/cli.rs (target/debug/deps/cli-57660fdc9c9efa53) + +running 2 tests +test server_cannot_be_mistaken_for_running_service ... ok +test conversion_preserves_source_and_refuses_overwrite ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/config.rs (target/debug/deps/config-eece74d883e4757d) + +running 12 tests +test include_cycles_rejected ... ok +test component_can_explicitly_enable_global_modules ... ok +test include_has_own_global_scope_and_parent_scope_is_restored ... ok +test declarations_and_overrides ... ok +test storage_paths_encode_like_prosody ... ok +test tables_cannot_recurse_forever ... ok +test supplied_configuration_scopes_arithmetic_and_modules ... ok +test runtime_capabilities_are_unavailable ... ok +test malformed_values_rejected ... ok +test supplied_config_matches_original_prosody_loader ... ok +test all_formats_round_trip_actual_config ... ok +test loops_are_bounded ... ok + +test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/csi.rs (target/debug/deps/csi-ebe2b18da2bcf4ee) + +running 2 tests +test native_importance_rules ... ok +test bounded_queue_flush_delay_activity_and_grace ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/csi_native.rs (target/debug/deps/csi_native-ae15f598b5d0a40e) + +running 1 test +test native_prosody_csi_importance_matches ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/data_lock.rs (target/debug/deps/data_lock-c65a27d7b5288f6c) + +running 2 tests +test child_process_probe ... ok +test excludes_another_process_and_releases_on_drop ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/debug_legacy.rs (target/debug/deps/debug_legacy-dee0cbfd67d7ad8f) + +running 1 test +test debug_empty_legacy_bind ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.09s + + Running tests/federation_tls.rs (target/debug/deps/federation_tls-2f12cf7c0233a9a4) + +running 3 tests +test native_prosody_tls_external_bidirectional ... ignored, requires native Prosody cafd74b9c5ea and Lua TLS dependencies +test mutual_tls_external_delivery_and_certificate_identity_rejection ... ok +test reuses_connections_serializes_writes_and_retires_idle_or_closed_peers ... ok + +test result: ok. 2 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 1.38s + + Running tests/gajim_features.rs (target/debug/deps/gajim_features-f4de5c735f024ae3) + +running 6 tests +test security_catalog_default_is_a_real_label_and_module_can_be_disabled ... ok +test security_catalogs_serve_both_versions_and_preserve_requested_target ... ok +test policy_catalog_selects_default_policy_and_maps_display_and_explicit_label ... ok +test unsupported_security_policy_configuration_returns_explicit_error ... ok +test actual_example_policy_configuration_exposes_classifications_as_catalog_labels ... ok +test malformed_or_unknown_policy_catalog_schemas_are_rejected ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/http_altconnect.rs (target/debug/deps/http_altconnect-bd40ad0185d14a9e) + +running 3 tests +test native_link_order_schemes_and_loaded_module_gates ... ok +test public_prefix_and_host_paths_inherit_independently ... ok +test bad_advertisement_types_and_route_collisions_are_rejected ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/http_transport.rs (target/debug/deps/http_transport-31a565c13311b23a) + +running 10 tests +test configured_origin_is_enforced ... ok +test bosh_preserves_namespace_inherited_from_body ... ok +test transport_paths_and_cors_preflight_match_prosody ... ok +test bosh_session_retries_restart_termination ... ok +test bosh_termination_waits_for_graceful_cleanup_before_reconnect ... ok +test bosh_reorders_two_requests_without_replaying_stanzas ... ok +test websocket_frames_are_independent_xml ... ok +test websocket_disconnect_delivers_eof_without_graceful_close ... ok +test websocket_close_allows_graceful_engine_cleanup ... ok +test bosh_retry_survives_aborted_http_request_without_replaying_stanzas ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + + Running tests/http_upload.rs (target/debug/deps/http_upload-d7c9290e01081e96) + +running 5 tests +test https_endpoint_uses_real_tls ... ok +test percent_escape_case_and_equivalent_url_spelling ... ok +test expired_upload_releases_quota ... ok +test persisted_prosody_archive_and_hs256_slot_survive_without_sidecar ... ok +test upload_authorization_streaming_ranges_and_restart ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/iq_errors.rs (target/debug/deps/iq_errors-dd580e9884198ecf) + +running 2 tests +test server_errors_without_target_remain_unaddressed_and_escape_attributes ... ok +test pep_errors_identify_the_account_and_echo_the_request ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/logging.rs (target/debug/deps/logging-b10fa512fcde5034) + +running 10 tests +test rejects_malformed_configuration ... ok +test native_console_shorthands_are_accepted ... ok +test overlapping_shorthand_destinations_receive_each_record_once ... ok +test filters_appends_escapes_and_redacts ... ok +test json_records_and_bounded_ranges ... ok +test explicit_sink_levels_select_exactly_and_deduplicate ... ok +test reopen_rotated_files_updates_clones ... ok +test failed_reopen_keeps_all_original_handles ... ok +test shared_sink_unions_levels_and_preserves_distinct_destinations ... ok +test cloned_loggers_serialize_records ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/message_routing.rs (target/debug/deps/message_routing-584b523f0fe41bd6) + +running 2 tests +test unknown_resource_falls_back_to_available_bare_jid ... ok +test headline_uses_all_nonnegative_available_resources_and_errors_never_fallback ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_access.rs (target/debug/deps/muc_access-c83485150bd43e36) + +running 3 tests +test moderators_cannot_kick_or_demote_other_moderators ... ok +test nested_affiliations_do_not_fall_back_to_obsolete_flat_entries ... ok +test binary_occupant_id_salt_survives_saves_and_restarts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/muc_domain_bans.rs (target/debug/deps/muc_domain_bans-534ec825c7201623) + +running 2 tests +test applying_domain_ban_removes_remote_occupants_and_blocks_rejoin ... ok +test native_domain_bans_block_remote_users_but_allow_explicit_members ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_extensions.rs (target/debug/deps/muc_extensions-a9f5fc7e3f8b7ee0) + +running 10 tests +test disabled_optional_vcard_and_occupant_ids_are_not_advertised_or_emitted ... ok +test broadcast_roles_hide_visitors_and_configuration_announces_transition ... ok +test messages_cannot_spoof_room_identity_hats_or_occupant_identifiers ... ok +test restricted_private_messages_allow_self_and_optional_moderators ... ok +test large_optional_history_does_not_prevent_checkpointing_membership ... ok +test language_config_sections_discovery_and_native_affiliation_hats ... ok +test mam_all_rooms_presence_logging_public_queries_and_optional_totals ... ok +test offline_reserved_affiliates_follow_anonymity_and_roominfo_counts_nicknames ... ok +test recovery_claim_is_durable_and_unrelated_disconnect_keeps_temporary_claims ... ok +test explicit_new_nickname_consumes_old_proof_and_pending_nicks_are_reserved ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_flow.rs (target/debug/deps/muc_flow-e0f0dc747cf31176) + +running 18 tests +test configured_lock_timeout_expires_without_owner_submission ... ok +test iq_response_requires_expected_sender ... ok +test authorization_and_conflict ... ok +test join_lock_chat_leave ... ok +test multiple_resources_share_nick_and_private_delivery ... ok +test nick_invite_destroy_and_local_creation ... ok +test occupant_id_is_stable_and_spoofed_ids_removed ... ok +test restored_temporary_room_is_deleted_after_last_disconnect ... ok +test preserves_unknown_binary_mixed_room_fields ... ok +test persistence_roundtrip ... ok +test switching_to_members_only_removes_unaffiliated_users ... ok +test room_vcard_owner_only_and_avatar_presence ... ok +test visitor_voice_request_cannot_spoof_identity_or_grant_itself_voice ... ok +test persistent_destroy_creates_prosody_tombstone ... ok +test reserved_nickname_blocks_other_users ... ok +test maintenance_expires_idle_archives_and_tombstones ... ok +test archive_history_and_permissions ... ok +test native_muc_archive_type_and_anonymity_are_preserved ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_history.rs (target/debug/deps/muc_history-4855797b481a8f39) + +running 10 tests +test capacity_changes_trim_and_zero_disables_history ... ok +test only_direct_history_children_are_parsed ... ok +test maxchars_counts_transmitted_utf8_bytes_including_address_and_delay ... ok +test default_count_applies_only_without_valid_explicit_constraints ... ok +test explicit_count_and_zero_limit ... ok +test replay_carries_room_delay_and_recipient ... ok +test ordinary_history_is_bounded_and_returns_oldest_first ... ok +test snapshot_restores_last_message_without_adding_another_delay ... ok +test subjects_and_hints_follow_native_history_eligibility ... ok +test since_and_seconds_use_stricter_cutoff_inclusive ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_native.rs (target/debug/deps/muc_native-90f713df575cd00c) + +running 1 test +test actual_prosody_muc_mam_reads_rust_and_writes_native_archive ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_recovery.rs (target/debug/deps/muc_recovery-4536d02c6db5e036) + +running 7 tests +test claim_is_one_time_and_does_not_change_untrusted_role ... ok +test pending_snapshot_keeps_original_checkpoint_time ... ok +test recovery_claim_requires_exact_authenticated_resource_and_expires ... ok +test native_checkpoint_roundtrip_filters_sensitive_extensions ... ok +test native_last_message_roundtrip_preserves_namespaces_and_rejects_other_rooms ... ok +test oversized_status_never_prevents_membership_checkpoint ... ok +test configuration_fingerprint_is_independent_of_lua_table_iteration_order ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/muc_restart_state.rs (target/debug/deps/muc_restart_state-6313d0e7fcfc0965) + +running 3 tests +test unrelated_disconnect_does_not_delete_dormant_temporary_room ... ok +test changed_configuration_invalidates_auto_recovery_without_disclosing_password ... ok +test repeated_restart_keeps_dormant_claim_and_deadline_without_ghost_routing ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_state_native.rs (target/debug/deps/muc_state_native-d14147ffa532b80c) + +running 1 test +test actual_prosody_restores_rust_checkpoint_and_rust_reads_native_freeze ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pep_client_interop.rs (target/debug/deps/pep_client_interop-e94bb07c3973779b) + +running 7 tests +test native_component_notification_type_applies_to_events_and_subscription_replay ... ok +test missing_own_omemo_bundle_returns_recoverable_item_not_found ... ok +test avatar_and_omemo_publish_options_support_readback_and_caps_notifications ... ok +test native_itemreply_controls_publisher_visibility_in_reads_and_replays ... ok +test publish_options_match_implicit_notification_defaults_on_old_nodes ... ok +test migrated_native_configuration_can_roundtrip_access_model_edit ... ok +test native_open_and_subscriber_publish_models_preserve_outcast_and_blocking_checks ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.20s + + Running tests/pep_diagnostics.rs (target/debug/deps/pep_diagnostics-9580f8658fa6ea58) + +running 6 tests +test nonempty_xml_can_be_an_empty_android_device_list ... ok +test avatar_classification_does_not_inspect_or_expose_avatar_data ... ok +test arbitrary_node_payload_and_wrong_namespace_do_not_leak ... ok +test signed_integer_limits_and_duplicates_match_java_parser ... ok +test payloadless_notification_is_visible_without_identifiers ... ok +test only_first_direct_item_is_parsed_and_protocols_are_distinct ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pep_discovery.rs (target/debug/deps/pep_discovery-76b2cbf7a11be6b6) + +running 4 tests +test standalone_component_does_not_claim_pep_automatic_subscriptions ... ok +test account_disco_advertises_pep_without_requiring_a_node ... ok +test account_discovery_does_not_intercept_server_or_resource_caps ... ok +test node_discovery_keeps_node_identity_errors_and_access_checks ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running tests/pep_item_order.rs (target/debug/deps/pep_item_order-6ee5b8ed27d40885) + +running 2 tests +test native_prosody_orders_retained_pep_items_newest_first ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA +test first_pep_item_is_current_device_list_even_with_retained_old_items ... ok + +test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.12s + + Running tests/preauth.rs (target/debug/deps/preauth-c3e953dce3c3a139) + +running 10 tests +test duplicate_or_wrong_namespace_tokens_do_not_qualify ... ok +test standard_adhoc_command_returns_contact_uri ... ok +test unknown_tokens_do_not_create_invite_stores_and_disabled_accounts_cannot_issue ... ok +test prosody_numeric_expiry_is_consumed_and_other_entries_preserved ... ok +test unknown_client_token_is_preserved_on_remote_and_pending_delivery ... ok +test concurrent_redemption_has_exactly_one_winner ... ok +test invitations_expire_are_single_use_and_owner_scoped ... ok +test identity_scope_does_not_burn_on_wrong_sender_or_leak_to_prosody ... ok +test valid_token_autoapproves_and_sends_reciprocal_request ... ok +test local_invitation_updates_both_rosters_without_granting_reverse_subscription ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/preauth_native.rs (target/debug/deps/preauth_native-156502f568bdc619) + +running 1 test +test native_prosody_invitation_tokens_roundtrip ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pubsub_native.rs (target/debug/deps/pubsub_native-254f4b34712a671e) + +running 1 test +test native_prosody_pep_reads_rust_and_rust_reads_native ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pubsub_persistence.rs (target/debug/deps/pubsub_persistence-b91fb3b49e96cd6d) + +running 18 tests +test modern_bookmarks_notify_legacy_capability ... ok +test native_string_max_items_satisfies_numeric_publish_precondition ... ok +test configurable_limits_component_creation_and_publisher ... ok +test configure_requires_owner ... ok +test capability_discovery_replays_and_filters_notifications ... ok +test nonpersistent_items_stay_in_memory_and_can_be_made_persistent ... ok +test open_model_and_notifications ... ok +test reads_prosody_legacy_fixture ... ok +test full_jid_block_applies_to_reads_subscription_replays_and_events ... ok +test publish_options_conflict_is_recoverable_and_preserves_item ... ok +test pep_persists_namespaces_and_enforces_owner ... ok +test roster_presence_access_and_foreign_subscription_denied ... ok +test remote_caps_replay_and_open_node_privacy_revoke ... ok +test remote_unavailable_is_scoped_to_addressed_owner_and_cancels_pending ... ok +test subscription_options_disable_events_and_require_subscriber ... ok +test payload_policy_applies_to_subscription_and_caps_replays ... ok +test native_notification_fields_and_retract_policy_are_independent ... ok +test component_default_retention_matches_prosody_twenty_items ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.29s + + Running tests/push.rs (target/debug/deps/push-f815f609aff0106d) + +running 7 tests +test native_push_body_lookup_excludes_extensions ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA +test foreign_account_cannot_register ... ok +test queue_coalesces_and_importance_matches_native ... ok +test push_summary_never_copies_foreign_namespace_body ... ok +test reenable_cancels_old_errors_and_failure_counts_are_per_account ... ok +test stable_client_registration_and_targeted_notification ... ok +test registrations_notifications_and_authenticated_errors ... ok + +test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running tests/reference_prosody.rs (target/debug/deps/reference_prosody-28d5b077c2534d6d) + +running 1 test +test real_prosody_storage_and_auth_roundtrip ... ignored, requires native supplied Prosody source; run scripts/prosody_compat/verify.sh + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/resource_sessions.rs (target/debug/deps/resource_sessions-e944df54464415e0) + +running 7 tests +test kick_new_leaves_existing_session_unchanged ... ok +test increment_skips_occupied_suffixes_and_preserves_sessions ... ok +test random_conflicts_get_distinct_resources_for_same_owner ... ok +test replacement_revokes_hibernating_resume_and_survives_old_cleanup ... ok +test invalid_resume_owner_or_ack_does_not_evict_active_session ... ok +test concurrent_default_binds_replace_atomically ... ok +test active_resume_preserves_state_and_replaces_transport_notifier ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/sasl_accounts.rs (target/debug/deps/sasl_accounts-d10120a27f08b817) + +running 1 test +test disabled_legacy_and_unicode_accounts ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.67s + + Running tests/storage_accounts.rs (target/debug/deps/storage_accounts-caa2428c7ab1794f) + +running 20 tests +test broken_roster_is_not_overwritten ... ok +test account_purge_refuses_symlink_store_without_touching_other_files ... ok +test inbound_subscription_requires_pending_to_grant_to ... ok +test malformed_legacy_vcard_and_private_bookmarks_rejected ... ok +test blocklist_enforcement_api ... ok +test malformed_roster_iq_never_mutates ... ok +test domain_roster_and_block_jid_normalization ... ok +test private_and_vcard_authorization ... ok +test privacy_migration_only_default_deny_jids ... ok +test roster_empty_name_and_foreign_group_ignored ... ok +test reads_timestamp_blocklist_and_keeps_metadata ... ok +test roster_preserves_subscription_and_metadata ... ok +test purge_account_exact_files_preserves_others_and_extras ... ok +test subscription_handshake_and_cancel ... ok +test legacy_bookmarks_respect_host_limit_and_read_over_256 ... ok +test bookmarks_private_pep_bidirectional_and_private ... ok +test legacy_migration_clears_source_after_success ... ok +test vcard_pep_conversion_and_avatar ... ok +test password_change_after_purge_never_recreates_account ... ok +test creation_and_purge_are_serialized ... ok + +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.45s + + Running tests/storage_compat.rs (target/debug/deps/storage_compat-8c065a953d7f1c5f) + +running 6 tests +test empty_namespace_preserved ... ok +test stanza_roundtrip ... ok +test mixed_binary_roundtrip ... ok +test path_and_atomic_update ... ok +test rejects_cycles_code_and_bad_files ... ok +test concurrent_updates_do_not_lose_entries ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/tls_binding.rs (target/debug/deps/tls_binding-7c9f069d2cb9d13b) + +running 4 tests +test malformed_der_and_pss_parameters_fail_closed ... ok +test unknown_and_eddsa_algorithms_fail_closed ... ok +test ecdsa_uses_certificate_signature_hash ... ok +test rsa_signature_digest_and_weak_hash_upgrade ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.38s + + Running tests/tls_plus_network.rs (target/debug/deps/tls_plus_network-7312b0d545fb875d) + +running 2 tests +test disabling_plus_hides_bindings_and_accepts_scram_y_with_mechanisms_only_ssdp ... ok +test starttls_scram_plus_binds_real_tls_and_authenticates_ssdp_sasl1_and_sasl2 ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.44s + + Doc-tests rust_xmpp + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.13s + Finished `release` profile [optimized] target(s) in 0.09s +PASS: integrated TCP roster/subscriptions, client IQ routing, CSI buffering/keepalive flush, MAM, blocking, OMEMO PEP, MUC, offline replay and SM resumption +PASS: graceful SIGTERM shutdown +PASS: independent SCRAM-SHA-1 over TLS, SASL1/SASL2, SSDP verification, server signatures and wrong-password rejection +PASS: independent SCRAM-SHA-256 over TLS, SASL1/SASL2, SSDP verification, server signatures and wrong-password rejection +PASS: Android TLS1.2 selector, TLS1.2/1.3 SCRAM SHA1/256 PLUS, SASL1/2, SSDP on/off and invalid binding rejection +PASS: JSON file logging, TLS/auth/bind correlation, rejection reasons, secret redaction and SIGHUP rotation +PASS: 20 TLS1.2/1.3 SASL1/2 cases: prepared PLAIN identities and canonical binds, invalid authz rejection, SCRAM abort/retry and active exchange restart +PASS: WSS SCRAM-SHA-256 SASL1/SASL2 with GS2 y, absent CB/PLUS, SSDP without CB separator, server signatures, bind and ping +PASS: HTTPS-only configured interfaces (127.0.0.1,::1), verified TLS, reproduced original Origin 403 then allowed Converse Origin 101, xmpp subprotocol, both WS paths, RFC 7395 open/features/close and private HTTP diagnostics +PASS: custom WS/BOSH routes, default-route 404, exact Origin policy, verified TLS proxy rewrite, trusted HTTPS metadata, untrusted/spoofed metadata denial, IPv6 when available, authenticated WS/BOSH bind/ping in legacy and RFC7239 modes and cached BOSH RID retry +PASS: XEP-0156 XRD/JRD parity, CORS, safe methods, ignored JSONP, host checks, secure URL preference, custom mounts/public prefixes, module and advertisement gates, HTTP fallback and verified WSS framing +PASS: SASL2 Bind2 TLS flow, inline SM/CSI/Carbons, private random resource, malformed requests, authentication failure and fallback +PASS: bind2-disabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-disabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-enabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-enabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: sm-disabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: sm-disabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: enabled-False-registration-False +PASS: enabled-False-registration-True +PASS: enabled-True-registration-False +PASS: enabled-True-registration-True +PASS: Gajim domain/account discovery, both label catalogs, authenticated password changes, IBR registration gates and fresh logins +PASS: SASL2 inline SM active/EOF resume, authenticated owner/id/h checks, replay ordering, full JID, Bind2 fallback and malformed requests +PASS: TLS account/domain discovery with PEP and push enabled/disabled; real push enable, publish options, per-node and service-wide disable +PASS: TLS Monocles/Conversations PEP first-run, no-to publishing, owner config recovery, migrated metadata, device item selectors and restart persistence +PASS: Android first-item device-list parsing sees latest publish and replacement before and after restart +PASS: STARTTLS PEP replay/live diagnostics, legacy/OMEMO2, payloadless notifications with intact IQ payloads, and private values excluded +PASS: TLS invitation command discovery, preapproval, single-use mutual roster, replay denial, unknown preauth forwarding/offline pending, account isolation +PASS: STARTTLS MUC SIGTERM restart restores live temporary/persistent rooms, exact full-JID membership, roles, presence and groupchat routing; rejects automatic cross-resource/account claims and nickname theft +PASS: TLS conflict_resolve=default, live and hibernating SM resources +PASS: TLS conflict_resolve=kick_old, live and hibernating SM resources +PASS: TLS conflict_resolve=kick_new, live and hibernating SM resources +PASS: TLS conflict_resolve=increment, live and hibernating SM resources +PASS: TLS conflict_resolve=random, live and hibernating SM resources +PASS: integrated WSS + HTTPS BOSH SASL/bind, CSI, inherited BOSH namespaces, roster, bidirectional chat, upload IQ/PUT/GET/reuse, graceful SM close and same-resource relogin under kick_new +PASS: integrated WSS + HTTPS BOSH SASL/bind, CSI, inherited BOSH namespaces, roster, bidirectional chat, upload IQ/PUT/GET/reuse, graceful SM close and same-resource relogin under kick_new +PASS: configured upload HTTP path with $host through integrated listener +PASS: multiple upload hosts/shared-host paths, discovery, quotas, JWT isolation, persistence, transport composition and conflict validation +PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation +{ + "total_load_wall_seconds": 7.782, + "requested_resources": 500, + "authenticated_bound_resources": 500, + "connection_setup_concurrency": 50, + "setup_wall_seconds": 0.63, + "setup_latency": { + "samples": 500, + "median_ms": 57.454, + "p95_ms": 86.829, + "max_ms": 95.615 + }, + "ping_rounds": 10, + "ping_round_trip": { + "samples": 5000, + "median_ms": 14.241, + "p95_ms": 23.05, + "max_ms": 27.298 + }, + "accounts": 100, + "message_rounds": 10, + "messages_sent": 5000, + "messages_received": 5000, + "message_batch_seconds": 4.835, + "metrics_before": { + "rss_kib": 8056, + "cpu_seconds": 0.0, + "fd_count": 11 + }, + "metrics_connected": { + "rss_kib": 26868, + "cpu_seconds": 1.22, + "fd_count": 511 + }, + "metrics_after_workload": { + "rss_kib": 27456, + "cpu_seconds": 9.05, + "fd_count": 511 + }, + "metrics_two_seconds_after_disconnect": { + "rss_kib": 27648, + "cpu_seconds": 9.09, + "fd_count": 11 + }, + "sampled_peak_rss_kib": 27648, + "fd_delta_after_disconnect": 0, + "failures": [], + "scope": "One loopback host, 100 accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.", + "mam": { + "archive_files": 100, + "records": 10000, + "expected_records": 10000 + } +} + Compiling rust-xmpp v0.7.8 (/workspace/scratch/362577cdf2e0/recovered/rust-xmpp) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.85s + Running tests/csi_native.rs (target/debug/deps/csi_native-ae15f598b5d0a40e) + +running 1 test +test native_prosody_csi_importance_matches ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.14s + + Running tests/muc_native.rs (target/debug/deps/muc_native-90f713df575cd00c) + +running 1 test +test actual_prosody_muc_mam_reads_rust_and_writes_native_archive ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.22s + + Running tests/muc_state_native.rs (target/debug/deps/muc_state_native-d14147ffa532b80c) + +running 1 test +test actual_prosody_restores_rust_checkpoint_and_rust_reads_native_freeze ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/pep_item_order.rs (target/debug/deps/pep_item_order-6ee5b8ed27d40885) + +running 1 test +test native_prosody_orders_retained_pep_items_newest_first ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.13s + + Running tests/preauth_native.rs (target/debug/deps/preauth_native-156502f568bdc619) + +running 1 test +test native_prosody_invitation_tokens_roundtrip ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.16s + + Running tests/pubsub_native.rs (target/debug/deps/pubsub_native-254f4b34712a671e) + +running 1 test +test native_prosody_pep_reads_rust_and_rust_reads_native ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.11s + + Running tests/push.rs (target/debug/deps/push-f815f609aff0106d) + +running 1 test +test native_push_body_lookup_excludes_extensions ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 6 filtered out; finished in 0.01s + + Running tests/reference_prosody.rs (target/debug/deps/reference_prosody-28d5b077c2534d6d) + +running 1 test +PROSODY_NATIVE_ROUNDTRIP_OK SHA-1 +PROSODY_NATIVE_FINAL_OK SHA-1 +PROSODY_NATIVE_ROUNDTRIP_OK SHA-256 +PROSODY_NATIVE_FINAL_OK SHA-256 +test real_prosody_storage_and_auth_roundtrip ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.92s + + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.06s + Running tests/federation_tls.rs (target/debug/deps/federation_tls-2f12cf7c0233a9a4) + +running 1 test +test native_prosody_tls_external_bidirectional ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 2 filtered out; finished in 0.56s + diff --git a/docs/validation-logs-0.7.9/migration.txt b/docs/validation-logs-0.7.9/migration.txt new file mode 100644 index 0000000..37b702b --- /dev/null +++ b/docs/validation-logs-0.7.9/migration.txt @@ -0,0 +1 @@ +PASS upload migration: local -> S3 -> S3 -> local, hashes, original URLs/JWT/HEAD, lock, dry-run, fail-closed resume diff --git a/docs/validation-logs-0.7.9/minio-provenance.json b/docs/validation-logs-0.7.9/minio-provenance.json new file mode 100644 index 0000000..a1d6bf6 --- /dev/null +++ b/docs/validation-logs-0.7.9/minio-provenance.json @@ -0,0 +1,12 @@ +{ + "source_url": "https://github.com/minio/minio/archive/refs/tags/RELEASE.2025-04-22T22-12-26Z.tar.gz", + "upstream_release": "RELEASE.2025-04-22T22-12-26Z", + "build": "Go1.24.3 go build -p4; no dependency changes", + "sha256": { + "/tmp/minio-source.tar.gz": "1a63209927b9d1fb239b2a776b7b01d1902eff19379a108f4bc7005edcc0968c", + "/tmp/minio-genuine": "3dd5d6756ad17c599fbc1c6f6a2f5b590242c049522e848c700949dbf8e60ad9", + "/tmp/minio-sandbox-compat": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "/tmp/minio-sandbox-compat.patch": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5" + }, + "compatibility_limitation": "Only net.Interfaces EPERM becomes loopback127.0.0.1; all S3/auth/multipart/storage source unchanged." +} diff --git a/docs/validation-logs-0.7.9/minio-sandbox-compat.patch b/docs/validation-logs-0.7.9/minio-sandbox-compat.patch new file mode 100644 index 0000000..bac66f4 --- /dev/null +++ b/docs/validation-logs-0.7.9/minio-sandbox-compat.patch @@ -0,0 +1,21 @@ +--- a/cmd/net.go ++++ b/cmd/net.go +@@ -24,6 +24,7 @@ + "runtime" + "sort" + "strings" ++ "syscall" + + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" +@@ -54,6 +55,10 @@ + // mustGetLocalIPs returns IPs of local interface + func mustGetLocalIPs() (ipList []net.IP) { + ifs, err := net.Interfaces() ++ // TEST ONLY: the sandbox denies netlink enumeration; all listeners bind loopback. ++ if errors.Is(err, syscall.EPERM) { ++ return []net.IP{net.ParseIP("127.0.0.1")} ++ } + logger.FatalIf(err, "Unable to get IP addresses of this host") + + for _, interf := range ifs { diff --git a/docs/validation-logs-0.7.9/minio.json b/docs/validation-logs-0.7.9/minio.json new file mode 100644 index 0000000..f1deb95 --- /dev/null +++ b/docs/validation-logs-0.7.9/minio.json @@ -0,0 +1,51 @@ +{ + "ok": true, + "minio_version": "minio-sandbox-compat version DEVELOPMENT.GOGET (commit-id=DEVELOPMENT.GOGET)\nRuntime: go1.24.3 linux/amd64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-0000 MinIO, Inc.", + "minio_binary_sha256": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "binary": "/workspace/scratch/362577cdf2e0/recovered/rust-xmpp/target/release/rust-xmpp", + "binary_version": "rust-xmpp 0.7.9 \u2014 experimental XMPP runtime", + "binary_sha256": "24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c", + "size": 10737418240, + "qualifies_over_5gib": true, + "minio_test_only_compatibility_patch": { + "sha256": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5", + "diff": "--- a/cmd/net.go\n+++ b/cmd/net.go\n@@ -24,6 +24,7 @@\n \t\"runtime\"\n \t\"sort\"\n \t\"strings\"\n+\t\"syscall\"\n \n \t\"github.com/minio/minio-go/v7/pkg/set\"\n \t\"github.com/minio/minio/internal/config\"\n@@ -54,6 +55,10 @@\n // mustGetLocalIPs returns IPs of local interface\n func mustGetLocalIPs() (ipList []net.IP) {\n \tifs, err := net.Interfaces()\n+\t// TEST ONLY: the sandbox denies netlink enumeration; all listeners bind loopback.\n+\tif errors.Is(err, syscall.EPERM) {\n+\t\treturn []net.IP{net.ParseIP(\"127.0.0.1\")}\n+\t}\n \tlogger.FatalIf(err, \"Unable to get IP addresses of this host\")\n \n \tfor _, interf := range ifs {\n", + "limitation": "Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream." + }, + "max_rss_bytes": { + "server": 106057728, + "client": 24338432, + "minio": 498958336 + }, + "put_bytes": 10737418240, + "put_seconds": 48.45253416699961, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "minio_etag": "958f42d583ac40ff209eb865c5eb4bd6-1280", + "multipart_parts": 1280, + "minio_allocated_bytes": 10747953152, + "free_disk_after_put_bytes": 15180328960, + "get_bytes": 10737418240, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "get_seconds": 13.709708632000911, + "range_offsets": [ + 0, + 4294967289, + 5368709113, + 10737418208 + ], + "restart_range_ok": true, + "restart_reuse_status": 409, + "crash_multipart_cleanup_ok": true, + "minio_upstream_provenance": { + "source_url": "https://github.com/minio/minio/archive/refs/tags/RELEASE.2025-04-22T22-12-26Z.tar.gz", + "upstream_release": "RELEASE.2025-04-22T22-12-26Z", + "build": "Go1.24.3 go build -p4; no dependency changes", + "sha256": { + "/tmp/minio-source.tar.gz": "1a63209927b9d1fb239b2a776b7b01d1902eff19379a108f4bc7005edcc0968c", + "/tmp/minio-genuine": "3dd5d6756ad17c599fbc1c6f6a2f5b590242c049522e848c700949dbf8e60ad9", + "/tmp/minio-sandbox-compat": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "/tmp/minio-sandbox-compat.patch": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5" + }, + "compatibility_limitation": "Only net.Interfaces EPERM becomes loopback127.0.0.1; all S3/auth/multipart/storage source unchanged." + } +} diff --git a/docs/validation-logs-0.7.9/minio.txt b/docs/validation-logs-0.7.9/minio.txt new file mode 100644 index 0000000..dfb88ac --- /dev/null +++ b/docs/validation-logs-0.7.9/minio.txt @@ -0,0 +1,52 @@ +MINIO PUT: 1074790400/10737418240 bytes (204.9 MiB/s) +MINIO PUT: 2181038080/10737418240 bytes (208.0 MiB/s) +MINIO PUT: 3321888768/10737418240 bytes (211.2 MiB/s) +MINIO PUT: 4504682496/10737418240 bytes (214.7 MiB/s) +MINIO PUT: 5596250112/10737418240 bytes (213.1 MiB/s) +MINIO PUT: 6719275008/10737418240 bytes (213.2 MiB/s) +MINIO PUT: 7837057024/10737418240 bytes (213.2 MiB/s) +MINIO PUT: 8984199168/10737418240 bytes (213.8 MiB/s) +MINIO PUT: 10084155392/10737418240 bytes (213.3 MiB/s) +MINIO PUT: 10737418240/10737418240 bytes (213.1 MiB/s) +MINIO GET: 3736076288/10737418240 bytes (712.4 MiB/s) +MINIO GET: 7658799104/10737418240 bytes (730.3 MiB/s) +MINIO GET: 10737418240/10737418240 bytes (746.9 MiB/s) +{ + "ok": true, + "minio_version": "minio-sandbox-compat version DEVELOPMENT.GOGET (commit-id=DEVELOPMENT.GOGET)\nRuntime: go1.24.3 linux/amd64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-0000 MinIO, Inc.", + "minio_binary_sha256": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "binary": "/workspace/scratch/362577cdf2e0/recovered/rust-xmpp/target/release/rust-xmpp", + "binary_version": "rust-xmpp 0.7.9 \u2014 experimental XMPP runtime", + "binary_sha256": "24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c", + "size": 10737418240, + "qualifies_over_5gib": true, + "minio_test_only_compatibility_patch": { + "sha256": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5", + "diff": "--- a/cmd/net.go\n+++ b/cmd/net.go\n@@ -24,6 +24,7 @@\n \t\"runtime\"\n \t\"sort\"\n \t\"strings\"\n+\t\"syscall\"\n \n \t\"github.com/minio/minio-go/v7/pkg/set\"\n \t\"github.com/minio/minio/internal/config\"\n@@ -54,6 +55,10 @@\n // mustGetLocalIPs returns IPs of local interface\n func mustGetLocalIPs() (ipList []net.IP) {\n \tifs, err := net.Interfaces()\n+\t// TEST ONLY: the sandbox denies netlink enumeration; all listeners bind loopback.\n+\tif errors.Is(err, syscall.EPERM) {\n+\t\treturn []net.IP{net.ParseIP(\"127.0.0.1\")}\n+\t}\n \tlogger.FatalIf(err, \"Unable to get IP addresses of this host\")\n \n \tfor _, interf := range ifs {\n", + "limitation": "Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream." + }, + "max_rss_bytes": { + "server": 106057728, + "client": 24338432, + "minio": 498958336 + }, + "put_bytes": 10737418240, + "put_seconds": 48.45253416699961, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "minio_etag": "958f42d583ac40ff209eb865c5eb4bd6-1280", + "multipart_parts": 1280, + "minio_allocated_bytes": 10747953152, + "free_disk_after_put_bytes": 15180328960, + "get_bytes": 10737418240, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "get_seconds": 13.709708632000911, + "range_offsets": [ + 0, + 4294967289, + 5368709113, + 10737418208 + ], + "restart_range_ok": true, + "restart_reuse_status": 409, + "crash_multipart_cleanup_ok": true +} diff --git a/docs/validation-logs-0.7.9/s3.txt b/docs/validation-logs-0.7.9/s3.txt new file mode 100644 index 0000000..68d17ed --- /dev/null +++ b/docs/validation-logs-0.7.9/s3.txt @@ -0,0 +1 @@ +PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation diff --git a/docs/validation-logs-0.7.9/verify.txt b/docs/validation-logs-0.7.9/verify.txt new file mode 100644 index 0000000..edb6959 --- /dev/null +++ b/docs/validation-logs-0.7.9/verify.txt @@ -0,0 +1,713 @@ + Finished `test` profile [unoptimized] target(s) in 0.09s + Running unittests src/lib.rs (target/debug/deps/rust_xmpp-0e8ce8f4bd38da6f) + +running 45 tests +test federation::tests::external_must_be_in_the_advertised_mechanisms ... ok +test federation::tests::starttls_must_be_an_immediate_stream_feature ... ok +test federation::tests::federation_domains_are_dns_names_and_case_insensitive ... ok +test http_config::tests::external_prefix_and_standard_port ... ok +test http_config::tests::literal_pchars_and_percent_encoding_are_supported ... ok +test http_config::tests::invalid_paths_types_urls_and_conflicts_are_errors ... ok +test http_config::tests::mounts_inherit_per_app_and_expand_virtual_host ... ok +test http_proxy::tests::config_validation_and_network_boundaries ... ok +test http_config::tests::routes_hosts_and_default_without_overriding_known_hosts ... ok +test http_proxy::tests::empty_native_lua_table_disables_proxy_trust ... ok +test federation::tests::stream_restarts_compare_canonical_domains ... ok +test http_proxy::tests::legacy_chain_uses_rightmost_untrusted_and_tls_is_preserved ... ok +test http_proxy::tests::forwarded_mode_stops_at_first_untrusted_hop ... ok +test http_proxy::tests::malformed_forwarded_and_unknown_identifiers_are_ignored ... ok +test http_proxy::tests::forwarded_tls_termination_cannot_upgrade_a_plaintext_chain_hop ... ok +test http_proxy::tests::untrusted_socket_ignores_spoofed_headers ... ok +test http_transport::tests::bosh_processing_capacity_is_bounded ... ok +test http_transport::tests::independent_framing ... ok +test http_transport::tests::configured_mount_host_origin_and_secure_dispatch ... ok +test logging::tests::console_aliases_share_one_sink_with_combined_levels ... ok +test http_transport::tests::websocket_preserves_inherited_namespaces_and_spaced_declarations ... ok +test sasl::tests::reject_malformed_and_downgrade ... ok +test http_service::tests::upload_claim_refreshes_completion_state_after_an_earlier_snapshot ... ok +test server::authorization_tests::http_interfaces_follow_secure_override_wildcards_and_global_fallback ... ok +test server::authorization_tests::pep_diagnostics_count_only_first_item_devices_without_logging_ids ... ok +test server::authorization_tests::pep_diagnostics_exclude_payloads_and_identifiers ... ok +test sasl::tests::rfc5802_sha1_vector ... ok +test sessions::push_timeout_tests::first_important_push_shortens_extended_hibernation_once ... ok +test sessions::tests::acknowledgement_wraps ... ok +test sessions::tests::graceful_disconnect_is_not_resumable ... ok +test sessions::tests::resume_replays_only_unacked_and_checks_owner ... ok +test upload_migration::tests::local_spool_checks_size_and_hash_and_preserves_source ... ok +test sasl::tests::sha256_independent_python_vector ... ok +test wire::tests::arbitrary_prefix_and_inherited_namespaces ... ok +test wire::tests::cdata_and_comments_frame_without_counting_brackets ... ok +test wire::tests::declarations_rejected ... ok +test wire::tests::fragmented_and_pipelined ... ok +test wire::tests::invalid_namespace_and_declarations_rejected ... ok +test upload_migration::tests::pending_and_corrupt_journals_block_startup ... ok +test http_service::tests::upload_internal_mount_is_independent_of_advertised_proxy_prefix ... ok +test http_service::tests::abandoned_slots_release_quota_without_pruning_existing_or_active_files ... ok +test sasl::tests::plus_channel_binding_and_ssdp_are_authenticated ... ok +test server::authorization_tests::upload_requires_registered_local_account_or_explicit_remote_access ... ok +test sasl::tests::wrong_proof_nonce_binding_and_duplicates ... ok +test admin::tests::real_admin_xml_protocol_over_duplex ... ok + +test result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s + + Running unittests src/main.rs (target/debug/deps/rust_xmpp-575c9d0fa7f7c3a1) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/admin_socket.rs (target/debug/deps/admin_socket-6b2d8131b7ed3ad8) + +running 1 test +test prosody_wire_commands_password_prompt_and_safe_errors ... ignored, requires AF_UNIX socket permission; run on target host with --ignored + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/archive_mam.rs (target/debug/deps/archive_mam-a4e00b6b646420db) + +running 14 tests +test archive_larger_than_128mib_streams ... ignored, large-file scalability regression; run explicitly +test mam_preferences_persist ... ok +test legacy_records_binary_cache_and_roundtrip ... ok +test append_is_incremental_and_recovers_torn_tail ... ok +test corrupt_archive_reports_server_error_and_namespace_stripping_is_precise ... ok +test archive_roundtrip_preserves_default_namespace_resets ... ok +test smart_enable_and_message_selection ... ok +test equal_cursors_and_normalized_query_jids ... ok +test muc_archive_filters_stanza_kind_instead_of_sender_jid ... ok +test real_archive_sid_and_sender_privacy ... ok +test mam_paging_filters_and_forbidden ... ok +test zero_page_dates_and_invalid_forms ... ok +test extended_ids_ranges_flip_and_metadata ... ok +test namespaces_expiry_and_concurrent_append ... ok + +test result: ok. 13 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/auth.rs (target/debug/deps/auth-5b3c63d14547ba48) + +running 3 tests +test traversal_and_malicious_account_records_rejected ... ok +test create_and_read_hashed_accounts ... ok +test authentication_prepares_names_without_relaxing_account_management ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.87s + + Running tests/cli.rs (target/debug/deps/cli-6ee6e0c0e23d76b2) + +running 2 tests +test server_cannot_be_mistaken_for_running_service ... ok +test conversion_preserves_source_and_refuses_overwrite ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/config.rs (target/debug/deps/config-1282641d140c9884) + +running 12 tests +test include_cycles_rejected ... ok +test component_can_explicitly_enable_global_modules ... ok +test declarations_and_overrides ... ok +test storage_paths_encode_like_prosody ... ok +test include_has_own_global_scope_and_parent_scope_is_restored ... ok +test tables_cannot_recurse_forever ... ok +test supplied_configuration_scopes_arithmetic_and_modules ... ok +test runtime_capabilities_are_unavailable ... ok +test supplied_config_matches_original_prosody_loader ... ok +test malformed_values_rejected ... ok +test all_formats_round_trip_actual_config ... ok +test loops_are_bounded ... ok + +test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/csi.rs (target/debug/deps/csi-92a31451168fba3d) + +running 2 tests +test bounded_queue_flush_delay_activity_and_grace ... ok +test native_importance_rules ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/csi_native.rs (target/debug/deps/csi_native-0e9b124aa7c3050e) + +running 1 test +test native_prosody_csi_importance_matches ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/data_lock.rs (target/debug/deps/data_lock-058fb82c6b41a755) + +running 2 tests +test child_process_probe ... ok +test excludes_another_process_and_releases_on_drop ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/debug_legacy.rs (target/debug/deps/debug_legacy-b04f69283eaa02df) + +running 1 test +test debug_empty_legacy_bind ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + + Running tests/federation_tls.rs (target/debug/deps/federation_tls-b06b682c04611173) + +running 3 tests +test native_prosody_tls_external_bidirectional ... ignored, requires native Prosody cafd74b9c5ea and Lua TLS dependencies +test mutual_tls_external_delivery_and_certificate_identity_rejection ... ok +test reuses_connections_serializes_writes_and_retires_idle_or_closed_peers ... ok + +test result: ok. 2 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 1.14s + + Running tests/gajim_features.rs (target/debug/deps/gajim_features-adc8e988cc8fd2e5) + +running 6 tests +test policy_catalog_selects_default_policy_and_maps_display_and_explicit_label ... ok +test unsupported_security_policy_configuration_returns_explicit_error ... ok +test security_catalog_default_is_a_real_label_and_module_can_be_disabled ... ok +test security_catalogs_serve_both_versions_and_preserve_requested_target ... ok +test actual_example_policy_configuration_exposes_classifications_as_catalog_labels ... ok +test malformed_or_unknown_policy_catalog_schemas_are_rejected ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/http_altconnect.rs (target/debug/deps/http_altconnect-bbe7f2775d33d129) + +running 3 tests +test native_link_order_schemes_and_loaded_module_gates ... ok +test public_prefix_and_host_paths_inherit_independently ... ok +test bad_advertisement_types_and_route_collisions_are_rejected ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/http_transport.rs (target/debug/deps/http_transport-c4efc8c30d83cff4) + +running 10 tests +test bosh_preserves_namespace_inherited_from_body ... ok +test configured_origin_is_enforced ... ok +test transport_paths_and_cors_preflight_match_prosody ... ok +test bosh_session_retries_restart_termination ... ok +test bosh_termination_waits_for_graceful_cleanup_before_reconnect ... ok +test bosh_reorders_two_requests_without_replaying_stanzas ... ok +test websocket_frames_are_independent_xml ... ok +test websocket_disconnect_delivers_eof_without_graceful_close ... ok +test websocket_close_allows_graceful_engine_cleanup ... ok +test bosh_retry_survives_aborted_http_request_without_replaying_stanzas ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + + Running tests/http_upload.rs (target/debug/deps/http_upload-cdc46d4347597b0e) + +running 5 tests +test https_endpoint_uses_real_tls ... ok +test percent_escape_case_and_equivalent_url_spelling ... ok +test persisted_prosody_archive_and_hs256_slot_survive_without_sidecar ... ok +test expired_upload_releases_quota ... ok +test upload_authorization_streaming_ranges_and_restart ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/iq_errors.rs (target/debug/deps/iq_errors-ec7bb749f1288cff) + +running 2 tests +test pep_errors_identify_the_account_and_echo_the_request ... ok +test server_errors_without_target_remain_unaddressed_and_escape_attributes ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/logging.rs (target/debug/deps/logging-024a229065a20f1e) + +running 10 tests +test native_console_shorthands_are_accepted ... ok +test rejects_malformed_configuration ... ok +test filters_appends_escapes_and_redacts ... ok +test json_records_and_bounded_ranges ... ok +test explicit_sink_levels_select_exactly_and_deduplicate ... ok +test overlapping_shorthand_destinations_receive_each_record_once ... ok +test failed_reopen_keeps_all_original_handles ... ok +test shared_sink_unions_levels_and_preserves_distinct_destinations ... ok +test reopen_rotated_files_updates_clones ... ok +test cloned_loggers_serialize_records ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/message_routing.rs (target/debug/deps/message_routing-95f55d0dd6f4b395) + +running 2 tests +test headline_uses_all_nonnegative_available_resources_and_errors_never_fallback ... ok +test unknown_resource_falls_back_to_available_bare_jid ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_access.rs (target/debug/deps/muc_access-d1849f09978afe69) + +running 3 tests +test moderators_cannot_kick_or_demote_other_moderators ... ok +test nested_affiliations_do_not_fall_back_to_obsolete_flat_entries ... ok +test binary_occupant_id_salt_survives_saves_and_restarts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/muc_domain_bans.rs (target/debug/deps/muc_domain_bans-14ddee5a021d8de4) + +running 2 tests +test applying_domain_ban_removes_remote_occupants_and_blocks_rejoin ... ok +test native_domain_bans_block_remote_users_but_allow_explicit_members ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/muc_extensions.rs (target/debug/deps/muc_extensions-5ba345ea905e6264) + +running 10 tests +test broadcast_roles_hide_visitors_and_configuration_announces_transition ... ok +test disabled_optional_vcard_and_occupant_ids_are_not_advertised_or_emitted ... ok +test messages_cannot_spoof_room_identity_hats_or_occupant_identifiers ... ok +test restricted_private_messages_allow_self_and_optional_moderators ... ok +test large_optional_history_does_not_prevent_checkpointing_membership ... ok +test mam_all_rooms_presence_logging_public_queries_and_optional_totals ... ok +test language_config_sections_discovery_and_native_affiliation_hats ... ok +test offline_reserved_affiliates_follow_anonymity_and_roominfo_counts_nicknames ... ok +test recovery_claim_is_durable_and_unrelated_disconnect_keeps_temporary_claims ... ok +test explicit_new_nickname_consumes_old_proof_and_pending_nicks_are_reserved ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_flow.rs (target/debug/deps/muc_flow-de6f85b343315b64) + +running 18 tests +test configured_lock_timeout_expires_without_owner_submission ... ok +test iq_response_requires_expected_sender ... ok +test authorization_and_conflict ... ok +test join_lock_chat_leave ... ok +test multiple_resources_share_nick_and_private_delivery ... ok +test nick_invite_destroy_and_local_creation ... ok +test occupant_id_is_stable_and_spoofed_ids_removed ... ok +test restored_temporary_room_is_deleted_after_last_disconnect ... ok +test persistence_roundtrip ... ok +test switching_to_members_only_removes_unaffiliated_users ... ok +test visitor_voice_request_cannot_spoof_identity_or_grant_itself_voice ... ok +test room_vcard_owner_only_and_avatar_presence ... ok +test preserves_unknown_binary_mixed_room_fields ... ok +test reserved_nickname_blocks_other_users ... ok +test persistent_destroy_creates_prosody_tombstone ... ok +test maintenance_expires_idle_archives_and_tombstones ... ok +test native_muc_archive_type_and_anonymity_are_preserved ... ok +test archive_history_and_permissions ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_history.rs (target/debug/deps/muc_history-0dcd89a1eef6847a) + +running 10 tests +test only_direct_history_children_are_parsed ... ok +test capacity_changes_trim_and_zero_disables_history ... ok +test maxchars_counts_transmitted_utf8_bytes_including_address_and_delay ... ok +test explicit_count_and_zero_limit ... ok +test replay_carries_room_delay_and_recipient ... ok +test ordinary_history_is_bounded_and_returns_oldest_first ... ok +test since_and_seconds_use_stricter_cutoff_inclusive ... ok +test snapshot_restores_last_message_without_adding_another_delay ... ok +test default_count_applies_only_without_valid_explicit_constraints ... ok +test subjects_and_hints_follow_native_history_eligibility ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_native.rs (target/debug/deps/muc_native-1de2cb4fa7044302) + +running 1 test +test actual_prosody_muc_mam_reads_rust_and_writes_native_archive ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/muc_recovery.rs (target/debug/deps/muc_recovery-184e6c1924b0b00b) + +running 7 tests +test claim_is_one_time_and_does_not_change_untrusted_role ... ok +test pending_snapshot_keeps_original_checkpoint_time ... ok +test recovery_claim_requires_exact_authenticated_resource_and_expires ... ok +test native_checkpoint_roundtrip_filters_sensitive_extensions ... ok +test native_last_message_roundtrip_preserves_namespaces_and_rejects_other_rooms ... ok +test oversized_status_never_prevents_membership_checkpoint ... ok +test configuration_fingerprint_is_independent_of_lua_table_iteration_order ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/muc_restart_state.rs (target/debug/deps/muc_restart_state-14d4048acd010890) + +running 3 tests +test unrelated_disconnect_does_not_delete_dormant_temporary_room ... ok +test changed_configuration_invalidates_auto_recovery_without_disclosing_password ... ok +test repeated_restart_keeps_dormant_claim_and_deadline_without_ghost_routing ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_state_native.rs (target/debug/deps/muc_state_native-2a5f6349c617dbee) + +running 1 test +test actual_prosody_restores_rust_checkpoint_and_rust_reads_native_freeze ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pep_client_interop.rs (target/debug/deps/pep_client_interop-3007dfe744090225) + +running 7 tests +test native_component_notification_type_applies_to_events_and_subscription_replay ... ok +test missing_own_omemo_bundle_returns_recoverable_item_not_found ... ok +test publish_options_match_implicit_notification_defaults_on_old_nodes ... ok +test avatar_and_omemo_publish_options_support_readback_and_caps_notifications ... ok +test migrated_native_configuration_can_roundtrip_access_model_edit ... ok +test native_itemreply_controls_publisher_visibility_in_reads_and_replays ... ok +test native_open_and_subscriber_publish_models_preserve_outcast_and_blocking_checks ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.21s + + Running tests/pep_diagnostics.rs (target/debug/deps/pep_diagnostics-56ce2b171d4d4544) + +running 6 tests +test nonempty_xml_can_be_an_empty_android_device_list ... ok +test avatar_classification_does_not_inspect_or_expose_avatar_data ... ok +test arbitrary_node_payload_and_wrong_namespace_do_not_leak ... ok +test payloadless_notification_is_visible_without_identifiers ... ok +test only_first_direct_item_is_parsed_and_protocols_are_distinct ... ok +test signed_integer_limits_and_duplicates_match_java_parser ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pep_discovery.rs (target/debug/deps/pep_discovery-e4c692a47f70144b) + +running 4 tests +test standalone_component_does_not_claim_pep_automatic_subscriptions ... ok +test account_discovery_does_not_intercept_server_or_resource_caps ... ok +test account_disco_advertises_pep_without_requiring_a_node ... ok +test node_discovery_keeps_node_identity_errors_and_access_checks ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.11s + + Running tests/pep_item_order.rs (target/debug/deps/pep_item_order-0557272b4979fb11) + +running 2 tests +test native_prosody_orders_retained_pep_items_newest_first ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA +test first_pep_item_is_current_device_list_even_with_retained_old_items ... ok + +test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running tests/preauth.rs (target/debug/deps/preauth-b98a0aa3fb33c1a3) + +running 10 tests +test duplicate_or_wrong_namespace_tokens_do_not_qualify ... ok +test unknown_tokens_do_not_create_invite_stores_and_disabled_accounts_cannot_issue ... ok +test standard_adhoc_command_returns_contact_uri ... ok +test prosody_numeric_expiry_is_consumed_and_other_entries_preserved ... ok +test concurrent_redemption_has_exactly_one_winner ... ok +test unknown_client_token_is_preserved_on_remote_and_pending_delivery ... ok +test identity_scope_does_not_burn_on_wrong_sender_or_leak_to_prosody ... ok +test invitations_expire_are_single_use_and_owner_scoped ... ok +test valid_token_autoapproves_and_sends_reciprocal_request ... ok +test local_invitation_updates_both_rosters_without_granting_reverse_subscription ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/preauth_native.rs (target/debug/deps/preauth_native-6d831575ad50feb3) + +running 1 test +test native_prosody_invitation_tokens_roundtrip ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pubsub_native.rs (target/debug/deps/pubsub_native-799d09bde6bd236f) + +running 1 test +test native_prosody_pep_reads_rust_and_rust_reads_native ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/pubsub_persistence.rs (target/debug/deps/pubsub_persistence-f8898a70c1d9229b) + +running 18 tests +test modern_bookmarks_notify_legacy_capability ... ok +test native_string_max_items_satisfies_numeric_publish_precondition ... ok +test configurable_limits_component_creation_and_publisher ... ok +test configure_requires_owner ... ok +test capability_discovery_replays_and_filters_notifications ... ok +test nonpersistent_items_stay_in_memory_and_can_be_made_persistent ... ok +test full_jid_block_applies_to_reads_subscription_replays_and_events ... ok +test reads_prosody_legacy_fixture ... ok +test open_model_and_notifications ... ok +test pep_persists_namespaces_and_enforces_owner ... ok +test publish_options_conflict_is_recoverable_and_preserves_item ... ok +test roster_presence_access_and_foreign_subscription_denied ... ok +test remote_unavailable_is_scoped_to_addressed_owner_and_cancels_pending ... ok +test remote_caps_replay_and_open_node_privacy_revoke ... ok +test subscription_options_disable_events_and_require_subscriber ... ok +test payload_policy_applies_to_subscription_and_caps_replays ... ok +test native_notification_fields_and_retract_policy_are_independent ... ok +test component_default_retention_matches_prosody_twenty_items ... ok + +test result: ok. 18 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.24s + + Running tests/push.rs (target/debug/deps/push-d6e33d3d0596e2e2) + +running 7 tests +test native_push_body_lookup_excludes_extensions ... ignored, requires PROSODY_REFERENCE_SOURCE and PROSODY_REFERENCE_LUA +test foreign_account_cannot_register ... ok +test push_summary_never_copies_foreign_namespace_body ... ok +test queue_coalesces_and_importance_matches_native ... ok +test stable_client_registration_and_targeted_notification ... ok +test reenable_cancels_old_errors_and_failure_counts_are_per_account ... ok +test registrations_notifications_and_authenticated_errors ... ok + +test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.08s + + Running tests/reference_prosody.rs (target/debug/deps/reference_prosody-9b33cba6508d6c00) + +running 1 test +test real_prosody_storage_and_auth_roundtrip ... ignored, requires native supplied Prosody source; run scripts/prosody_compat/verify.sh + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/resource_sessions.rs (target/debug/deps/resource_sessions-02f64f1ac2a22878) + +running 7 tests +test increment_skips_occupied_suffixes_and_preserves_sessions ... ok +test invalid_resume_owner_or_ack_does_not_evict_active_session ... ok +test kick_new_leaves_existing_session_unchanged ... ok +test replacement_revokes_hibernating_resume_and_survives_old_cleanup ... ok +test random_conflicts_get_distinct_resources_for_same_owner ... ok +test concurrent_default_binds_replace_atomically ... ok +test active_resume_preserves_state_and_replaces_transport_notifier ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/sasl_accounts.rs (target/debug/deps/sasl_accounts-5d8cb6db7592204f) + +running 1 test +test disabled_legacy_and_unicode_accounts ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.54s + + Running tests/storage_accounts.rs (target/debug/deps/storage_accounts-6b03eae10230e3ea) + +running 20 tests +test broken_roster_is_not_overwritten ... ok +test account_purge_refuses_symlink_store_without_touching_other_files ... ok +test domain_roster_and_block_jid_normalization ... ok +test malformed_legacy_vcard_and_private_bookmarks_rejected ... ok +test blocklist_enforcement_api ... ok +test malformed_roster_iq_never_mutates ... ok +test inbound_subscription_requires_pending_to_grant_to ... ok +test private_and_vcard_authorization ... ok +test privacy_migration_only_default_deny_jids ... ok +test reads_timestamp_blocklist_and_keeps_metadata ... ok +test roster_empty_name_and_foreign_group_ignored ... ok +test roster_preserves_subscription_and_metadata ... ok +test purge_account_exact_files_preserves_others_and_extras ... ok +test subscription_handshake_and_cancel ... ok +test legacy_bookmarks_respect_host_limit_and_read_over_256 ... ok +test bookmarks_private_pep_bidirectional_and_private ... ok +test legacy_migration_clears_source_after_success ... ok +test vcard_pep_conversion_and_avatar ... ok +test password_change_after_purge_never_recreates_account ... ok +test creation_and_purge_are_serialized ... ok + +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.22s + + Running tests/storage_compat.rs (target/debug/deps/storage_compat-fe6cff3b2d381659) + +running 6 tests +test empty_namespace_preserved ... ok +test stanza_roundtrip ... ok +test mixed_binary_roundtrip ... ok +test path_and_atomic_update ... ok +test rejects_cycles_code_and_bad_files ... ok +test concurrent_updates_do_not_lose_entries ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/tls_binding.rs (target/debug/deps/tls_binding-5bf92a1e0024d0ed) + +running 4 tests +test unknown_and_eddsa_algorithms_fail_closed ... ok +test malformed_der_and_pss_parameters_fail_closed ... ok +test ecdsa_uses_certificate_signature_hash ... ok +test rsa_signature_digest_and_weak_hash_upgrade ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.64s + + Running tests/tls_plus_network.rs (target/debug/deps/tls_plus_network-d375c1fb4af9d25c) + +running 2 tests +test disabling_plus_hides_bindings_and_accepts_scram_y_with_mechanisms_only_ssdp ... ok +test starttls_scram_plus_binds_real_tls_and_authenticates_ssdp_sasl1_and_sasl2 ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.41s + + Doc-tests rust_xmpp + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Finished `dev` profile [unoptimized] target(s) in 0.12s + Finished `release` profile [optimized] target(s) in 0.11s +PASS: integrated TCP roster/subscriptions, client IQ routing, CSI buffering/keepalive flush, MAM, blocking, OMEMO PEP, MUC, offline replay and SM resumption +PASS: graceful SIGTERM shutdown +PASS: independent SCRAM-SHA-1 over TLS, SASL1/SASL2, SSDP verification, server signatures and wrong-password rejection +PASS: independent SCRAM-SHA-256 over TLS, SASL1/SASL2, SSDP verification, server signatures and wrong-password rejection +PASS: Android TLS1.2 selector, TLS1.2/1.3 SCRAM SHA1/256 PLUS, SASL1/2, SSDP on/off and invalid binding rejection +PASS: JSON file logging, TLS/auth/bind correlation, rejection reasons, secret redaction and SIGHUP rotation +PASS: 20 TLS1.2/1.3 SASL1/2 cases: prepared PLAIN identities and canonical binds, invalid authz rejection, SCRAM abort/retry and active exchange restart +PASS: WSS SCRAM-SHA-256 SASL1/SASL2 with GS2 y, absent CB/PLUS, SSDP without CB separator, server signatures, bind and ping +PASS: HTTPS-only configured interfaces (127.0.0.1,::1), verified TLS, reproduced original Origin 403 then allowed Converse Origin 101, xmpp subprotocol, both WS paths, RFC 7395 open/features/close and private HTTP diagnostics +PASS: custom WS/BOSH routes, default-route 404, exact Origin policy, verified TLS proxy rewrite, trusted HTTPS metadata, untrusted/spoofed metadata denial, IPv6 when available, authenticated WS/BOSH bind/ping in legacy and RFC7239 modes and cached BOSH RID retry +PASS: XEP-0156 XRD/JRD parity, CORS, safe methods, ignored JSONP, host checks, secure URL preference, custom mounts/public prefixes, module and advertisement gates, HTTP fallback and verified WSS framing +PASS: SASL2 Bind2 TLS flow, inline SM/CSI/Carbons, private random resource, malformed requests, authentication failure and fallback +PASS: bind2-disabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-disabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-enabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: bind2-enabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: sm-disabled, SSDP=False, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: sm-disabled, SSDP=True, SCRAM-SHA-1/PLAIN success and wrong-password rejection +PASS: enabled-False-registration-False +PASS: enabled-False-registration-True +PASS: enabled-True-registration-False +PASS: enabled-True-registration-True +PASS: Gajim domain/account discovery, both label catalogs, authenticated password changes, IBR registration gates and fresh logins +PASS: SASL2 inline SM active/EOF resume, authenticated owner/id/h checks, replay ordering, full JID, Bind2 fallback and malformed requests +PASS: TLS account/domain discovery with PEP and push enabled/disabled; real push enable, publish options, per-node and service-wide disable +PASS: TLS Monocles/Conversations PEP first-run, no-to publishing, owner config recovery, migrated metadata, device item selectors and restart persistence +PASS: Android first-item device-list parsing sees latest publish and replacement before and after restart +PASS: STARTTLS PEP replay/live diagnostics, legacy/OMEMO2, payloadless notifications with intact IQ payloads, and private values excluded +PASS: TLS invitation command discovery, preapproval, single-use mutual roster, replay denial, unknown preauth forwarding/offline pending, account isolation +PASS: STARTTLS MUC SIGTERM restart restores live temporary/persistent rooms, exact full-JID membership, roles, presence and groupchat routing; rejects automatic cross-resource/account claims and nickname theft +PASS: TLS conflict_resolve=default, live and hibernating SM resources +PASS: TLS conflict_resolve=kick_old, live and hibernating SM resources +PASS: TLS conflict_resolve=kick_new, live and hibernating SM resources +PASS: TLS conflict_resolve=increment, live and hibernating SM resources +PASS: TLS conflict_resolve=random, live and hibernating SM resources +PASS: integrated WSS + HTTPS BOSH SASL/bind, CSI, inherited BOSH namespaces, roster, bidirectional chat, upload IQ/PUT/GET/reuse, graceful SM close and same-resource relogin under kick_new +PASS: integrated WSS + HTTPS BOSH SASL/bind, CSI, inherited BOSH namespaces, roster, bidirectional chat, upload IQ/PUT/GET/reuse, graceful SM close and same-resource relogin under kick_new +PASS: configured upload HTTP path with $host through integrated listener +PASS: multiple upload hosts/shared-host paths, discovery, quotas, JWT isolation, persistence, transport composition and conflict validation +PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation +PASS upload migration: local -> S3 -> S3 -> local, hashes, original URLs/JWT/HEAD, lock, dry-run, fail-closed resume +{ + "total_load_wall_seconds": 7.094, + "requested_resources": 500, + "authenticated_bound_resources": 500, + "connection_setup_concurrency": 50, + "setup_wall_seconds": 0.671, + "setup_latency": { + "samples": 500, + "median_ms": 62.192, + "p95_ms": 83.883, + "max_ms": 114.563 + }, + "ping_rounds": 10, + "ping_round_trip": { + "samples": 5000, + "median_ms": 13.502, + "p95_ms": 19.31, + "max_ms": 24.054 + }, + "accounts": 100, + "message_rounds": 10, + "messages_sent": 5000, + "messages_received": 5000, + "message_batch_seconds": 4.134, + "metrics_before": { + "rss_kib": 8048, + "cpu_seconds": 0.0, + "fd_count": 11 + }, + "metrics_connected": { + "rss_kib": 26804, + "cpu_seconds": 1.02, + "fd_count": 511 + }, + "metrics_after_workload": { + "rss_kib": 27400, + "cpu_seconds": 8.85, + "fd_count": 511 + }, + "metrics_two_seconds_after_disconnect": { + "rss_kib": 27464, + "cpu_seconds": 8.9, + "fd_count": 11 + }, + "sampled_peak_rss_kib": 27464, + "fd_delta_after_disconnect": 0, + "failures": [], + "scope": "One loopback host, 100 accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.", + "mam": { + "archive_files": 100, + "records": 10000, + "expected_records": 10000 + } +} + Finished `test` profile [unoptimized] target(s) in 0.08s + Running tests/csi_native.rs (target/debug/deps/csi_native-0e9b124aa7c3050e) + +running 1 test +test native_prosody_csi_importance_matches ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/muc_native.rs (target/debug/deps/muc_native-1de2cb4fa7044302) + +running 1 test +test actual_prosody_muc_mam_reads_rust_and_writes_native_archive ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + Running tests/muc_state_native.rs (target/debug/deps/muc_state_native-2a5f6349c617dbee) + +running 1 test +test actual_prosody_restores_rust_checkpoint_and_rust_reads_native_freeze ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/pep_item_order.rs (target/debug/deps/pep_item_order-0557272b4979fb11) + +running 1 test +test native_prosody_orders_retained_pep_items_newest_first ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.12s + + Running tests/preauth_native.rs (target/debug/deps/preauth_native-6d831575ad50feb3) + +running 1 test +test native_prosody_invitation_tokens_roundtrip ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.14s + + Running tests/pubsub_native.rs (target/debug/deps/pubsub_native-799d09bde6bd236f) + +running 1 test +test native_prosody_pep_reads_rust_and_rust_reads_native ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.14s + + Running tests/push.rs (target/debug/deps/push-d6e33d3d0596e2e2) + +running 1 test +test native_push_body_lookup_excludes_extensions ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 6 filtered out; finished in 0.00s + + Running tests/reference_prosody.rs (target/debug/deps/reference_prosody-9b33cba6508d6c00) + +running 1 test +PROSODY_NATIVE_ROUNDTRIP_OK SHA-1 +PROSODY_NATIVE_FINAL_OK SHA-1 +PROSODY_NATIVE_ROUNDTRIP_OK SHA-256 +PROSODY_NATIVE_FINAL_OK SHA-256 +test real_prosody_storage_and_auth_roundtrip ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.98s + + Finished `test` profile [unoptimized] target(s) in 0.07s + Running tests/federation_tls.rs (target/debug/deps/federation_tls-b06b682c04611173) + +running 1 test +test native_prosody_tls_external_bidirectional ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 2 filtered out; finished in 0.66s + diff --git a/docs/validation-results-0.7.7.json b/docs/validation-results-0.7.7.json new file mode 100644 index 0000000..b1f5de3 --- /dev/null +++ b/docs/validation-results-0.7.7.json @@ -0,0 +1,93 @@ +{ + "version": "0.7.7", + "date_utc": "2026-10-20", + "ordinary_tests": { + "passed": 262, + "failed": 0, + "ignored": 11 + }, + "fmt": "passed", + "clippy": "passed", + "release": "passed", + "native_module_tests": 8, + "native_federation_tests": 1, + "http_altconnect": { + "native_reference": "supplied plugins/mod_http_altconnect.lua cafd74b9c5ea", + "unit_tests": 3, + "network_configurations": 10, + "checks": [ + "HTTP and verified HTTPS XRD/JRD parity", + "MIME/CORS/GET/HEAD/OPTIONS and method405", + "query ignored, no JSONP", + "advertise_bosh/websocket combinations and loaded transport gates", + "discovery disabled while WSS still works", + "custom altconnect/BOSH/WS mounts and public URL prefix", + "local/global inheritance and HTTPS preference", + "HTTP native compatibility fallback", + "HTTP Host denial and overflowing port rejection", + "config type/path collision rejection" + ], + "public_urls": "configuration only; no request Host/Forwarded interpolation", + "standard_caveat": "XEP0156 requires HTTPS metadata and HTTPS/WSS href; supplied-native HTTP/WS fallback retained and documented" + }, + "load": { + "total_load_wall_seconds": 7.447, + "requested_resources": 500, + "authenticated_bound_resources": 500, + "connection_setup_concurrency": 50, + "setup_wall_seconds": 0.653, + "setup_latency": { + "samples": 500, + "median_ms": 57.419, + "p95_ms": 87.729, + "max_ms": 107.7 + }, + "ping_rounds": 10, + "ping_round_trip": { + "samples": 5000, + "median_ms": 14.618, + "p95_ms": 19.347, + "max_ms": 30.719 + }, + "accounts": 100, + "message_rounds": 10, + "messages_sent": 5000, + "messages_received": 5000, + "message_batch_seconds": 4.483, + "metrics_before": { + "rss_kib": 7792, + "cpu_seconds": 0.0, + "fd_count": 11 + }, + "metrics_connected": { + "rss_kib": 26488, + "cpu_seconds": 1.17, + "fd_count": 511 + }, + "metrics_after_workload": { + "rss_kib": 27100, + "cpu_seconds": 8.93, + "fd_count": 511 + }, + "metrics_two_seconds_after_disconnect": { + "rss_kib": 27228, + "cpu_seconds": 8.95, + "fd_count": 11 + }, + "sampled_peak_rss_kib": 27228, + "fd_delta_after_disconnect": 0, + "failures": [], + "scope": "One loopback host, 100 accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.", + "mam": { + "archive_files": 100, + "records": 10000, + "expected_records": 10000 + } + }, + "release_binary_sha256": "4678c35c9ad53f2632f0f0245d842589a15dd4e02b9cb172f15dd7b035422294", + "limitations": [ + "Real production proxy/discovery/client UI not run; operator must enable module and publish well-known on HTTPS domain", + "Exactly one runtime VirtualHost; no Host Meta2/XEP0487", + "Previous parity/PEP/DNSSEC limitations unchanged; COMPATIBILITY.md" + ] +} diff --git a/docs/validation-results-0.7.8.json b/docs/validation-results-0.7.8.json new file mode 100644 index 0000000..884315e --- /dev/null +++ b/docs/validation-results-0.7.8.json @@ -0,0 +1,126 @@ +{ + "version": "0.7.8", + "date_utc": "2026-10-20", + "ordinary_tests": { + "passed": 262, + "failed": 0, + "ignored": 11 + }, + "fmt": "passed", + "clippy": "passed", + "release": "passed", + "native_module_tests": 8, + "native_federation_tests": 1, + "upload": { + "multi_component_network": "passed", + "independent_s3_protocol_fixture": "passed", + "large_https_local": { + "status": "passed", + "requested_bytes": 10737418240, + "python_chunk_bytes": 1048576, + "transport": "verified TLS HTTPS", + "storage": "local streaming file", + "rss_sample_interval_seconds": 0.05, + "free_disk_before_bytes": 14037065728, + "max_rss_bytes": { + "server": 18079744, + "client": 21168128 + }, + "rss_proc_status_paths": { + "server": "/proc/68012/status", + "client": "/proc/self/status" + }, + "put_bytes": 10737418240, + "put_seconds": 14.389516601999276, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "stored_bytes": 10737418240, + "allocated_bytes": 10737422336, + "free_disk_after_put_bytes": 2734645248, + "head_status": 200, + "get_bytes": 10737418240, + "get_read_calls": 10241, + "get_seconds": 69.22072244599985, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "range_status": 206, + "range_offset": 4294967289, + "reuse_status": 409, + "over_quota_condition": "not-acceptable", + "temporary_files_cleaned": true, + "free_disk_after_cleanup_bytes": 12774170624, + "total_seconds": 84.19309745400096 + }, + "s3_cases": [ + "independent SigV4 and session token", + "path/query escaping", + "multipart progress before input EOF", + "GET/HEAD/range/conditionals", + "init/part/completion/200-Error failures and abort/retry", + "committed object with lost completion response, live HEAD recovery", + "SIGKILL durable multipart recovery", + "old pending completed object before startup prune", + "quota, single use, expiry delete, credentials redaction, local backend isolation" + ] + }, + "load": { + "total_load_wall_seconds": 7.782, + "requested_resources": 500, + "authenticated_bound_resources": 500, + "connection_setup_concurrency": 50, + "setup_wall_seconds": 0.63, + "setup_latency": { + "samples": 500, + "median_ms": 57.454, + "p95_ms": 86.829, + "max_ms": 95.615 + }, + "ping_rounds": 10, + "ping_round_trip": { + "samples": 5000, + "median_ms": 14.241, + "p95_ms": 23.05, + "max_ms": 27.298 + }, + "accounts": 100, + "message_rounds": 10, + "messages_sent": 5000, + "messages_received": 5000, + "message_batch_seconds": 4.835, + "metrics_before": { + "rss_kib": 8056, + "cpu_seconds": 0.0, + "fd_count": 11 + }, + "metrics_connected": { + "rss_kib": 26868, + "cpu_seconds": 1.22, + "fd_count": 511 + }, + "metrics_after_workload": { + "rss_kib": 27456, + "cpu_seconds": 9.05, + "fd_count": 511 + }, + "metrics_two_seconds_after_disconnect": { + "rss_kib": 27648, + "cpu_seconds": 9.09, + "fd_count": 11 + }, + "sampled_peak_rss_kib": 27648, + "fd_delta_after_disconnect": 0, + "failures": [], + "scope": "One loopback host, 100 accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.", + "mam": { + "archive_files": 100, + "records": 10000, + "expected_records": 10000 + } + }, + "release_binary_sha256": "f3226a6151122601d6a6e74ea2bcbbd24a41c29087debe223098bbf37e4126d1", + "limitations": [ + "S3 tested with independent loopback protocol fixture, not AWS/MinIO deployment", + "Real 10 GiB test uses local backend; no real 10 GiB object-store upload", + "Bucket lifecycle AbortIncompleteMultipartUpload required for initiation crash window before upload ID journal", + "No automatic storage migration or arbitrary external driver", + "Production DNS, proxy, client UI and previous parity limitations unchanged" + ] +} diff --git a/docs/validation-results.json b/docs/validation-results.json index b1f5de3..7823d9e 100644 --- a/docs/validation-results.json +++ b/docs/validation-results.json @@ -1,8 +1,8 @@ { - "version": "0.7.7", + "version": "0.7.9", "date_utc": "2026-10-20", "ordinary_tests": { - "passed": 262, + "passed": 265, "failed": 0, "ignored": 11 }, @@ -11,70 +11,108 @@ "release": "passed", "native_module_tests": 8, "native_federation_tests": 1, - "http_altconnect": { - "native_reference": "supplied plugins/mod_http_altconnect.lua cafd74b9c5ea", - "unit_tests": 3, - "network_configurations": 10, - "checks": [ - "HTTP and verified HTTPS XRD/JRD parity", - "MIME/CORS/GET/HEAD/OPTIONS and method405", - "query ignored, no JSONP", - "advertise_bosh/websocket combinations and loaded transport gates", - "discovery disabled while WSS still works", - "custom altconnect/BOSH/WS mounts and public URL prefix", - "local/global inheritance and HTTPS preference", - "HTTP native compatibility fallback", - "HTTP Host denial and overflowing port rejection", - "config type/path collision rejection" - ], - "public_urls": "configuration only; no request Host/Forwarded interpolation", - "standard_caveat": "XEP0156 requires HTTPS metadata and HTTPS/WSS href; supplied-native HTTP/WS fallback retained and documented" + "release_binary_sha256": "24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c", + "upload": { + "minio_10gib": { + "ok": true, + "minio_version": "minio-sandbox-compat version DEVELOPMENT.GOGET (commit-id=DEVELOPMENT.GOGET)\nRuntime: go1.24.3 linux/amd64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-0000 MinIO, Inc.", + "minio_binary_sha256": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "binary": "/workspace/scratch/362577cdf2e0/recovered/rust-xmpp/target/release/rust-xmpp", + "binary_version": "rust-xmpp 0.7.9 — experimental XMPP runtime", + "binary_sha256": "24c0dca637e7ce4e238ef6becad16bb5a18a33546554b0eecf310fd8927f021c", + "size": 10737418240, + "qualifies_over_5gib": true, + "minio_test_only_compatibility_patch": { + "sha256": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5", + "diff": "--- a/cmd/net.go\n+++ b/cmd/net.go\n@@ -24,6 +24,7 @@\n \t\"runtime\"\n \t\"sort\"\n \t\"strings\"\n+\t\"syscall\"\n \n \t\"github.com/minio/minio-go/v7/pkg/set\"\n \t\"github.com/minio/minio/internal/config\"\n@@ -54,6 +55,10 @@\n // mustGetLocalIPs returns IPs of local interface\n func mustGetLocalIPs() (ipList []net.IP) {\n \tifs, err := net.Interfaces()\n+\t// TEST ONLY: the sandbox denies netlink enumeration; all listeners bind loopback.\n+\tif errors.Is(err, syscall.EPERM) {\n+\t\treturn []net.IP{net.ParseIP(\"127.0.0.1\")}\n+\t}\n \tlogger.FatalIf(err, \"Unable to get IP addresses of this host\")\n \n \tfor _, interf := range ifs {\n", + "limitation": "Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream." + }, + "max_rss_bytes": { + "server": 106057728, + "client": 24338432, + "minio": 498958336 + }, + "put_bytes": 10737418240, + "put_seconds": 48.45253416699961, + "sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "minio_etag": "958f42d583ac40ff209eb865c5eb4bd6-1280", + "multipart_parts": 1280, + "minio_allocated_bytes": 10747953152, + "free_disk_after_put_bytes": 15180328960, + "get_bytes": 10737418240, + "get_sha256": "e4c296c41d3a9605f0348961e41ed835b5f42c220751ed2269a726951546c935", + "get_seconds": 13.709708632000911, + "range_offsets": [ + 0, + 4294967289, + 5368709113, + 10737418208 + ], + "restart_range_ok": true, + "restart_reuse_status": 409, + "crash_multipart_cleanup_ok": true, + "minio_upstream_provenance": { + "source_url": "https://github.com/minio/minio/archive/refs/tags/RELEASE.2025-04-22T22-12-26Z.tar.gz", + "upstream_release": "RELEASE.2025-04-22T22-12-26Z", + "build": "Go1.24.3 go build -p4; no dependency changes", + "sha256": { + "/tmp/minio-source.tar.gz": "1a63209927b9d1fb239b2a776b7b01d1902eff19379a108f4bc7005edcc0968c", + "/tmp/minio-genuine": "3dd5d6756ad17c599fbc1c6f6a2f5b590242c049522e848c700949dbf8e60ad9", + "/tmp/minio-sandbox-compat": "8bbb9bccfbac69869ab499c39cfd8731b2b770b86ccf37cca1c8267556ad9bf0", + "/tmp/minio-sandbox-compat.patch": "e60592a65273f28a1375eb6ecbc0966f7fbd2ba7567d2ffedc13ffd95ed224f5" + }, + "compatibility_limitation": "Only net.Interfaces EPERM becomes loopback127.0.0.1; all S3/auth/multipart/storage source unchanged." + } + }, + "migration_network": "passed: local -> S3 A -> S3 B -> local, dry-run, lock, native metadata/source preservation, interrupted copy/resume, native final file with incomplete cache", + "s3_fault_network": "passed: SigV4, paginated exact-key orphan cleanup, foreign prefix safety, missing UploadId crash recovery, conditional Complete, ambiguous completion quarantine and late recovery", + "claim_race_regression": "passed: atomic fresh snapshot and active claim retains latest completion/quarantine flags" }, "load": { - "total_load_wall_seconds": 7.447, + "total_load_wall_seconds": 7.094, "requested_resources": 500, "authenticated_bound_resources": 500, "connection_setup_concurrency": 50, - "setup_wall_seconds": 0.653, + "setup_wall_seconds": 0.671, "setup_latency": { "samples": 500, - "median_ms": 57.419, - "p95_ms": 87.729, - "max_ms": 107.7 + "median_ms": 62.192, + "p95_ms": 83.883, + "max_ms": 114.563 }, "ping_rounds": 10, "ping_round_trip": { "samples": 5000, - "median_ms": 14.618, - "p95_ms": 19.347, - "max_ms": 30.719 + "median_ms": 13.502, + "p95_ms": 19.31, + "max_ms": 24.054 }, "accounts": 100, "message_rounds": 10, "messages_sent": 5000, "messages_received": 5000, - "message_batch_seconds": 4.483, + "message_batch_seconds": 4.134, "metrics_before": { - "rss_kib": 7792, + "rss_kib": 8048, "cpu_seconds": 0.0, "fd_count": 11 }, "metrics_connected": { - "rss_kib": 26488, - "cpu_seconds": 1.17, + "rss_kib": 26804, + "cpu_seconds": 1.02, "fd_count": 511 }, "metrics_after_workload": { - "rss_kib": 27100, - "cpu_seconds": 8.93, + "rss_kib": 27400, + "cpu_seconds": 8.85, "fd_count": 511 }, "metrics_two_seconds_after_disconnect": { - "rss_kib": 27228, - "cpu_seconds": 8.95, + "rss_kib": 27464, + "cpu_seconds": 8.9, "fd_count": 11 }, - "sampled_peak_rss_kib": 27228, + "sampled_peak_rss_kib": 27464, "fd_delta_after_disconnect": 0, "failures": [], "scope": "One loopback host, 100 accounts with distinct resources, TLS+PLAIN, ping and directed messages; not a production benchmark or proof of no memory leaks.", @@ -84,10 +122,12 @@ "expected_records": 10000 } }, - "release_binary_sha256": "4678c35c9ad53f2632f0f0245d842589a15dd4e02b9cb172f15dd7b035422294", "limitations": [ - "Real production proxy/discovery/client UI not run; operator must enable module and publish well-known on HTTPS domain", - "Exactly one runtime VirtualHost; no Host Meta2/XEP0487", - "Previous parity/PEP/DNSSEC limitations unchanged; COMPATIBILITY.md" + "MinIO upstream RELEASE.2025-04-22T22-12-26Z with disclosed five-line EPERM loopback interface enumeration adapter; S3/auth/storage/multipart code unchanged", + "Single-node loopback MinIO, HTTP gateway-to-store and verified HTTPS client-to-gateway; no AWS account or distributed/production storage qualification", + "Missing/corrupt ownership metadata cannot identify arbitrary bucket orphans; lifecycle remains defense in depth", + "Ambiguous completion with HEAD404 quarantines slot until a trustworthy remote outcome; no automatic destructive reset", + "Migration is offline, source preserving, requires temporary disk up to one largest file; no live migration", + "Previous full Prosody parity / production client / DNSSEC limits unchanged" ] } diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index bc029af..b365742 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -30,6 +30,13 @@ python3 tests/muc_restart_network.py target/release/rust-xmpp python3 tests/resource_rebind.py target/release/rust-xmpp python3 scripts/integration_http.py python3 scripts/integration_http.py --custom-upload-path +python3 tests/upload_multi_network.py --binary target/release/rust-xmpp +python3 tests/upload_s3_network.py target/release/rust-xmpp +python3 tests/upload_migration_network.py target/release/rust-xmpp +# Opt-in real object-store run: default payload is 10 GiB, not a fixture. +if [[ -n ${MINIO_BINARY:-} ]]; then + python3 tests/upload_minio_network.py --binary target/release/rust-xmpp --minio "$MINIO_BINARY" +fi if [[ ${XMPP_LOAD_CHECK:-0} == 1 ]]; then python3 scripts/load_smoke.py --clients 500 --accounts 100 --rounds 10 --message-rounds 10 --archive fi diff --git a/src/http_config.rs b/src/http_config.rs index d2c371b..f4a9f34 100644 --- a/src/http_config.rs +++ b/src/http_config.rs @@ -28,7 +28,7 @@ fn string(options: &Options, key: &str) -> Result> { }) .transpose() } -fn valid_authority(authority: &axum::http::uri::Authority) -> bool { +pub(crate) fn valid_authority(authority: &axum::http::uri::Authority) -> bool { let raw = authority.as_str(); if raw.contains('@') { return false; diff --git a/src/http_service.rs b/src/http_service.rs index ab13b3e..a3f4668 100644 --- a/src/http_service.rs +++ b/src/http_service.rs @@ -1,6 +1,7 @@ //! Streaming XEP-0363 upload endpoint. Slots are single-use and bounded before streaming. use crate::{ logging::{Level, Logger}, + object_store::{ObjectStore, PART_SIZE, UploadBackendConfig}, storage, wire::escape, }; @@ -56,13 +57,29 @@ struct Record { token: String, complete: bool, #[serde(default)] + upload_id: Option, + // Durable before CreateMultipartUpload: UploadId may be lost in a crash. + #[serde(default)] + multipart_intent: bool, + #[serde(default)] + completion_attempted: bool, + #[serde(default)] xml: Option, } +#[derive(Debug)] +struct CompletionUncertain; +impl std::fmt::Display for CompletionUncertain { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("S3 completion outcome uncertain; reservation quarantined for reconciliation") + } +} +impl std::error::Error for CompletionUncertain {} #[derive(Clone)] pub struct UploadService { config: Arc, records: Arc>>, active: Arc>>, + backend: Option>, } pub struct Slot { pub url: String, @@ -105,6 +122,13 @@ fn url_decode(value: &str) -> Option> { } impl UploadService { pub fn new(config: UploadConfig) -> Result { + Self::new_with_backend(config, UploadBackendConfig::Local) + } + pub fn new_with_backend(config: UploadConfig, backend: UploadBackendConfig) -> Result { + let backend = match backend { + UploadBackendConfig::Local => None, + UploadBackendConfig::S3(c) => Some(Arc::new(ObjectStore::new(c)?)), + }; ensure!( config.base_url.starts_with("https://") || config.base_url.starts_with("http://"), "invalid upload base URL" @@ -131,7 +155,24 @@ impl UploadService { .attribute("size") .ok_or_else(|| anyhow::anyhow!("upload size missing"))? .parse()?; - let token = records.remove(&e.id).map(|r| r.token).unwrap_or_default(); + let cached = records.remove(&e.id); + let complete = cached.as_ref().is_some_and(|r| { + r.complete && r.owner == e.peer && r.filename == filename && r.size == size + }); + let upload_id = cached + .as_ref() + .filter(|r| r.owner == e.peer && r.filename == filename && r.size == size) + .and_then(|r| r.upload_id.clone()); + let multipart_intent = cached.as_ref().is_some_and(|r| { + r.owner == e.peer && r.filename == filename && r.size == size && r.multipart_intent + }); + let completion_attempted = cached.as_ref().is_some_and(|r| { + r.owner == e.peer + && r.filename == filename + && r.size == size + && r.completion_attempted + }); + let token = cached.map(|r| r.token).unwrap_or_default(); authoritative.insert( e.id, Record { @@ -144,20 +185,151 @@ impl UploadService { .into(), created: e.when as u64, token, - complete: false, + complete, + upload_id, + multipart_intent, + completion_attempted, xml: Some(e.xml), }, ); } records = authoritative; + let identity = backend + .as_ref() + .map(|b| b.identity()) + .unwrap_or_else(|| serde_json::json!({"backend":"local"})); + let identity_path = config + .root + .join(storage::encode_name(&config.host)) + .join("rust_upload_backend.json"); + if identity_path.exists() { + let previous: serde_json::Value = + serde_json::from_slice(&std::fs::read(&identity_path)?)?; + ensure!( + records.is_empty() || previous == identity, + "upload backend changed while slots still exist" + ); + } else { + ensure!( + backend.is_none() || records.is_empty(), + "existing upload slots require an explicit storage migration" + ); + } + storage::atomic_write(&identity_path, &serde_json::to_vec(&identity)?)?; let s = Self { config: Arc::new(config), records: Arc::new(Mutex::new(records)), active: Arc::new(Mutex::new(BTreeSet::new())), + backend, }; - s.prune()?; + // Remote pending records must be reconciled before the abandoned-slot + // timeout is applied. The caller awaits recover_backend before listening. + if s.backend.is_none() { + s.prune()?; + } Ok(s) } + /// Recover durable multipart state before enabling listeners or slot requests. + pub async fn recover_backend(&self) -> Result<()> { + if self.backend.is_none() { + return Ok(()); + } + let pending: Vec<_> = self + .records + .lock() + .unwrap() + .iter() + .filter(|(_, r)| { + !r.complete || r.upload_id.is_some() || r.multipart_intent || r.completion_attempted + }) + .map(|(id, r)| (id.clone(), r.clone())) + .collect(); + for (id, record) in pending { + if let Err(error) = self.reconcile_record(id, record).await + && error.downcast_ref::().is_none() + { + return Err(error); + } + } + let service = self.clone(); + tokio::task::spawn_blocking(move || service.prune()).await??; + Ok(()) + } + async fn reconcile_record(&self, id: String, record: Record) -> Result<()> { + let Some(backend) = &self.backend else { + return Ok(()); + }; + if !record.complete || record.multipart_intent { + // Also cover old journals from before multipart_intent existed. + // This exact durable reservation is exclusively claimed by the + // startup barrier or the active-set lock. Never sweep a bucket. + backend.abort_pending(&id).await?; + } + if let Some(upload_id) = &record.upload_id { + backend.abort(&id, upload_id).await?; + } + // Completion may have committed while abort was in flight. Only the + // post-abort HEAD can establish the current single-use object state. + let head = backend.read(&id, true, None).await?; + let complete = match head.status() { + StatusCode::OK => { + ensure!( + head.headers() + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + == Some(record.size), + "S3 recovered object size mismatch" + ); + true + } + StatusCode::NOT_FOUND => false, + _ => anyhow::bail!("S3 recovery HEAD failed"), + }; + if !complete && record.completion_attempted { + return Err(CompletionUncertain.into()); + } + let service = self.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut records = service.records.lock().unwrap(); + if let Some(r) = records.get_mut(&id) + && r.upload_id == record.upload_id + && r.multipart_intent == record.multipart_intent + { + r.complete = complete; + r.upload_id = None; + r.multipart_intent = false; + r.completion_attempted = false; + } + service.persist(&records) + }) + .await??; + Ok(()) + } + /// Read-only identifiers for unresolved completion attempts. Credentials, + /// bearer tokens and object contents are never included. + pub fn quarantined_slots(&self) -> Vec { + let records = self.records.lock().unwrap(); + let active = self.active.lock().unwrap(); + records + .iter() + .filter(|(id, r)| r.completion_attempted && !r.complete && !active.contains(*id)) + .map(|(id, _)| id.clone()) + .collect() + } + fn claim_upload(&self, id: &str) -> std::result::Result { + // Match prune's lock order, and read all mutable completion state under + // the same records lock used to establish the active claim. + let records = self.records.lock().unwrap(); + let record = records.get(id).ok_or(StatusCode::GONE)?; + if record.complete || (self.backend.is_none() && self.file(id).exists()) { + return Err(StatusCode::CONFLICT); + } + if !self.active.lock().unwrap().insert(id.to_owned()) { + return Err(StatusCode::CONFLICT); + } + Ok(record.clone()) + } fn persist(&self, r: &BTreeMap) -> Result<()> { let mut entries: Vec<_> = r .iter() @@ -214,11 +386,41 @@ impl UploadService { } pub fn prune(&self) -> Result<()> { let mut r = self.records.lock().unwrap(); - let active = self.active.lock().unwrap(); + let mut active = self.active.lock().unwrap(); + if self.backend.is_some() + && let Ok(handle) = tokio::runtime::Handle::try_current() + { + for (id, record) in r + .iter() + .filter(|(id, r)| { + (r.upload_id.is_some() || r.multipart_intent || r.completion_attempted) + && !active.contains(*id) + }) + .map(|(id, r)| (id.clone(), r.clone())) + .collect::>() + { + active.insert(id.clone()); + let service = self.clone(); + handle.spawn(async move { + let recovered = service.reconcile_record(id.clone(), record).await.is_ok(); + tokio::task::spawn_blocking(move || { + service.active.lock().unwrap().remove(&id); + if recovered { + let _ = service.prune(); + } + }) + .await + .ok(); + }); + } + } let obsolete: Vec = r .iter() .filter(|(id, v)| { !active.contains(*id) + && v.upload_id.is_none() + && !v.multipart_intent + && !v.completion_attempted && (self .config .expires_after @@ -230,6 +432,29 @@ impl UploadService { .map(|(id, _)| id.clone()) .collect(); for id in obsolete { + if let Some(backend) = self.backend.clone() { + // Keep the reservation and metadata until remote deletion succeeds. + // The active claim prevents duplicate workers and concurrent PUTs. + if let Ok(handle) = tokio::runtime::Handle::try_current() { + active.insert(id.clone()); + let service = self.clone(); + handle.spawn(async move { + let deleted = backend.delete(&id).await.is_ok(); + tokio::task::spawn_blocking(move || { + let mut records = service.records.lock().unwrap(); + let mut active = service.active.lock().unwrap(); + if deleted { + records.remove(&id); + let _ = service.persist(&records); + } + active.remove(&id); + }) + .await + .ok(); + }); + } + continue; + } match std::fs::remove_file(self.file(&id)) { Ok(()) => {} Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} @@ -314,6 +539,9 @@ impl UploadService { created: now(), token, complete: false, + upload_id: None, + multipart_intent: false, + completion_attempted: false, xml: None, }, ); @@ -358,9 +586,10 @@ impl UploadService { let config = Arc::downgrade(&self.config); let records = Arc::downgrade(&self.records); let active = Arc::downgrade(&self.active); + let backend = self.backend.clone(); handle.spawn(async move { loop { - tokio::time::sleep(std::time::Duration::from_secs(3600)).await; + tokio::time::sleep(std::time::Duration::from_secs(60)).await; let (Some(config), Some(records), Some(active)) = (config.upgrade(), records.upgrade(), active.upgrade()) else { @@ -370,6 +599,7 @@ impl UploadService { config, records, active, + backend: backend.clone(), }; let _ = tokio::task::spawn_blocking(move || service.prune()).await; } @@ -380,6 +610,14 @@ impl UploadService { .with_state(self.clone()) .layer(axum::Extension(UploadPath(prefix))) } + pub async fn handle_at(&self, req: Request, internal_path: &str) -> Response { + endpoint( + State(self.clone()), + axum::Extension(UploadPath(internal_path.trim_end_matches('/').to_owned())), + req, + ) + .await + } pub async fn serve(&self, listener: TcpListener, tls: Option) -> Result<()> { serve_router(listener, tls, self.router()).await } @@ -496,6 +734,7 @@ async fn serve_router_inner( Some(crate::http_transport::HttpRoute::Websocket) => "websocket", Some(crate::http_transport::HttpRoute::Bosh) => "bosh", Some(crate::http_transport::HttpRoute::Altconnect) => "altconnect", + Some(crate::http_transport::HttpRoute::Upload) => "upload", None => default_route, }; let status = response.status(); @@ -693,20 +932,15 @@ async fn handle_at(s: UploadService, req: Request, prefix: &str) -> Response { { return response(StatusCode::PAYLOAD_TOO_LARGE); } - if record.complete || s.file(id).exists() { + if record.complete || (s.backend.is_none() && s.file(id).exists()) { return response(StatusCode::CONFLICT); }; - { - // Use the same lock order as prune: a reservation must still exist - // when it becomes active, so pruning cannot orphan an upload. - let records = s.records.lock().unwrap(); - if !records.contains_key(id) { - return response(StatusCode::GONE); - } - if !s.active.lock().unwrap().insert(id.into()) { - return response(StatusCode::CONFLICT); - } - } + // Refresh the snapshot atomically with the exclusive claim. A request + // may have waited while an earlier PUT committed or was quarantined. + let record = match s.claim_upload(id) { + Ok(record) => record, + Err(status) => return response(status), + }; let dest = s.file(id); let temp = dest.with_extension(format!("{}.tmp", uuid::Uuid::new_v4())); let _guard = UploadGuard { @@ -714,6 +948,212 @@ async fn handle_at(s: UploadService, req: Request, prefix: &str) -> Response { id: id.into(), temp: temp.clone(), }; + if let Some(backend) = &s.backend { + if (record.upload_id.is_some() + || record.multipart_intent + || record.completion_attempted) + && let Err(error) = s.reconcile_record(id.to_owned(), record.clone()).await + { + return response(if error.downcast_ref::().is_some() { + StatusCode::CONFLICT + } else { + StatusCode::BAD_GATEWAY + }); + } + // Reconcile a remote commit whose response or local metadata write + // was interrupted, rather than overwriting a single-use object. + match backend.read(id, true, None).await { + Ok(reply) if reply.status() == StatusCode::NOT_FOUND => {} + Ok(reply) if reply.status() == StatusCode::OK => { + if reply + .headers() + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + != Some(record.size) + { + return response(StatusCode::BAD_GATEWAY); + } + let service = s.clone(); + let id = id.to_owned(); + return response( + tokio::task::spawn_blocking(move || { + let _guard = _guard; + let mut records = service.records.lock().unwrap(); + if let Some(r) = records.get_mut(&id) { + r.complete = true; + r.upload_id = None; + r.multipart_intent = false; + r.completion_attempted = false; + } + if service.persist(&records).is_err() { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::CONFLICT + } + }) + .await + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + ); + } + _ => return response(StatusCode::BAD_GATEWAY), + } + // Hold the single-use claim across initiation and journaling even + // if the HTTP request is cancelled. A persisted intent closes the + // crash window before S3 returns its UploadId. + let service = s.clone(); + let backend = backend.clone(); + let slot_id = id.to_owned(); + let content_type = record.content_type.clone(); + let initiation = tokio::spawn(async move { + let guard = _guard; + let journal = service.clone(); + let journal_id = slot_id.clone(); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut records = journal.records.lock().unwrap(); + let r = records + .get_mut(&journal_id) + .ok_or_else(|| anyhow::anyhow!("upload reservation missing"))?; + r.multipart_intent = true; + journal.persist(&records) + }) + .await??; + let mut upload = backend.begin(&slot_id, &content_type).await?; + let upload_id = upload.upload_id().to_owned(); + let cleanup = service.clone(); + upload.after_abort(move || { + // HEAD must reconcile a possible remote commit before + // clearing the intent, even when abort returns 404. + tokio::task::spawn_blocking(move || { + let _ = cleanup.prune(); + }); + }); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut records = service.records.lock().unwrap(); + let r = records + .get_mut(&slot_id) + .ok_or_else(|| anyhow::anyhow!("upload reservation missing"))?; + r.upload_id = Some(upload_id); + service.persist(&records) + }) + .await??; + Ok::<_, anyhow::Error>((upload, guard)) + }); + let (mut upload, _guard) = match initiation.await { + Ok(Ok(value)) => value, + _ => return response(StatusCode::BAD_GATEWAY), + }; + let mut body = req.into_body(); + let mut count = 0u64; + let mut part = Vec::with_capacity(PART_SIZE); + loop { + let frame = + match tokio::time::timeout(std::time::Duration::from_secs(120), body.frame()) + .await + { + Ok(Some(Ok(frame))) => frame, + Ok(Some(Err(_))) => return response(StatusCode::BAD_REQUEST), + Ok(None) => break, + Err(_) => return response(StatusCode::REQUEST_TIMEOUT), + }; + if let Ok(bytes) = frame.into_data() { + count = count.saturating_add(bytes.len() as u64); + if count > record.size { + return response(StatusCode::PAYLOAD_TOO_LARGE); + } + let mut bytes = bytes.as_ref(); + while !bytes.is_empty() { + let take = (PART_SIZE - part.len()).min(bytes.len()); + part.extend_from_slice(&bytes[..take]); + bytes = &bytes[take..]; + if part.len() == PART_SIZE + && upload + .part(std::mem::replace(&mut part, Vec::with_capacity(PART_SIZE))) + .await + .is_err() + { + return response(StatusCode::BAD_GATEWAY); + } + } + } + } + if count != record.size { + return response(StatusCode::BAD_REQUEST); + } + if (!part.is_empty() || count == 0) && upload.part(part).await.is_err() { + return response(StatusCode::BAD_GATEWAY); + } + // Once completing begins, finish the durable commit even if the + // HTTP client disconnects. Keep the single-use claim until then. + let service = s.clone(); + let id = id.to_owned(); + let commit = tokio::spawn(async move { + let _guard = _guard; + let journal = service.clone(); + let journal_id = id.clone(); + if !matches!( + tokio::task::spawn_blocking(move || -> Result<()> { + let mut records = journal.records.lock().unwrap(); + let r = records + .get_mut(&journal_id) + .ok_or_else(|| anyhow::anyhow!("upload reservation missing"))?; + r.completion_attempted = true; + journal.persist(&records) + }) + .await, + Ok(Ok(())) + ) { + return StatusCode::INTERNAL_SERVER_ERROR; + } + if let Err(error) = upload.complete().await { + if error + .downcast_ref::() + .is_some() + { + // Explicit terminal rejection is retryable after HEAD + // reconciliation. Network/XML uncertainty stays frozen. + let journal = service.clone(); + let journal_id = id.clone(); + let _ = tokio::task::spawn_blocking(move || -> Result<()> { + let mut records = journal.records.lock().unwrap(); + if let Some(r) = records.get_mut(&journal_id) { + r.completion_attempted = false; + } + journal.persist(&records) + }) + .await; + } + // Multipart::drop may finish its abort while this commit + // still owns the active claim. Its callback's prune then + // correctly skips the slot, so it cannot be our only + // reconciliation trigger. Reconcile under this same claim + // before returning the ambiguous completion response. + let pending = service.records.lock().unwrap().get(&id).cloned(); + if let Some(record) = pending { + let _ = service.reconcile_record(id.clone(), record).await; + } + return StatusCode::BAD_GATEWAY; + } + tokio::task::spawn_blocking(move || { + let _guard = _guard; + let mut records = service.records.lock().unwrap(); + if let Some(r) = records.get_mut(&id) { + r.complete = true; + r.upload_id = None; + r.multipart_intent = false; + r.completion_attempted = false; + } + if service.persist(&records).is_err() { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::CREATED + } + }) + .await + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) + }); + return response(commit.await.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR)); + } if tokio::fs::create_dir_all(dest.parent().unwrap()) .await .is_err() @@ -769,36 +1209,78 @@ async fn handle_at(s: UploadService, req: Request, prefix: &str) -> Response { return response(StatusCode::INTERNAL_SERVER_ERROR); } } - let mut records = s.records.lock().unwrap(); - if let Some(r) = records.get_mut(id) { - r.complete = true; - } - if s.persist(&records).is_err() { - return response(StatusCode::INTERNAL_SERVER_ERROR); - }; - return response(StatusCode::CREATED); + let id = id.to_owned(); + return response( + tokio::task::spawn_blocking(move || { + let _guard = _guard; + let mut records = s.records.lock().unwrap(); + if let Some(r) = records.get_mut(&id) { + r.complete = true; + } + if s.persist(&records).is_err() { + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::CREATED + } + }) + .await + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + ); } if req.method() != "GET" && req.method() != "HEAD" { return response(StatusCode::METHOD_NOT_ALLOWED); } - let Ok(mut file) = tokio::fs::File::open(s.file(id)).await else { + if s.backend.is_some() && !record.complete { return response(StatusCode::GONE); + } + let mut local_file = if s.backend.is_none() { + match tokio::fs::File::open(s.file(id)).await { + Ok(file) => Some(file), + Err(_) => return response(StatusCode::GONE), + } + } else { + None }; let modified = chrono::DateTime::from_timestamp(record.created as i64, 0) .map(|d| d.format("%a, %d %b %Y %H:%M:%S GMT").to_string()) .unwrap_or_default(); - if req - .headers() - .get("if-modified-since") - .and_then(|v| v.to_str().ok()) - == Some(modified.as_str()) - { - return response(StatusCode::NOT_MODIFIED); + let etag = format!("\"{id}-{}\"", record.size); + let date_matches = |value: &str| { + chrono::DateTime::parse_from_rfc2822(value) + .ok() + .is_some_and(|d| d.timestamp() >= record.created as i64) + }; + let not_modified = if let Some(value) = req.headers().get("if-none-match") { + value.to_str().ok().is_some_and(|v| { + v.split(',').any(|candidate| { + let candidate = candidate.trim(); + candidate == "*" || candidate.strip_prefix("W/").unwrap_or(candidate) == etag + }) + }) + } else { + req.headers() + .get("if-modified-since") + .and_then(|v| v.to_str().ok()) + .is_some_and(date_matches) + }; + if not_modified { + return Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header("etag", &etag) + .header("last-modified", &modified) + .body(Body::empty()) + .unwrap(); } + let range_allowed = req.headers().get("if-range").is_none_or(|value| { + value + .to_str() + .ok() + .is_some_and(|v| v == etag || date_matches(v)) + }); let mut status = StatusCode::OK; let mut start = 0; let mut end = record.size.saturating_sub(1); - if let Some(range) = req.headers().get("range") { + if let Some(range) = req.headers().get("range").filter(|_| range_allowed) { let parsed = range .to_str() .ok() @@ -833,14 +1315,50 @@ async fn handle_at(s: UploadService, req: Request, prefix: &str) -> Response { status = StatusCode::PARTIAL_CONTENT; } let length = if record.size == 0 { 0 } else { end - start + 1 }; - if file.seek(std::io::SeekFrom::Start(start)).await.is_err() { - return response(StatusCode::INTERNAL_SERVER_ERROR); - }; use tokio::io::AsyncReadExt; - let body = if req.method() == "HEAD" { - Body::empty() + let body = if let Some(backend) = &s.backend { + let range = (status == StatusCode::PARTIAL_CONTENT).then(|| format!("bytes={start}-{end}")); + let Ok(reply) = backend + .read(id, req.method() == "HEAD", range.as_deref()) + .await + else { + return response(StatusCode::BAD_GATEWAY); + }; + if reply.status() == StatusCode::NOT_FOUND { + return response(StatusCode::GONE); + } + if reply.status() != status { + return response(StatusCode::BAD_GATEWAY); + } + if reply + .headers() + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + != Some(length) + { + return response(StatusCode::BAD_GATEWAY); + } + if status == StatusCode::PARTIAL_CONTENT + && reply + .headers() + .get("content-range") + .and_then(|v| v.to_str().ok()) + != Some(format!("bytes {start}-{end}/{}", record.size).as_str()) + { + return response(StatusCode::BAD_GATEWAY); + } + reply.into_body() } else { - Body::from_stream(ReaderStream::new(file.take(length))) + let mut file = local_file.take().unwrap(); + if file.seek(std::io::SeekFrom::Start(start)).await.is_err() { + return response(StatusCode::INTERNAL_SERVER_ERROR); + } + if req.method() == "HEAD" { + Body::empty() + } else { + Body::from_stream(ReaderStream::new(file.take(length))) + } }; let disposition = if record.content_type == "text/plain" || ["image/", "video/", "audio/"] @@ -869,6 +1387,7 @@ async fn handle_at(s: UploadService, req: Request, prefix: &str) -> Response { ) .header("access-control-allow-origin", "*") .header("accept-ranges", "bytes") + .header("etag", etag) .header("last-modified", modified) .header("cache-control", "max-age=31556952, immutable") .header("x-frame-options", "DENY") @@ -991,6 +1510,60 @@ mod tests { ); } + #[test] + fn upload_claim_refreshes_completion_state_after_an_earlier_snapshot() { + let temp = tempfile::tempdir().unwrap(); + let service = UploadService::new(UploadConfig { + root: temp.path().into(), + host: "upload.example.test".into(), + base_url: "https://upload.example.test/file_share".into(), + size_limit: 1, + daily_quota: 3, + global_quota: 3, + expires_after: None, + allowed_types: vec![], + secret: None, + }) + .unwrap(); + service + .request_slot("a@example.test", "pending", 1, "text/plain") + .unwrap(); + let (id, stale) = service + .records + .lock() + .unwrap() + .iter() + .next() + .map(|(id, r)| (id.clone(), r.clone())) + .unwrap(); + assert!(!stale.completion_attempted); + // Another PUT finishes ambiguously after this request has read its + // initial snapshot, but before it acquires the active claim. + service + .records + .lock() + .unwrap() + .get_mut(&id) + .unwrap() + .completion_attempted = true; + let fresh = service.claim_upload(&id).unwrap(); + assert!(fresh.completion_attempted); + assert!(service.active.lock().unwrap().contains(&id)); + service.active.lock().unwrap().remove(&id); + service + .records + .lock() + .unwrap() + .get_mut(&id) + .unwrap() + .complete = true; + assert!(matches!( + service.claim_upload(&id), + Err(StatusCode::CONFLICT) + )); + assert!(!service.active.lock().unwrap().contains(&id)); + } + #[test] fn abandoned_slots_release_quota_without_pruning_existing_or_active_files() { let temp = tempfile::tempdir().unwrap(); diff --git a/src/http_transport.rs b/src/http_transport.rs index 5b61e87..9865422 100644 --- a/src/http_transport.rs +++ b/src/http_transport.rs @@ -30,6 +30,7 @@ pub enum HttpRoute { Websocket, Bosh, Altconnect, + Upload, } #[derive(Clone)] diff --git a/src/lib.rs b/src/lib.rs index a380501..07f49dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,3 +47,7 @@ pub mod muc_recovery; pub mod muc_history; pub mod http_altconnect; + +pub mod object_store; + +pub mod upload_migration; diff --git a/src/main.rs b/src/main.rs index e9d898d..96500e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,19 @@ async fn main() { async fn run() -> Result<()> { let args: Vec<_> = env::args().skip(1).collect(); match args.first().map(String::as_str) { + Some("upload-migrate") if args.len() == 5 || args.len() == 6 => { + anyhow::ensure!( + args[3] == "--component" && (args.len() == 5 || args[5] == "--execute"), + "usage: upload-migrate OLD_CONFIG NEW_CONFIG --component HOST [--execute]" + ); + rust_xmpp::upload_migration::migrate( + Path::new(&args[1]), + Path::new(&args[2]), + &args[4], + args.len() == 6, + ) + .await?; + } Some("check") => { let path = args .get(1) @@ -108,7 +121,7 @@ async fn run() -> Result<()> { println!("Account created: {}@{}", args[2], domain); } Some("--help") | None => println!( - "rust-xmpp check-config PATH\nrust-xmpp convert PATH json|yaml|toml OUTPUT\nrust-xmpp --version\n\nExperimental runtime: serve CONFIG --experimental\ncreate-user CONFIG USER --password-stdin\ncheck [CONFIG] (running server healthcheck)\nshell CONFIG COMMAND\n\nSTARTTLS, SCRAM/PLAIN, local messaging, roster, MAM, MUC, PEP/pubsub, stream management, push, federation and HTTP transports are implemented with limitations; see docs/COMPATIBILITY.md." + "rust-xmpp upload-migrate OLD_CONFIG NEW_CONFIG --component HOST [--execute]\nrust-xmpp check-config PATH\nrust-xmpp convert PATH json|yaml|toml OUTPUT\nrust-xmpp --version\n\nExperimental runtime: serve CONFIG --experimental\ncreate-user CONFIG USER --password-stdin\ncheck [CONFIG] (running server healthcheck)\nshell CONFIG COMMAND\n\nSTARTTLS, SCRAM/PLAIN, local messaging, roster, MAM, MUC, PEP/pubsub, stream management, push, federation and HTTP transports are implemented with limitations; see docs/COMPATIBILITY.md." ), _ => bail!("invalid command; use --help"), } diff --git a/src/object_store.rs b/src/object_store.rs new file mode 100644 index 0000000..c77e142 --- /dev/null +++ b/src/object_store.rs @@ -0,0 +1,605 @@ +//! Path-style S3-compatible storage with authenticated, bounded multipart writes. +use anyhow::{Result, ensure}; +use axum::{ + body::{Body, Bytes}, + http::{Request, Response, Uri}, +}; +use hmac::{Hmac, Mac}; +use http_body_util::{BodyExt, Full, Limited}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::TcpStream, +}; +use tokio_rustls::{ + TlsConnector, + rustls::{ClientConfig, pki_types::ServerName}, +}; + +#[derive(Clone, Default)] +pub enum UploadBackendConfig { + #[default] + Local, + S3(S3Config), +} +// Deliberately no Debug implementation: credentials must never enter diagnostics. +#[derive(Clone)] +pub struct S3Config { + pub endpoint: String, + pub bucket: String, + pub region: String, + pub access_key: String, + pub secret_key: String, + pub session_token: Option, + pub key_prefix: String, +} +#[derive(Clone)] +pub struct ObjectStore { + config: S3Config, + endpoint: Uri, + tls: Option>, + uploads: Arc, +} +trait Transport: AsyncRead + AsyncWrite + Unpin + Send {} +impl Transport for T {} +fn encode(s: &str) -> String { + s.bytes() + .map(|b| { + if b.is_ascii_alphanumeric() || b"-._~".contains(&b) { + (b as char).to_string() + } else { + format!("%{b:02X}") + } + }) + .collect() +} +fn decode_key(value: &str) -> Result { + let mut bytes = value.bytes(); + let mut decoded = Vec::with_capacity(value.len()); + while let Some(byte) = bytes.next() { + if byte == b'%' { + let high = bytes.next().and_then(|v| (v as char).to_digit(16)); + let low = bytes.next().and_then(|v| (v as char).to_digit(16)); + match (high, low) { + (Some(h), Some(l)) => decoded.push((h * 16 + l) as u8), + _ => anyhow::bail!("invalid S3 listing key encoding"), + } + } else { + decoded.push(byte); + } + } + Ok(String::from_utf8(decoded)?) +} +fn mac(key: &[u8], data: &str) -> Vec { + let mut h = Hmac::::new_from_slice(key).unwrap(); + h.update(data.as_bytes()); + h.finalize().into_bytes().to_vec() +} +impl ObjectStore { + pub fn new(config: S3Config) -> Result { + let endpoint: Uri = config + .endpoint + .parse() + .map_err(|_| anyhow::anyhow!("invalid S3 endpoint"))?; + let scheme = endpoint.scheme_str().unwrap_or(""); + let host = endpoint.host().unwrap_or("").trim_matches(['[', ']']); + ensure!( + !host.is_empty() + && endpoint.query().is_none() + && crate::http_config::valid_authority(endpoint.authority().unwrap()) + && !config.endpoint.contains('#') + && !endpoint.path().contains('%') + && !endpoint.path().split('/').any(|p| p == "." || p == ".."), + "invalid S3 endpoint" + ); + ensure!( + scheme == "https" + || (scheme == "http" + && (host == "localhost" + || host + .parse::() + .is_ok_and(|a| a.is_loopback()))), + "S3 endpoint requires HTTPS (HTTP is allowed only for loopback)" + ); + ensure!( + !config.bucket.is_empty() + && config + .bucket + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-_".contains(&b)), + "invalid S3 bucket" + ); + ensure!( + !config.region.is_empty() + && !config.access_key.is_empty() + && !config.secret_key.is_empty(), + "S3 credentials and region are required" + ); + let tls = if scheme == "https" { + let roots = crate::federation::Federation::native_roots()?; + Some(Arc::new( + ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(), + )) + } else { + None + }; + Ok(Self { + config, + endpoint, + tls, + uploads: Arc::new(tokio::sync::Semaphore::new(8)), + }) + } + pub fn identity(&self) -> serde_json::Value { + serde_json::json!({"backend":"s3", "endpoint":self.config.endpoint.trim_end_matches('/'), "bucket":self.config.bucket, "prefix":self.config.key_prefix.trim_matches('/')}) + } + fn path(&self, id: &str) -> String { + let key = [self.config.key_prefix.trim_matches('/'), id] + .into_iter() + .filter(|s| !s.is_empty()) + .collect::>() + .join("/"); + format!( + "{}/{}/{}", + self.endpoint.path().trim_end_matches('/'), + encode(&self.config.bucket), + key.split('/').map(encode).collect::>().join("/") + ) + } + async fn request( + &self, + method: &str, + id: &str, + query: &[(&str, &str)], + body: Bytes, + extra: &[(&str, &str)], + ) -> Result> { + // A deadline includes DNS, TLS, request transmission and response headers. + tokio::time::timeout( + Duration::from_secs(120), + self.request_inner(method, self.path(id), query, body, extra), + ) + .await + .map_err(|_| anyhow::anyhow!("S3 request timed out"))? + } + async fn request_inner( + &self, + method: &str, + path: String, + query: &[(&str, &str)], + body: Bytes, + extra: &[(&str, &str)], + ) -> Result> { + let host = self.endpoint.host().unwrap().trim_matches(['[', ']']); + let authority = self.endpoint.authority().unwrap().as_str(); + let port = self + .endpoint + .port_u16() + .unwrap_or(if self.tls.is_some() { 443 } else { 80 }); + let mut query: Vec<_> = query.iter().map(|(k, v)| (encode(k), encode(v))).collect(); + query.sort(); + let query = query + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + let uri = if query.is_empty() { + path.clone() + } else { + format!("{path}?{query}") + }; + let date = chrono::Utc::now(); + let stamp = date.format("%Y%m%dT%H%M%SZ").to_string(); + let day = date.format("%Y%m%d").to_string(); + let hash = hex::encode(Sha256::digest(&body)); + let mut headers = BTreeMap::from([ + ("host".to_string(), authority.to_string()), + ("x-amz-content-sha256".into(), hash.clone()), + ("x-amz-date".into(), stamp.clone()), + ]); + if let Some(token) = &self.config.session_token { + headers.insert("x-amz-security-token".into(), token.clone()); + } + let signed = headers.keys().cloned().collect::>().join(";"); + let canonical_headers = headers + .iter() + .map(|(k, v)| { + format!( + "{k}:{}\n", + v.split_whitespace().collect::>().join(" ") + ) + }) + .collect::(); + let canonical = format!("{method}\n{path}\n{query}\n{canonical_headers}\n{signed}\n{hash}"); + let scope = format!("{day}/{}/s3/aws4_request", self.config.region); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{}", + hex::encode(Sha256::digest(canonical.as_bytes())) + ); + let key = mac(format!("AWS4{}", self.config.secret_key).as_bytes(), &day); + let key = mac(&key, &self.config.region); + let key = mac(&key, "s3"); + let key = mac(&key, "aws4_request"); + let auth = format!( + "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed}, Signature={}", + self.config.access_key, + hex::encode(mac(&key, &to_sign)) + ); + let mut req = Request::builder() + .method(method) + .uri(uri) + .header("authorization", auth) + .header("content-length", body.len()); + for (k, v) in &headers { + req = req.header(k, v); + } + for (k, v) in extra { + req = req.header(*k, *v); + } + let req = req.body(Full::new(body))?; + let tcp = TcpStream::connect((host, port)).await?; + let stream: Box = if let Some(tls) = &self.tls { + Box::new( + TlsConnector::from(tls.clone()) + .connect(ServerName::try_from(host.to_string())?, tcp) + .await?, + ) + } else { + Box::new(tcp) + }; + let (mut sender, conn) = + hyper::client::conn::http1::handshake(hyper_util::rt::TokioIo::new(stream)).await?; + tokio::spawn(async move { + let _ = conn.await; + }); + Ok(sender.send_request(req).await?) + } + async fn xml(response: Response) -> Result { + ensure!(response.status().is_success(), "S3 operation failed"); + Self::xml_unchecked(response).await + } + async fn xml_unchecked(response: Response) -> Result { + let bytes = tokio::time::timeout( + Duration::from_secs(120), + Limited::new(response.into_body(), 1024 * 1024).collect(), + ) + .await + .map_err(|_| anyhow::anyhow!("S3 response timed out"))? + .map_err(|_| anyhow::anyhow!("invalid or oversized S3 response"))? + .to_bytes(); + Ok(String::from_utf8(bytes.to_vec())?) + } + pub async fn begin(&self, id: &str, content_type: &str) -> Result { + // Shield initiation: cancellation must still consume UploadId and abort it. + let store = self.clone(); + let id = id.to_owned(); + let content_type = content_type.to_owned(); + tokio::spawn(async move { store.begin_inner(&id, &content_type).await }).await? + } + async fn begin_inner(&self, id: &str, content_type: &str) -> Result { + let permit = self + .uploads + .clone() + .try_acquire_owned() + .map_err(|_| anyhow::anyhow!("S3 uploads busy"))?; + let r = self + .request( + "POST", + id, + &[("uploads", "")], + Bytes::new(), + &[("content-type", content_type)], + ) + .await?; + let xml = Self::xml(r).await?; + let doc = roxmltree::Document::parse(&xml)?; + let upload_id = doc + .descendants() + .find(|n| n.has_tag_name("UploadId")) + .and_then(|n| n.text()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| anyhow::anyhow!("S3 upload identifier missing"))? + .to_string(); + Ok(Multipart { + store: self.clone(), + id: id.into(), + upload_id, + etags: Vec::new(), + finished: false, + after_abort: None, + _permit: permit, + }) + } + pub async fn read(&self, id: &str, head: bool, range: Option<&str>) -> Result> { + let extra: Vec<_> = range.map(|r| vec![("range", r)]).unwrap_or_default(); + let r = self + .request( + if head { "HEAD" } else { "GET" }, + id, + &[], + Bytes::new(), + &extra, + ) + .await?; + let (parts, body) = r.into_parts(); + Ok(Response::from_parts( + parts, + if head { Body::empty() } else { Body::new(body) }, + )) + } + /// Abort only multiparts for an exclusively owned, durable slot key. The + /// caller must exclude concurrent initiation for this key throughout recovery. + /// Listing a prefix alone does not establish ownership: compare the full key. + pub async fn abort_pending(&self, id: &str) -> Result<()> { + let key = [self.config.key_prefix.trim_matches('/'), id] + .into_iter() + .filter(|v| !v.is_empty()) + .collect::>() + .join("/"); + let bucket_path = format!( + "{}/{}", + self.endpoint.path().trim_end_matches('/'), + encode(&self.config.bucket) + ); + let mut markers: Option<(String, String)> = None; + let mut seen = std::collections::BTreeSet::new(); + loop { + let mut query = vec![ + ("uploads", ""), + ("prefix", key.as_str()), + ("max-uploads", "1000"), + ("encoding-type", "url"), + ]; + if let Some((key_marker, upload_marker)) = &markers { + query.push(("key-marker", key_marker)); + query.push(("upload-id-marker", upload_marker)); + } + let response = tokio::time::timeout( + Duration::from_secs(120), + self.request_inner("GET", bucket_path.clone(), &query, Bytes::new(), &[]), + ) + .await + .map_err(|_| anyhow::anyhow!("S3 listing timed out"))??; + let xml = Self::xml(response).await?; + let doc = roxmltree::Document::parse(&xml)?; + let root = doc.root_element(); + ensure!( + root.has_tag_name("ListMultipartUploadsResult"), + "invalid S3 multipart listing" + ); + let field = |node: roxmltree::Node<'_, '_>, name: &str| { + node.children() + .find(|n| n.has_tag_name(name)) + .and_then(|n| n.text()) + .map(str::to_owned) + }; + for upload in root.children().filter(|n| n.has_tag_name("Upload")) { + let listed_key = field(upload, "Key") + .ok_or_else(|| anyhow::anyhow!("S3 listing key missing"))?; + let upload_id = field(upload, "UploadId") + .filter(|s| !s.is_empty()) + .ok_or_else(|| anyhow::anyhow!("S3 listing upload identifier missing"))?; + if decode_key(&listed_key)? == key { + self.abort(id, &upload_id).await?; + } + } + match field(root, "IsTruncated").as_deref() { + Some("false") => break, + Some("true") => { + let next = ( + decode_key( + &field(root, "NextKeyMarker") + .ok_or_else(|| anyhow::anyhow!("S3 listing marker missing"))?, + )?, + field(root, "NextUploadIdMarker") + .ok_or_else(|| anyhow::anyhow!("S3 listing upload marker missing"))?, + ); + ensure!( + !next.0.is_empty() + && !next.1.is_empty() + && markers.as_ref().is_none_or(|previous| &next > previous) + && seen.insert(next.clone()), + "S3 listing did not advance" + ); + markers = Some(next); + } + _ => anyhow::bail!("invalid S3 listing truncation flag"), + } + } + Ok(()) + } + /// HEAD with an explicit distinction between absence and service errors. + pub async fn stat(&self, id: &str) -> Result> { + let reply = self.read(id, true, None).await?; + if reply.status() == 404 { + return Ok(None); + } + ensure!(reply.status() == 200, "S3 stat failed"); + let size = reply + .headers() + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse().ok()) + .ok_or_else(|| anyhow::anyhow!("S3 object size missing"))?; + Ok(Some(size)) + } + /// Bounded-memory streaming upload. Callers journal ownership before calling + /// this method and retain it until commit, allowing abort_pending after crash. + pub async fn write_stream( + &self, + id: &str, + content_type: &str, + mut body: Body, + expected_size: u64, + ) -> Result<()> { + let mut upload = self.begin(id, content_type).await?; + let mut count = 0u64; + let mut part = Vec::with_capacity(PART_SIZE); + while let Some(frame) = tokio::time::timeout(Duration::from_secs(120), body.frame()) + .await + .map_err(|_| anyhow::anyhow!("S3 source timed out"))? + { + let frame = frame.map_err(|_| anyhow::anyhow!("S3 source read failed"))?; + if let Ok(bytes) = frame.into_data() { + count = count + .checked_add(bytes.len() as u64) + .ok_or_else(|| anyhow::anyhow!("S3 source too large"))?; + ensure!(count <= expected_size, "S3 source too large"); + let mut bytes = bytes.as_ref(); + while !bytes.is_empty() { + let take = (PART_SIZE - part.len()).min(bytes.len()); + part.extend_from_slice(&bytes[..take]); + bytes = &bytes[take..]; + if part.len() == PART_SIZE { + upload + .part(std::mem::replace(&mut part, Vec::with_capacity(PART_SIZE))) + .await?; + } + } + } + } + ensure!(count == expected_size, "S3 source size mismatch"); + if !part.is_empty() || count == 0 { + upload.part(part).await?; + } + upload.complete().await + } + pub async fn copy_from( + &self, + source: &ObjectStore, + id: &str, + content_type: &str, + expected_size: u64, + ) -> Result<()> { + let reply = source.read(id, false, None).await?; + ensure!(reply.status() == 200, "S3 copy source unavailable"); + self.write_stream(id, content_type, reply.into_body(), expected_size) + .await + } + pub async fn abort(&self, id: &str, upload_id: &str) -> Result<()> { + let r = self + .request("DELETE", id, &[("uploadId", upload_id)], Bytes::new(), &[]) + .await?; + ensure!( + r.status().is_success() || r.status() == 404, + "S3 multipart abort failed" + ); + Ok(()) + } + pub async fn delete(&self, id: &str) -> Result<()> { + let r = self.request("DELETE", id, &[], Bytes::new(), &[]).await?; + ensure!( + r.status().is_success() || r.status() == 404, + "S3 deletion failed" + ); + Ok(()) + } +} +/// A well-formed terminal rejection from CompleteMultipartUpload. Transport, +/// timeout and malformed response errors deliberately do not carry this marker. +#[derive(Debug)] +pub struct CompletionRejected; +impl std::fmt::Display for CompletionRejected { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("S3 completion rejected") + } +} +impl std::error::Error for CompletionRejected {} +pub const PART_SIZE: usize = 8 * 1024 * 1024; +pub struct Multipart { + store: ObjectStore, + id: String, + upload_id: String, + etags: Vec, + finished: bool, + after_abort: Option>, + _permit: tokio::sync::OwnedSemaphorePermit, +} +impl Multipart { + pub fn upload_id(&self) -> &str { + &self.upload_id + } + pub fn after_abort(&mut self, callback: impl FnOnce() + Send + 'static) { + self.after_abort = Some(Box::new(callback)); + } + pub async fn part(&mut self, data: Vec) -> Result<()> { + ensure!(self.etags.len() < 10000, "S3 part limit exceeded"); + let number = (self.etags.len() + 1).to_string(); + let r = self + .store + .request( + "PUT", + &self.id, + &[("partNumber", &number), ("uploadId", &self.upload_id)], + Bytes::from(data), + &[], + ) + .await?; + ensure!(r.status().is_success(), "S3 part failed"); + let etag = r + .headers() + .get("etag") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| anyhow::anyhow!("S3 part ETag missing"))? + .to_string(); + self.etags.push(etag); + Ok(()) + } + pub async fn complete(mut self) -> Result<()> { + let mut xml = "".to_string(); + for (i, etag) in self.etags.iter().enumerate() { + xml.push_str(&format!( + "{}{}", + i + 1, + crate::wire::escape(etag) + )); + } + xml.push_str(""); + let r = self + .store + .request( + "POST", + &self.id, + &[("uploadId", &self.upload_id)], + Bytes::from(xml), + &[("content-type", "application/xml"), ("if-none-match", "*")], + ) + .await?; + let successful = r.status().is_success(); + let xml = ObjectStore::xml_unchecked(r).await?; + let doc = roxmltree::Document::parse(&xml)?; + if doc.root_element().has_tag_name("Error") { + return Err(CompletionRejected.into()); + } + ensure!(successful, "S3 completion failed"); + ensure!( + doc.root_element() + .has_tag_name("CompleteMultipartUploadResult"), + "S3 completion failed" + ); + self.finished = true; + Ok(()) + } +} +impl Drop for Multipart { + fn drop(&mut self) { + if !self.finished { + let store = self.store.clone(); + let id = self.id.clone(); + let upload_id = self.upload_id.clone(); + let after_abort = self.after_abort.take(); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { + if store.abort(&id, &upload_id).await.is_ok() + && let Some(callback) = after_abort + { + callback(); + } + }); + } + } + } +} diff --git a/src/server.rs b/src/server.rs index 4a3f84d..e4787e1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -49,6 +49,136 @@ pub struct Runtime { federation: Option, stopping: Arc, } +/// Keep upload services separate: their HTTP hosts, mounts and credentials belong +/// to the component, while all listeners share one dispatcher. +#[derive(Clone)] +struct UploadRoute { + name: String, + config: crate::http_config::HttpConfig, + service: crate::http_service::UploadService, +} +pub(crate) fn upload_backend( + config: &Config, + name: &str, +) -> Result { + use crate::object_store::{S3Config, UploadBackendConfig}; + let backend = config + .option(name, "http_file_share_backend") + .map(|value| { + value + .as_str() + .context("http_file_share_backend must be a string") + }) + .transpose()? + .unwrap_or("local"); + match backend { + "local" => Ok(UploadBackendConfig::Local), + "s3" => { + let options = config + .option(name, "http_file_share_s3") + .and_then(|value| value.as_object()) + .context("http_file_share_s3 must be a table for S3 uploads")?; + let field = |key: &str| -> Result> { + if let Some(value) = options.get(&format!("{key}_env")) { + let variable = value.as_str().with_context(|| { + format!("http_file_share_s3.{key}_env must be a string") + })?; + return Ok(Some(std::env::var(variable).with_context(|| { + format!("environment variable for http_file_share_s3.{key} is unavailable") + })?)); + } + options + .get(key) + .map(|value| { + value + .as_str() + .map(str::to_owned) + .with_context(|| format!("http_file_share_s3.{key} must be a string")) + }) + .transpose() + }; + let required = |key: &str| { + field(key)? + .filter(|value| !value.is_empty()) + .with_context(|| format!("http_file_share_s3.{key} is required")) + }; + Ok(UploadBackendConfig::S3(S3Config { + endpoint: required("endpoint")?, + bucket: required("bucket")?, + region: required("region")?, + access_key: required("access_key")?, + secret_key: required("secret_key")?, + session_token: field("session_token")?, + key_prefix: field("key_prefix")?.unwrap_or_default(), + })) + } + other => bail!("unsupported http_file_share_backend: {other}"), + } +} +fn upload_mount_matches(mount: &str, path: &str) -> bool { + mount == "/" + || path == mount + || path + .strip_prefix(mount) + .is_some_and(|rest| rest.starts_with('/')) +} +fn upload_routes( + config: &Config, + uploads: &BTreeMap, +) -> Result>> { + let mut routes: Vec = Vec::new(); + for (name, service) in uploads { + let http = crate::http_config::HttpConfig::for_host(config, name)?; + for existing in &routes { + ensure!( + existing.config.host != http.host + || !(upload_mount_matches( + &existing.config.file_share_path, + &http.file_share_path + ) || upload_mount_matches( + &http.file_share_path, + &existing.config.file_share_path + )), + "ambiguous HTTP upload components '{}' and '{}': HTTP host '{}' has overlapping mounts '{}' and '{}'", + existing.name, + name, + http.host, + existing.config.file_share_path, + http.file_share_path + ); + } + routes.push(UploadRoute { + name: name.clone(), + config: http, + service: service.clone(), + }); + } + Ok(Arc::new(routes)) +} +async fn dispatch_upload( + routes: Arc>, + request: axum::extract::Request, +) -> axum::response::Response { + use axum::response::IntoResponse; + let host = request + .headers() + .get(axum::http::header::HOST) + .and_then(|value| value.to_str().ok()); + if let Some(route) = routes.iter().find(|route| { + host.is_some_and(|host| route.config.accepts_host(host)) + && upload_mount_matches(&route.config.file_share_path, request.uri().path()) + }) { + let mut response = route + .service + .handle_at(request, &route.config.file_share_path) + .await; + response + .extensions_mut() + .insert(crate::http_transport::HttpRoute::Upload); + return response; + } + axum::http::StatusCode::NOT_FOUND.into_response() +} struct ConnectionLog { logger: Logger, id: String, @@ -557,6 +687,7 @@ pub async fn serve(path: &Path) -> Result<()> { let (data, _, hash) = account_config(path)?; let data_guard = Arc::new(crate::data_lock::DataDirectoryGuard::acquire(&data)?); + crate::upload_migration::ensure_no_pending(&data)?; let ssl = c.option(&domain, "ssl").context("ssl required")?; let cert = ssl .get("certificate") @@ -630,8 +761,8 @@ pub async fn serve(path: &Path) -> Result<()> { .and_then(|v| v.as_str()) .map(|s| s.trim_end_matches('/').to_owned()) .unwrap_or(base); - let service = - crate::http_service::UploadService::new(crate::http_service::UploadConfig { + let service = crate::http_service::UploadService::new_with_backend( + crate::http_service::UploadConfig { root: data.clone(), host: name.clone(), base_url: base, @@ -667,17 +798,30 @@ pub async fn serve(path: &Path) -> Result<()> { .option(name, "http_file_share_secret") .and_then(|v| v.as_str()) .map(str::to_owned), - })?; + }, + upload_backend(&c, name)?, + )?; + service.recover_backend().await?; + for slot in service.quarantined_slots() { + logger.event( + Level::Warn, + "upload", + None, + "completion-quarantined", + &[ + ("component", name), + ("slot", &slot), + ("reason", "remote-completion-outcome-uncertain"), + ], + ); + } uploads.insert(name.clone(), service); } Some("pubsub") | None => {} Some(other) => bail!("component not integrated: {other}"), } } - ensure!( - uploads.len() <= 1, - "multiple HTTP upload components not yet integrated" - ); + let upload_routes = upload_routes(&c, &uploads)?; let runtime = Runtime { _data_guard: data_guard, domain: domain.clone(), @@ -892,31 +1036,11 @@ pub async fn serve(path: &Path) -> Result<()> { }) }; let http_config = crate::http_config::HttpConfig::for_host(&c, &domain)?; - let fallback = if let Some((name, service)) = runtime.uploads.iter().next() { - let config = crate::http_config::HttpConfig::for_host(&c, name)?; - service - .router_at(&config.file_share_path)? - .layer(axum::middleware::from_fn( - move |req: axum::extract::Request, next: axum::middleware::Next| { - let accepted = req - .headers() - .get(axum::http::header::HOST) - .and_then(|v| v.to_str().ok()) - .is_some_and(|h| config.accepts_host(h)); - async move { - if accepted { - next.run(req).await - } else { - axum::response::IntoResponse::into_response( - axum::http::StatusCode::NOT_FOUND, - ) - } - } - }, - )) - } else { - axum::Router::new() - }; + let routes = upload_routes.clone(); + let fallback = axum::Router::new().fallback(move |request: axum::extract::Request| { + let routes = routes.clone(); + async move { dispatch_upload(routes, request).await } + }); let app = crate::http_transport::configured_router_with_fallback( handler, http_config, diff --git a/src/upload_migration.rs b/src/upload_migration.rs new file mode 100644 index 0000000..718bc00 --- /dev/null +++ b/src/upload_migration.rs @@ -0,0 +1,565 @@ +//! Offline, source-preserving upload migration. The journal is the startup barrier. +use crate::{ + config::Config, + object_store::{ObjectStore, PART_SIZE, UploadBackendConfig}, + storage, +}; +use anyhow::{Context, Result, ensure}; +use http_body_util::BodyExt; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + io::{Read, Write}, + path::{Path, PathBuf}, +}; + +const JOURNAL: &str = "rust_upload_migration.json"; +#[derive(Serialize, Deserialize)] +struct Journal { + version: u32, + component: String, + source: Value, + destination: Value, + metadata: String, + committed: bool, + objects: BTreeMap, +} +#[derive(Default, Serialize, Deserialize)] +struct Progress { + checksum: Option, + verified: bool, + upload_id: Option, +} +struct Backend(Option); +impl Backend { + fn new(c: UploadBackendConfig) -> Result { + Ok(Self(match c { + UploadBackendConfig::Local => None, + UploadBackendConfig::S3(c) => Some(ObjectStore::new(c)?), + })) + } + fn identity(&self) -> Value { + self.0 + .as_ref() + .map(ObjectStore::identity) + .unwrap_or_else(|| serde_json::json!({"backend":"local"})) + } +} +fn save(path: &Path, journal: &Journal) -> Result<()> { + storage::atomic_write(path, &serde_json::to_vec_pretty(journal)?) +} +fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} +fn local_path(dir: &Path, id: &str) -> PathBuf { + dir.join("http_file_share") + .join(format!("{}.bin", storage::encode_name(id))) +} + +/// Check all component directories, including components removed from the new config. +pub fn ensure_no_pending(root: &Path) -> Result<()> { + if !root.exists() { + return Ok(()); + } + for entry in std::fs::read_dir(root)? { + let path = entry?.path().join(JOURNAL); + if path.exists() { + let journal: Journal = serde_json::from_slice(&std::fs::read(&path)?) + .context("invalid upload migration journal; offline recovery required")?; + ensure!( + journal.version == 1 && journal.committed, + "pending upload migration; resume upload-migrate before starting the server" + ); + } + } + Ok(()) +} +fn validate_component(config: &Config, component: &str) -> Result<()> { + ensure!( + config + .hosts + .get(component) + .and_then(|h| h.component.as_deref()) + == Some("http_file_share"), + "component must be an HTTP file share in both configurations" + ); + Ok(()) +} +fn stable_settings(old: &Config, new: &Config, component: &str) -> Result<()> { + // Keep URL construction and JWT verification exactly as configured, including inherited options. + for key in [ + "http_file_share_secret", + "http_file_share_base_url", + "http_host", + "http_default_host", + "http_external_url", + "http_paths", + "http_ports", + "https_ports", + ] { + ensure!( + old.option(component, key) == new.option(component, key), + "migration must preserve {key}; change public routing separately after migration" + ); + } + Ok(()) +} + +/// Plan by default. Execution never deletes source objects or rewrites native metadata. +pub async fn migrate( + old_path: &Path, + new_path: &Path, + component: &str, + execute: bool, +) -> Result<()> { + let (old, old_domain) = crate::server::settings_config(old_path)?; + let (new, new_domain) = crate::server::settings_config(new_path)?; + ensure!( + old_domain == new_domain, + "virtual host must remain unchanged" + ); + for key in ["http_ports", "https_ports", "data_path"] { + ensure!( + old.option(&old_domain, key) == new.option(&new_domain, key), + "migration must preserve runtime {key}" + ); + } + stable_settings(&old, &new, component)?; + validate_component(&old, component)?; + validate_component(&new, component)?; + let root = PathBuf::from( + old.option(&old_domain, "data_path") + .and_then(Value::as_str) + .context("data_path required")?, + ); + let new_root = PathBuf::from( + new.option(&new_domain, "data_path") + .and_then(Value::as_str) + .context("data_path required")?, + ); + ensure!( + root.is_dir() && new_root.is_dir(), + "existing data directories required" + ); + ensure!( + root.canonicalize()? == new_root.canonicalize()?, + "migration requires the same data directory; native metadata must stay in place" + ); + let _guard = crate::data_lock::DataDirectoryGuard::acquire(&root)?; + let source = Backend::new(crate::server::upload_backend(&old, component)?)?; + let target = Backend::new(crate::server::upload_backend(&new, component)?)?; + ensure!( + source.identity() != target.identity(), + "source and destination backend identities are identical" + ); + let dir = root.join(storage::encode_name(component)); + let cache = match std::fs::read(dir.join("rust_uploads.json")) { + Ok(v) => v, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => b"{}".to_vec(), + Err(e) => return Err(e.into()), + }; + let records: BTreeMap = serde_json::from_slice(&cache)?; + ensure!( + !storage::path(&root, component, None, "uploads") + .with_extension("rappend") + .exists(), + "upload archive has pending append recovery; recover source server before migration" + ); + let entries = crate::archive::load_store(&root, component, None, "uploads")?; + let mut seen = std::collections::BTreeSet::new(); + let mut objects = BTreeMap::new(); + let mut fingerprint = Sha256::new(); + // Lua table traversal can reorder reconstructed XML attributes between + // processes. Bind exact authoritative archive bytes, not reconstructed XML. + fingerprint.update((cache.len() as u64).to_le_bytes()); + fingerprint.update(&cache); + let archive_path = storage::path(&root, component, None, "uploads").with_extension("list"); + let archive_bytes = match std::fs::read(archive_path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(e.into()), + }; + fingerprint.update((archive_bytes.len() as u64).to_le_bytes()); + fingerprint.update(&archive_bytes); + for e in entries { + ensure!( + seen.insert(e.id.clone()), + "duplicate authoritative upload id" + ); + let doc = roxmltree::Document::parse(&e.xml)?; + let node = doc.root_element(); + let size: u64 = node + .attribute("size") + .context("upload size missing")? + .parse()?; + let filename = node + .attribute("filename") + .context("upload filename missing")?; + let r = records.get(&e.id); + if let Some(r) = r { + ensure!( + r.get("owner").and_then(Value::as_str) == Some(&e.peer) + && r.get("filename").and_then(Value::as_str) == Some(filename) + && r.get("size").and_then(Value::as_u64) == Some(size), + "slot metadata/cache mismatch; recover source server first" + ); + ensure!( + r.get("upload_id").is_none_or(Value::is_null) + && r.get("multipart_intent").and_then(Value::as_bool) != Some(true) + && r.get("completion_attempted").and_then(Value::as_bool) != Some(true), + "slot has unfinished multipart state; recover source server first" + ); + } + // Native Prosody files need no Rust completion sidecar. A present final + // object of exact declared size is completed; absent reservations stay pending. + let complete = r.and_then(|r| r.get("complete")).and_then(Value::as_bool) == Some(true); + let present = if complete { + true + } else if let Some(store) = &source.0 { + match store.stat(&e.id).await? { + Some(n) => { + ensure!(n == size, "source object size mismatch"); + true + } + None => false, + } + } else { + local_path(&dir, &e.id).exists() + }; + if complete || present { + ensure!( + objects + .insert( + e.id.clone(), + ( + size, + node.attribute("content-type") + .unwrap_or("application/octet-stream") + .to_owned() + ) + ) + .is_none(), + "duplicate upload id" + ); + } + } + let metadata = hex::encode(fingerprint.finalize()); + let journal_path = dir.join(JOURNAL); + let mut resumed = false; + let fresh = || Journal { + version: 1, + component: component.into(), + source: source.identity(), + destination: target.identity(), + metadata: metadata.clone(), + committed: false, + objects: objects + .keys() + .map(|id| (id.clone(), Progress::default())) + .collect(), + }; + let mut journal = if journal_path.exists() { + let j: Journal = serde_json::from_slice(&std::fs::read(&journal_path)?)?; + if j.version == 1 + && j.committed + && (j.source != source.identity() + || j.destination != target.identity() + || j.metadata != metadata) + { + fresh() + } else { + ensure!( + j.version == 1 + && j.component == component + && j.source == source.identity() + && j.destination == target.identity() + && j.metadata == metadata + && j.objects.keys().eq(objects.keys()), + "migration journal does not match configurations or metadata" + ); + resumed = true; + j + } + } else { + fresh() + }; + let identity_path = dir.join("rust_upload_backend.json"); + let current: Value = if identity_path.exists() { + serde_json::from_slice(&std::fs::read(&identity_path)?)? + } else { + serde_json::json!({"backend":"local"}) + }; + ensure!( + current == source.identity() || (resumed && current == target.identity()), + "recorded backend is not the migration source" + ); + println!( + "{} completed objects; source objects and all native metadata will be preserved", + objects.len() + ); + if !execute { + println!( + "Dry run: no objects, journal or backend identity changed. Use --execute to copy and verify." + ); + return Ok(()); + } + if journal.committed { + ensure!( + current == target.identity(), + "committed migration identity mismatch" + ); + println!("Migration already committed."); + return Ok(()); + } + save(&journal_path, &journal)?; + for (id, (size, content_type)) in &objects { + // Deterministic journal-owned spools can be removed safely after a crash. + let temp = dir.join(format!(".migration-{}.tmp", digest(id.as_bytes()))); + if temp.exists() { + std::fs::remove_file(&temp)?; + } + let result = async { + let checksum = spool(&source, &dir, id, *size, &temp) + .await? + .context("source object missing")?; + if let Some(previous) = &journal.objects[id].checksum { + ensure!( + *previous == checksum, + "source checksum changed during migration" + ); + } + journal.objects.get_mut(id).unwrap().checksum = Some(checksum.clone()); + save(&journal_path, &journal)?; + // The durable per-object checksum precedes initiation. Sweep this + // exact key, including an initiation whose response was lost. + if let Some(store) = &target.0 { + store.abort_pending(id).await?; + } + if let Some(upload_id) = journal.objects[id].upload_id.as_ref() { + target + .0 + .as_ref() + .context("invalid local multipart journal")? + .abort(id, upload_id) + .await?; + journal.objects.get_mut(id).unwrap().upload_id = None; + save(&journal_path, &journal)?; + } + let existing = object_checksum(&target, &dir, id, *size).await; + let existing = existing?; + if let Some(existing) = existing { + ensure!( + existing == checksum, + "destination object conflicts; refusing overwrite" + ); + } else { + ensure!( + !journal.objects[id].verified, + "previously verified destination object disappeared" + ); + if let Some(store) = &target.0 { + let mut upload = store.begin(id, content_type).await?; + journal.objects.get_mut(id).unwrap().upload_id = + Some(upload.upload_id().into()); + save(&journal_path, &journal)?; + let mut file = std::fs::File::open(&temp)?; + loop { + let mut part = vec![0; PART_SIZE]; + let mut n = 0; + while n < part.len() { + let read = file.read(&mut part[n..])?; + if read == 0 { + break; + } + n += read; + } + part.truncate(n); + if n == 0 { + if *size == 0 { + upload.part(part).await?; + } + break; + } + upload.part(part).await?; + } + upload.complete().await?; + journal.objects.get_mut(id).unwrap().upload_id = None; + save(&journal_path, &journal)?; + } else { + let path = local_path(&dir, id); + std::fs::create_dir_all(path.parent().unwrap())?; + // Hard-link publishes without clobbering an existing object; spool is on the same filesystem. + std::fs::hard_link(&temp, &path)?; + std::fs::File::open(path.parent().unwrap())?.sync_all()?; + } + let verified = object_checksum(&target, &dir, id, *size).await; + ensure!( + verified?.as_deref() == Some(checksum.as_str()), + "destination checksum mismatch" + ); + } + journal.objects.get_mut(id).unwrap().verified = true; + save(&journal_path, &journal)?; + Ok::<_, anyhow::Error>(()) + } + .await; + let _ = std::fs::remove_file(&temp); + result.with_context(|| { + format!( + "migration stopped at object {id}; source preserved, resume with the same command" + ) + })?; + } + ensure!( + journal.objects.values().all(|p| p.verified), + "unverified migration objects" + ); + storage::atomic_write(&identity_path, &serde_json::to_vec(&target.identity())?)?; + journal.committed = true; + save(&journal_path, &journal)?; + println!("Migration committed: all destination objects checksum verified; source preserved."); + Ok(()) +} + +/// Stream through a private, bounded spool; never trust ETag as a content checksum. +async fn spool( + backend: &Backend, + dir: &Path, + id: &str, + size: u64, + temp: &Path, +) -> Result> { + spool_to(backend, dir, id, size, Some(temp)).await +} +async fn object_checksum( + backend: &Backend, + dir: &Path, + id: &str, + size: u64, +) -> Result> { + spool_to(backend, dir, id, size, None).await +} +async fn spool_to( + backend: &Backend, + dir: &Path, + id: &str, + size: u64, + temp: Option<&Path>, +) -> Result> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = temp.map(|temp| options.open(temp)).transpose()?; + let mut hash = Sha256::new(); + let mut total = 0u64; + if let Some(store) = &backend.0 { + let response = store.read(id, false, None).await?; + if response.status() == 404 { + return Ok(None); + } + ensure!( + response.status() == 200, + "object download failed or returned partial content" + ); + let mut body = response.into_body(); + while let Some(frame) = + tokio::time::timeout(std::time::Duration::from_secs(120), body.frame()) + .await + .context("migration download timed out")? + { + let frame = frame?; + if let Some(bytes) = frame.data_ref() { + total = total + .checked_add(bytes.len() as u64) + .context("size overflow")?; + ensure!(total <= size, "object exceeds metadata size"); + hash.update(bytes); + if let Some(file) = &mut file { + file.write_all(bytes)?; + } + } + } + } else { + let path = local_path(dir, id); + let metadata = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + ensure!( + metadata.file_type().is_file() && metadata.len() == size, + "local object is not a regular file of metadata size" + ); + let mut source = std::fs::File::open(path)?; + let mut bytes = vec![0; PART_SIZE]; + loop { + let n = source.read(&mut bytes)?; + if n == 0 { + break; + } + total += n as u64; + ensure!(total <= size, "object exceeds metadata size"); + hash.update(&bytes[..n]); + if let Some(file) = &mut file { + file.write_all(&bytes[..n])?; + } + } + } + ensure!(total == size, "object size differs from metadata"); + if let Some(file) = file { + file.sync_all()?; + } + Ok(Some(hex::encode(hash.finalize()))) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn local_spool_checks_size_and_hash_and_preserves_source() { + let d = tempfile::tempdir().unwrap(); + let path = local_path(d.path(), "a/b"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"payload").unwrap(); + let temp = d.path().join("spool"); + assert_eq!( + spool(&Backend(None), d.path(), "a/b", 7, &temp) + .await + .unwrap(), + Some(digest(b"payload")) + ); + assert_eq!(std::fs::read(&path).unwrap(), b"payload"); + assert!( + spool(&Backend(None), d.path(), "a/b", 8, &d.path().join("bad")) + .await + .is_err() + ); + } + #[test] + fn pending_and_corrupt_journals_block_startup() { + let d = tempfile::tempdir().unwrap(); + let path = d.path().join("removed-component").join(JOURNAL); + let mut j = Journal { + version: 1, + component: "upload".into(), + source: Value::Null, + destination: Value::Null, + metadata: "a".into(), + committed: false, + objects: BTreeMap::new(), + }; + save(&path, &j).unwrap(); + assert!(ensure_no_pending(d.path()).is_err()); + j.committed = true; + save(&path, &j).unwrap(); + assert!(ensure_no_pending(d.path()).is_ok()); + std::fs::write(path, b"broken").unwrap(); + assert!(ensure_no_pending(d.path()).is_err()); + } +} diff --git a/tests/upload_large_network.py b/tests/upload_large_network.py new file mode 100644 index 0000000..1bff3bf --- /dev/null +++ b/tests/upload_large_network.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Opt-in real XEP-0363 HTTPS large-file regression (default: 10 GiB). + +Example: python3 tests/upload_large_network.py --binary /tmp/rust-xmpp +Writes the actual complete payload through TLS to the server's local file +stream storage, then reads every byte back through verified TLS. No source +file, sparse allocation, external server, or body-sized Python buffer is used. +Not part of the default local check because it needs >10 GiB free disk. +""" +import argparse +import hashlib +import http.client +import json +from pathlib import Path +import shutil +import socket +import ssl +import subprocess +import tempfile +import threading +import time +from urllib.parse import urlsplit +import xml.etree.ElementTree as ET + +from _wire_client import Client, BIN + +NS = 'urn:xmpp:http:upload:0' +CHUNK = bytes(range(256)) * 4096 # one deterministic 1 MiB buffer + + +def free_port(): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + return probe.getsockname()[1] + + +def proc_status_for_child(pid): + # /proc can expose host PIDs while Popen.pid is in a nested PID namespace. + parent = str(Path('/proc/self').resolve().name) + for entry in Path('/proc').iterdir(): + if not entry.name.isdigit(): + continue + try: + fields = dict(line.split(':', 1) for line in (entry/'status').read_text().splitlines() if ':' in line) + namespace_pids = fields.get('NSpid', fields.get('Pid', '')).split() + if fields.get('PPid', '').strip() == parent and namespace_pids[-1:] == [str(pid)]: + return entry/'status' + except OSError: + pass + return None + + +def rss(status_path): + if status_path is None: + return 0 + try: + for line in status_path.read_text().splitlines(): + if line.startswith('VmRSS:'): + return int(line.split()[1]) * 1024 + except (OSError, ValueError): + pass + return 0 + + +def progress(phase, count, size, start, last): + now = time.monotonic() + if now - last >= 5 or count == size: + print(f'{phase}: {count}/{size} bytes ({count / max(now-start, .001) / 2**20:.1f} MiB/s)', flush=True) + return now + return last + + +def run(args, report): + with tempfile.TemporaryDirectory(prefix='xmpp-large-', dir=args.temp_dir) as tmp: + root = Path(tmp) + report['free_disk_before_bytes'] = shutil.disk_usage(root).free + assert report['free_disk_before_bytes'] >= args.size + 256 * 2**20, 'Insufficient free disk for real upload' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-keyout', str(root/'key.pem'), '-out', str(root/'cert.pem'), + '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + port, https = free_port(), free_port() + while https == port: + https = free_port() + cert = {'key': str(root/'key.pem'), 'certificate': str(root/'cert.pem')} + config = {'global': {'data_path': str(root/'data'), 'interfaces': ['127.0.0.1'], + 'c2s_ports': [port], 'http_ports': [], 'https_ports': [https], 's2s_ports': [], + 'modules_enabled': ['tls', 'saslauth', 'disco', 'ping'], 'ssl': cert}, + 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'ssl': cert}}, + 'upload.localhost': {'component': 'http_file_share', 'options': { + 'http_host': 'localhost', 'http_file_share_backend': 'local', 'http_file_share_base_url': f'https://localhost:{https}/file_share', + 'http_file_share_size_limit': args.size, + 'http_file_share_daily_quota': args.size, + 'http_file_share_global_quota': args.size, + 'http_file_share_expires_after': 'never'}}}} + config_path = root/'config.json' + config_path.write_text(json.dumps(config)) + subprocess.run([str(args.binary), 'create-user', str(config_path), 'alice', '--password-stdin'], + input='large-file-secret\n', text=True, check=True, capture_output=True) + with (root/'server.log').open('w+') as log: + server = subprocess.Popen([str(args.binary), 'serve', str(config_path), '--experimental'], stdout=log, stderr=log) + stop = threading.Event() + report['max_rss_bytes'] = {'server': 0, 'client': 0} + server_status = proc_status_for_child(server.pid) + report['rss_proc_status_paths'] = {'server': str(server_status), 'client': '/proc/self/status'} + def sample(): + while not stop.is_set(): + for role, status_path in [('server', server_status), ('client', Path('/proc/self/status'))]: + report['max_rss_bytes'][role] = max(report['max_rss_bytes'][role], rss(status_path)) + stop.wait(.05) + monitor = threading.Thread(target=sample, daemon=True) + monitor.start() + client = None + try: + assert server_status is not None, 'cannot resolve actual server /proc status for RSS sampling' + for _ in range(200): + assert server.poll() is None, 'server exited during startup' + try: + with socket.create_connection(('127.0.0.1', port), .1): + break + except OSError: + time.sleep(.05) + else: + raise AssertionError('server startup timed out') + client = Client(port, root/'cert.pem', 'alice', 'large-file-secret') + client.send(f"") + slot = client.iq('large-slot').find(f'{{{NS}}}slot') + assert slot is not None + put = slot.find(f'{{{NS}}}put') + get = slot.find(f'{{{NS}}}get') + put_url, get_url = urlsplit(put.get('url')), urlsplit(get.get('url')) + assert put_url.scheme == get_url.scheme == 'https' + assert put_url.hostname == get_url.hostname == 'localhost' + headers = {h.get('name'): h.text for h in put.findall(f'{{{NS}}}header')} + context = ssl.create_default_context(cafile=str(root/'cert.pem')) + def connect(url): + return http.client.HTTPSConnection(url.hostname, url.port, context=context, timeout=120) + def path(url): + return url.path + ('?' + url.query if url.query else '') + start = last = time.monotonic() + digest = hashlib.sha256() + sent = 0 + conn = connect(put_url) + try: + conn.putrequest('PUT', path(put_url)) + for name, value in {**headers, 'Content-Length': str(args.size), 'Content-Type': 'application/octet-stream'}.items(): + conn.putheader(name, value) + conn.endheaders() + while sent < args.size: + piece = memoryview(CHUNK)[:min(len(CHUNK), args.size - sent)] + conn.send(piece) + digest.update(piece) + sent += len(piece) + last = progress('PUT', sent, args.size, start, last) + response = conn.getresponse() + assert response.status == 201, (response.status, response.read(4096)) + response.read() + finally: + conn.close() + report.update(put_bytes=sent, put_seconds=time.monotonic()-start, sha256=digest.hexdigest()) + # A real complete on-disk file, including allocated blocks, is required. + stored = list((root/'data').glob('*/http_file_share/*.bin')) + assert len(stored) == 1, stored + stat = stored[0].stat() + assert stat.st_size == args.size + assert stat.st_blocks * 512 >= args.size, 'file unexpectedly sparse' + report['stored_bytes'] = stat.st_size + report['allocated_bytes'] = stat.st_blocks * 512 + report['free_disk_after_put_bytes'] = shutil.disk_usage(root).free + conn = connect(get_url) + try: + conn.request('HEAD', path(get_url)) + response = conn.getresponse() + assert response.status == 200 and int(response.getheader('Content-Length')) == args.size + assert response.read() == b'' + finally: + conn.close() + report['head_status'] = 200 + start = last = time.monotonic() + received, reads = 0, 0 + downloaded = hashlib.sha256() + conn = connect(get_url) + try: + conn.request('GET', path(get_url)) + response = conn.getresponse() + assert response.status == 200 and int(response.getheader('Content-Length')) == args.size + while True: + piece = response.read(len(CHUNK)) + reads += 1 + if not piece: + break + downloaded.update(piece) + received += len(piece) + last = progress('GET', received, args.size, start, last) + assert received == args.size and downloaded.digest() == digest.digest() + finally: + conn.close() + report.update(get_bytes=received, get_read_calls=reads, get_seconds=time.monotonic()-start, + get_sha256=downloaded.hexdigest()) + # Exercise a range crossing 4 GiB when the selected size permits it. + offset = min(2**32 - 7, args.size - 32) + conn = connect(get_url) + try: + conn.request('GET', path(get_url), headers={'Range': f'bytes={offset}-{offset+31}'}) + response = conn.getresponse() + assert response.status == 206 + assert response.getheader('Content-Range') == f'bytes {offset}-{offset+31}/{args.size}' + assert response.read(33) == bytes((offset+i) % 256 for i in range(32)) + finally: + conn.close() + report.update(range_status=206, range_offset=offset) + conn = connect(put_url) + try: + # No second large body: single-use check must reject at headers. + conn.putrequest('PUT', path(put_url)) + for name, value in {**headers, 'Content-Length': str(args.size)}.items(): + conn.putheader(name, value) + conn.endheaders() + response = conn.getresponse() + assert response.status == 409, response.status + response.read() + finally: + conn.close() + report['reuse_status'] = 409 + client.send(f"") + rejected = client.iq('over-quota', error=True) + assert rejected.find('.//{urn:ietf:params:xml:ns:xmpp-stanzas}not-acceptable') is not None, ET.tostring(rejected) + report['over_quota_condition'] = 'not-acceptable' + assert 0 < report['max_rss_bytes']['client'] < 256 * 2**20, report['max_rss_bytes'] + assert 0 < report['max_rss_bytes']['server'] < 512 * 2**20, report['max_rss_bytes'] + report['status'] = 'passed' + except BaseException: + log.flush() + log.seek(0) + print(log.read()[-20000:], flush=True) + raise + finally: + if client: + client.close() + server.terminate() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + server.kill() + server.wait() + stop.set() + monitor.join(timeout=2) + report['temporary_files_cleaned'] = not root.exists() + report['free_disk_after_cleanup_bytes'] = shutil.disk_usage(args.temp_dir or tempfile.gettempdir()).free + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, default=BIN) + parser.add_argument('--size', type=int, default=10 * 2**30, help='actual network bytes; default 10 GiB') + parser.add_argument('--report', type=Path, default=Path('/tmp/xmpp078-large.json')) + parser.add_argument('--temp-dir', type=Path) + args = parser.parse_args() + assert args.size >= 32 + args.binary = args.binary.resolve() + report = {'status': 'failed', 'requested_bytes': args.size, 'python_chunk_bytes': len(CHUNK), + 'transport': 'verified TLS HTTPS', 'storage': 'local streaming file', 'rss_sample_interval_seconds': .05} + started = time.monotonic() + try: + run(args, report) + except BaseException as error: + report['error'] = repr(error) + raise + finally: + report['total_seconds'] = time.monotonic() - started + args.report.write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == '__main__': + main() diff --git a/tests/upload_migration_network.py b/tests/upload_migration_network.py new file mode 100644 index 0000000..ca54b7d --- /dev/null +++ b/tests/upload_migration_network.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Offline upload-migrate regression using the independent signed S3 fixture. + +Usage: python3 tests/upload_migration_network.py target/release/rust-xmpp +Exercises real XMPP slots and HTTP bodies; no cloud service or large disk is needed. +The fixture validates SigV4 independently (see upload_s3_network.py). +""" +import copy +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +from urllib.parse import parse_qs, urlsplit + +from _wire_client import Client +from upload_s3_network import (ACCESS, PART, REGION, SECRET, TOKEN, Store, + free_port, request, slot, start, stop) + +HOST = 'upload.localhost' + + +class MigrationStore(Store): + """Fail a chosen completion after earlier objects have really been copied.""" + def __init__(self): + super().__init__() + self.completions = 0 + self.fail_completion = None + + def verify(self, handler, payload): + super().verify(handler, payload) + query = parse_qs(urlsplit(handler.path).query, keep_blank_values=True) + if handler.command == 'POST' and 'uploadId' in query: + self.completions += 1 + if self.completions == self.fail_completion: + self.fail = 'complete-error' + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def checked(command, success=True): + result = subprocess.run(command, capture_output=True, text=True, timeout=90) + output = result.stdout + result.stderr + assert (result.returncode == 0) == success, (command, result.returncode, output) + assert all(value not in output for value in (ACCESS, SECRET, TOKEN)), 'credentials leaked by CLI' + return output + + +def migrate(binary, old, new, execute=False, success=True): + command = [binary, 'upload-migrate', str(old), str(new), '--component', HOST] + if execute: + command.append('--execute') + return checked(command, success) + + +def source_files(directory): + return {str(path.relative_to(directory)): sha(path.read_bytes()) + for path in directory.rglob('*.bin')} + + +def run(binary): + with tempfile.TemporaryDirectory(prefix='xmpp-upload-migration-') as directory, \ + MigrationStore() as first, MigrationStore() as second: + root = Path(directory) + stores = (first, second) + workers = [threading.Thread(target=store.serve_forever, daemon=True) for store in stores] + for worker in workers: + worker.start() + checked(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem'), + '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost']) + c2s, http_port = free_port(), free_port() + while c2s == http_port: + http_port = free_port() + cert = {'key': str(root / 'key.pem'), 'certificate': str(root / 'cert.pem')} + options = {'http_host': '127.0.0.1', 'http_file_share_backend': 'local', + 'http_file_share_base_url': f'http://127.0.0.1:{http_port}/files', + 'http_paths': {'file_share': '/files'}, + 'http_file_share_size_limit': PART + 123, + 'http_file_share_daily_quota': PART * 8, + 'http_file_share_global_quota': PART * 8, + 'http_file_share_expires_after': False} + base = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'], + 'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [http_port], + 'https_ports': [], 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, + 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', + 'password_hash': 'SHA-256', 'ssl': cert}}, + HOST: {'component': 'http_file_share', 'options': options}}} + configs = [] + for name, store in [('local', None), ('s3-first', first), ('s3-second', second)]: + config = copy.deepcopy(base) + if store: + component = config['hosts'][HOST]['options'] + component['http_file_share_backend'] = 's3' + component['http_file_share_s3'] = { + 'endpoint': f'http://127.0.0.1:{store.server_address[1]}/api', + 'bucket': 'fixture-bucket', 'region': REGION, 'access_key': ACCESS, + 'secret_key': SECRET, 'session_token': TOKEN, 'key_prefix': 'space +'} + path = root / f'{name}.json' + path.write_text(json.dumps(config)) + configs.append(path) + local, remote_a, remote_b = configs + result = subprocess.run([binary, 'create-user', str(local), 'alice', '--password-stdin'], + input='secret\n', text=True, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr + component_path = root / 'data' / 'upload%2elocalhost' + identity_path = component_path / 'rust_upload_backend.json' + metadata_path = component_path / 'rust_uploads.json' + archive_path = component_path / 'uploads.list' + process = client = None + uploads = [] + with (root / 'server.log').open('w+') as log: + try: + process = start(binary, local, c2s, log) + client = Client(c2s, root / 'cert.pem', 'alice', 'secret') + for index, payload in enumerate((b'A' * PART + b'B' * 123, b'migration\x00hash\xff', b'')): + url, headers = slot(client, HOST, len(payload), f'migrate-{index}') + assert request(url, 'PUT', payload, headers)[0] == 201 + status, head, body = request(url, 'HEAD') + assert status == 200 and not body + head = {key.lower(): value for key, value in head.items()} + uploads.append((url, headers, payload, head['etag'])) + # Both dry-run and execute need the same exclusive data lock. + migrate(binary, local, remote_a, success=False) + migrate(binary, local, remote_a, execute=True, success=False) + assert not first.events, 'live-server lock denial reached S3' + client.close(); client = None + stop(process); process = None + original_files = source_files(component_path / 'http_file_share') + assert len(original_files) == len(uploads) + original_records = json.loads(metadata_path.read_text()) + original_archive = archive_path.read_bytes() + + def verify_serving(config, preserve_metadata=True): + nonlocal process + process = start(binary, config, c2s, log) + for url, headers, payload, etag in uploads: + status, _, data = request(url) + assert status == 200 and sha(data) == sha(payload), 'original URL body/hash changed' + status, head, body = request(url, 'HEAD') + head = {key.lower(): value for key, value in head.items()} + assert status == 200 and body == b'' + assert int(head['content-length']) == len(payload) and head['etag'] == etag + assert request(url, headers={'If-None-Match': etag})[0] == 304 + assert request(url, 'PUT', payload, headers)[0] == 409, 'original JWT became reusable' + stop(process); process = None + if preserve_metadata: + records = json.loads(metadata_path.read_text()) + stable = lambda rows: {key: {field: value for field, value in record.items() if field != 'xml'} + for key, record in rows.items()} + assert stable(records) == stable(original_records), 'slot/JWT metadata changed' + + # Invalid destination configuration must not mutate source state. + invalid = json.loads(remote_a.read_text()) + del invalid['hosts'][HOST]['options']['http_file_share_s3']['bucket'] + bad = root / 'invalid.json' + bad.write_text(json.dumps(invalid)) + identity = identity_path.read_bytes() + migrate(binary, local, bad, execute=True, success=False) + assert identity_path.read_bytes() == identity and not first.objects + assert source_files(component_path / 'http_file_share') == original_files + + # Default is a plan only: no writes, no identity/metadata cutover. + migrate(binary, local, remote_a) + assert not first.objects and not first.uploads + assert not any(stage in ('init', 'part', 'complete', 'abort', 'delete') + for stage, _, _ in first.events), 'dry-run wrote to destination' + assert identity_path.read_bytes() == identity + assert json.loads(metadata_path.read_text()) == original_records + + # One object is durable before a later copy fails. Neither the + # failed command nor a server restart may cut over early. + first.fail_completion = first.completions + 2 + migrate(binary, local, remote_a, execute=True, success=False) + assert first.objects, 'failure injection did not leave a resumable copied object' + assert identity_path.read_bytes() == identity, 'incomplete migration changed active backend' + assert json.loads(metadata_path.read_text()) == original_records + assert source_files(component_path / 'http_file_share') == original_files + # A pending journal fences both configurations until resume. + for blocked in (local, remote_a): + rejected = subprocess.Popen([binary, 'serve', str(blocked), '--experimental'], + stdout=log, stderr=log) + try: + assert rejected.wait(timeout=15) != 0, 'pending migration started serving' + finally: + if rejected.poll() is None: + rejected.kill(); rejected.wait(timeout=10) + first.fail_completion = None + completed_before_resume = first.completions + migration_cache, migration_archive = metadata_path.read_bytes(), archive_path.read_bytes() + migrate(binary, local, remote_a, execute=True) + assert metadata_path.read_bytes() == migration_cache and archive_path.read_bytes() == migration_archive + assert first.completions - completed_before_resume < len(uploads), 'resume recopied verified objects' + assert source_files(component_path / 'http_file_share') == original_files + verify_serving(remote_a) + assert sorted(sha(data) for data in first.objects.values()) == sorted(sha(row[2]) for row in uploads) + + # S3 -> S3 and S3 -> local preserve remote source objects, even + # though final local files already exist from the first backend. + first_objects = dict(first.objects) + for old, new, destination in ((remote_a, remote_b, second), (remote_b, local, None)): + second_objects = dict(second.objects) + before = identity_path.read_bytes() + destination_objects = dict(destination.objects) if destination else None + migrate(binary, old, new) + assert identity_path.read_bytes() == before + if destination: + assert destination.objects == destination_objects + migration_cache, migration_archive = metadata_path.read_bytes(), archive_path.read_bytes() + migrate(binary, old, new, execute=True) + assert metadata_path.read_bytes() == migration_cache and archive_path.read_bytes() == migration_archive + verify_serving(new) + assert first.objects == first_objects, 'S3 source objects were deleted/changed' + if destination is None: + assert second.objects == second_objects, 'second S3 source objects were deleted/changed' + assert sorted(sha(data) for data in second.objects.values()) == sorted(sha(row[2]) for row in uploads) + assert source_files(component_path / 'http_file_share') == original_files + # A native final local file is authoritative even when its Rust + # cache completion flag is false. Unfinished multipart intent + # still rejects the operation without modifying the backend. + native = copy.deepcopy(original_records) + native_id = next(iter(native)) + native[native_id]['multipart_intent'] = True + metadata_path.write_text(json.dumps(native)) + before = identity_path.read_bytes() + migrate(binary, local, remote_a, execute=True, success=False) + assert identity_path.read_bytes() == before + native[native_id] = copy.deepcopy(original_records[native_id]) + native[native_id]['complete'] = False + metadata_path.write_text(json.dumps(native)) + native_bytes = metadata_path.read_bytes() + native_archive = archive_path.read_bytes() + migrate(binary, local, remote_a, execute=True) + assert metadata_path.read_bytes() == native_bytes, 'migration rewrote native completion cache' + assert archive_path.read_bytes() == native_archive + verify_serving(remote_a, preserve_metadata=False) + assert all(not store.errors for store in stores), [store.errors for store in stores] + print('PASS upload migration: local -> S3 -> S3 -> local, hashes, original URLs/JWT/HEAD, lock, dry-run, fail-closed resume') + except BaseException: + log.flush(); log.seek(0) + output = log.read() + for credential in (ACCESS, SECRET, TOKEN): + output = output.replace(credential, '[REDACTED]') + print('SERVER LOG:\n' + output) + print('FIXTURE ERRORS:', [store.errors for store in stores]) + raise + finally: + if client is not None: + client.close() + if process is not None: + stop(process) + for store in stores: + store.shutdown() + for worker in workers: + worker.join(timeout=5) + + +if __name__ == '__main__': + run(str(Path(sys.argv[1] if len(sys.argv) > 1 else 'target/release/rust-xmpp').resolve())) diff --git a/tests/upload_minio_network.py b/tests/upload_minio_network.py new file mode 100644 index 0000000..51cceb8 --- /dev/null +++ b/tests/upload_minio_network.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Opt-in genuine MinIO integration; no S3 fixture is used. + +MINIO_BINARY=/path/to/minio python3 tests/upload_minio_network.py --binary target/release/rust-xmpp +Default sends an actual 10 GiB HTTPS upload through Rust into isolated MinIO, +then verifies every byte through HTTPS, multipart ETag, ranges, restart and crash +cleanup. Requires openssl, a genuine MinIO executable, and size + 512 MiB free. +--size permits a small smoke check; only sizes >5 GiB qualify the large-object test. +""" +import argparse +import hashlib +import hmac +import http.client +import json +import os +from pathlib import Path +import shutil +import socket +import ssl +import subprocess +import tempfile +import threading +import time +from urllib.parse import quote, urlsplit +import xml.etree.ElementTree as ET + +from _wire_client import Client +from upload_large_network import CHUNK, free_port, proc_status_for_child, rss, progress +from upload_s3_network import slot, start, stop, wait_for + +ACCESS = 'isolated-minio-test-access' +SECRET = 'isolated-minio-test-secret-not-production' +BUCKET = 'xmpp-integration' +PART = 8 * 2**20 + + +def signed(port, method, path, query='', body=b''): + """Small independent SigV4 administrative client, never stores object bodies.""" + date = time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + scope = date[:8] + '/us-east-1/s3/aws4_request' + payload = hashlib.sha256(body).hexdigest() + headers = {'host': f'127.0.0.1:{port}', 'x-amz-content-sha256': payload, 'x-amz-date': date} + names = ';'.join(sorted(headers)) + canonical = '\n'.join((method, path, query, ''.join(f'{k}:{headers[k]}\n' for k in sorted(headers)), names, payload)) + key = ('AWS4' + SECRET).encode() + for value in (date[:8], 'us-east-1', 's3', 'aws4_request'): + key = hmac.new(key, value.encode(), hashlib.sha256).digest() + value = '\n'.join(('AWS4-HMAC-SHA256', date, scope, hashlib.sha256(canonical.encode()).hexdigest())) + signature = hmac.new(key, value.encode(), hashlib.sha256).hexdigest() + headers['Authorization'] = f'AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, SignedHeaders={names}, Signature={signature}' + conn = http.client.HTTPConnection('127.0.0.1', port, timeout=30) + try: + conn.request(method, path + ('?' + query if query else ''), body, headers) + response = conn.getresponse() + return response.status, dict(response.getheaders()), response.read() + finally: + conn.close() + + +def uploads(port): + status, _, body = signed(port, 'GET', '/' + BUCKET, 'uploads=') + assert status == 200, (status, body[:1024]) + return [n for n in ET.fromstring(body).iter() if n.tag.endswith('}Upload')] + + +def has_uploaded_part(port): + for upload in uploads(port): + values = {node.tag.rsplit('}', 1)[-1]: node.text for node in upload} + path = '/' + BUCKET + '/' + quote(values['Key'], safe='/') + status, _, body = signed(port, 'GET', path, 'uploadId=' + quote(values['UploadId'], safe='-_.~')) + if status == 200 and any(node.tag.endswith('}Part') for node in ET.fromstring(body).iter()): + return True + return False + + +def run(args, report): + try: + version = subprocess.check_output([str(args.minio), '--version'], text=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as error: + raise RuntimeError('MinIO preflight failed before any upload: ' + error.output[-4000:]) from error + assert 'version ' in version.lower() and 'MinIO, Inc.' in version, 'supplied executable does not identify as MinIO' + binary_hash = hashlib.sha256() + with args.minio.open('rb') as stream: + for piece in iter(lambda: stream.read(2**20), b''): binary_hash.update(piece) + rust_hash = hashlib.sha256() + with args.binary.open('rb') as stream: + for piece in iter(lambda: stream.read(2**20), b''): rust_hash.update(piece) + rust_version = subprocess.check_output([str(args.binary), '--version'], text=True).strip() + report.update(minio_version=version.strip(), minio_binary_sha256=binary_hash.hexdigest(), binary=str(args.binary), binary_version=rust_version, binary_sha256=rust_hash.hexdigest(), size=args.size, qualifies_over_5gib=args.size > 5 * 2**30) + if args.minio_compat_patch: + patch = args.minio_compat_patch.read_bytes() + report['minio_test_only_compatibility_patch'] = {'sha256': hashlib.sha256(patch).hexdigest(), 'diff': patch.decode(), 'limitation': 'Modified upstream MinIO: EPERM interface enumeration uses loopback; all S3/auth/storage code remains upstream.'} + with tempfile.TemporaryDirectory(prefix='xmpp-minio-', dir=args.temp_dir) as directory: + root = Path(directory) + assert shutil.disk_usage(root).free > args.size + 512 * 2**20, 'insufficient free disk' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-keyout', str(root/'key.pem'), '-out', str(root/'cert.pem'), '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + c2s, https, minio_port, console = [free_port() for _ in range(4)] + cert = {'key': str(root/'key.pem'), 'certificate': str(root/'cert.pem')} + options = {'http_host': 'localhost', 'http_file_share_backend': 's3', 'http_file_share_s3': {'endpoint': f'http://127.0.0.1:{minio_port}', 'bucket': BUCKET, 'region': 'us-east-1', 'access_key': ACCESS, 'secret_key': SECRET, 'key_prefix': 'integration'}, 'http_file_share_base_url': f'https://localhost:{https}/file_share', 'http_file_share_size_limit': args.size, 'http_file_share_daily_quota': args.size * 3, 'http_file_share_global_quota': args.size * 3, 'http_file_share_expires_after': False} + config = {'global': {'data_path': str(root/'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [], 'https_ports': [https], 'ssl': cert, 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed', 'ssl': cert}}, 'upload.localhost': {'component': 'http_file_share', 'options': options}}} + config_path = root/'config.json' + config_path.write_text(json.dumps(config)) + subprocess.run([str(args.binary), 'create-user', str(config_path), 'alice', '--password-stdin'], input='integration-secret\n', text=True, check=True, capture_output=True) + minio = server = client = None + done = threading.Event() + report['max_rss_bytes'] = {'server': 0, 'client': 0, 'minio': 0} + paths = {'client': Path('/proc/self/status')} + def sample(): + while not done.wait(.05): + for role, path in list(paths.items()): + report['max_rss_bytes'][role] = max(report['max_rss_bytes'][role], rss(path)) + monitor = threading.Thread(target=sample, daemon=True) + context = ssl.create_default_context(cafile=str(root/'cert.pem')) + def request(url, method='GET', headers=None): + parsed = urlsplit(url) + connection = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120) + connection.request(method, parsed.path, headers=headers or {}) + return connection, connection.getresponse() + def check_range(url, offset): + conn, response = request(url, headers={'Range': f'bytes={offset}-{offset+31}'}) + try: + assert response.status == 206 + assert response.getheader('Content-Range') == f'bytes {offset}-{offset+31}/{args.size}' + assert response.read(33) == bytes((offset+i) % 256 for i in range(32)) + finally: + conn.close() + with (root/'minio.log').open('w+') as minio_log, (root/'server.log').open('w+') as log: + try: + env = dict(os.environ, MINIO_ROOT_USER=ACCESS, MINIO_ROOT_PASSWORD=SECRET, MINIO_BROWSER='off', GOMAXPROCS='4', GOMEMLIMIT='1GiB') + minio = subprocess.Popen([str(args.minio), 'server', str(root/'minio-data'), '--address', f'127.0.0.1:{minio_port}', '--console-address', f'127.0.0.1:{console}'], env=env, stdout=minio_log, stderr=minio_log) + paths['minio'] = proc_status_for_child(minio.pid) + for _ in range(200): + assert minio.poll() is None, 'MinIO exited at startup' + try: + status, _, _ = signed(minio_port, 'PUT', '/' + BUCKET) + if status in (200, 409): break + except OSError: pass + time.sleep(.05) + else: raise AssertionError('MinIO startup timeout') + server = start(str(args.binary), config_path, c2s, log) + paths['server'] = proc_status_for_child(server.pid) + assert all(paths.values()), 'cannot resolve actual child process RSS paths' + monitor.start() + client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret') + url, headers = slot(client, 'upload.localhost', args.size, 'real-minio') + parsed = urlsplit(url) + assert parsed.scheme == 'https' and parsed.hostname == 'localhost' + conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=120) + digest = hashlib.sha256(); sent = 0; began = last = time.monotonic() + try: + conn.putrequest('PUT', parsed.path) + for name, value in {**headers, 'Content-Length': str(args.size)}.items(): conn.putheader(name, value) + conn.endheaders() + while sent < args.size: + piece = memoryview(CHUNK)[:min(len(CHUNK), args.size-sent)] + conn.send(piece); digest.update(piece); sent += len(piece) + last = progress('MINIO PUT', sent, args.size, began, last) + response = conn.getresponse() + assert response.status == 201, (response.status, response.read(4096)) + response.read() + finally: conn.close() + report.update(put_bytes=sent, put_seconds=time.monotonic()-began, sha256=digest.hexdigest()) + # MinIO's own HEAD proves the object is multipart, independent of Rust metadata. + object_id = parsed.path.split('/')[-2] + status, remote, body = signed(minio_port, 'HEAD', '/' + BUCKET + '/integration/' + object_id) + assert status == 200, (status, body) + remote = {k.lower(): v for k, v in remote.items()} + assert int(remote['content-length']) == args.size + etag = remote['etag'].strip('"') + assert '-' in etag and int(etag.rsplit('-', 1)[1]) == (args.size + PART - 1)//PART, etag + allocated = sum(path.stat().st_blocks * 512 for path in (root/'minio-data').rglob('*') if path.is_file()) + assert allocated >= args.size, 'MinIO payload storage unexpectedly sparse' + report.update(minio_etag=etag, multipart_parts=int(etag.rsplit('-', 1)[1]), minio_allocated_bytes=allocated, free_disk_after_put_bytes=shutil.disk_usage(root).free) + conn, response = request(url, 'HEAD') + assert response.status == 200 and int(response.getheader('Content-Length')) == args.size + response.read(); conn.close() + received = 0; downloaded = hashlib.sha256(); began = last = time.monotonic() + conn, response = request(url) + try: + assert response.status == 200 + while True: + piece = response.read(len(CHUNK)) + if not piece: break + downloaded.update(piece); received += len(piece) + last = progress('MINIO GET', received, args.size, began, last) + finally: conn.close() + assert received == args.size and downloaded.digest() == digest.digest() + report.update(get_bytes=received, get_sha256=downloaded.hexdigest(), get_seconds=time.monotonic()-began) + offsets = sorted(set([0, min(2**32-7, args.size-32), min(5*2**30-7, args.size-32), args.size-32])) + for offset in offsets: check_range(url, offset) + report['range_offsets'] = offsets + client.close(); client = None + stop(server); server = None + server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid) + check_range(url, args.size-32) + report['restart_range_ok'] = True + replay = http.client.HTTPSConnection(parsed.hostname, parsed.port, context=context, timeout=30) + try: + replay.putrequest('PUT', parsed.path) + for name, value in {**headers, 'Content-Length': str(args.size)}.items(): replay.putheader(name, value) + replay.endheaders() + response = replay.getresponse() + assert response.status == 409, 'single-use upload state did not survive restart' + response.read() + finally: replay.close() + report['restart_reuse_status'] = 409 + client = Client(c2s, root/'cert.pem', 'alice', 'integration-secret') + crash_size = max(PART*2, min(args.size, PART*3)) + # The configured maximum must also permit the crash probe in small smoke mode. + crash_size = min(crash_size, args.size) + assert crash_size > PART, 'size must exceed 8 MiB for crash multipart test' + crash_url, crash_headers = slot(client, 'upload.localhost', crash_size, 'crash-minio') + crash = urlsplit(crash_url) + conn = http.client.HTTPSConnection(crash.hostname, crash.port, context=context, timeout=30) + conn.putrequest('PUT', crash.path) + for name, value in {**crash_headers, 'Content-Length': str(crash_size)}.items(): conn.putheader(name, value) + conn.endheaders() + for _ in range(PART//len(CHUNK)): conn.send(CHUNK) + wait_for(lambda: has_uploaded_part(minio_port), 'MinIO never stored crash multipart part') + crash_id = crash.path.split('/')[-2] + metadata = root/'data'/'upload%2elocalhost'/'rust_uploads.json' + wait_for(lambda: json.loads(metadata.read_text())[crash_id].get('upload_id'), 'multipart id not durable') + server.kill(); server.wait(timeout=10); server = None + conn.close(); client.close(); client = None + assert uploads(minio_port), 'no real abandoned multipart at crash' + server = start(str(args.binary), config_path, c2s, log); paths['server'] = proc_status_for_child(server.pid) + wait_for(lambda: not uploads(minio_port), 'startup did not abort genuine MinIO multipart') + check_range(url, args.size-32) + report['crash_multipart_cleanup_ok'] = True + assert 0 < report['max_rss_bytes']['server'] < args.server_rss_mib * 2**20, report['max_rss_bytes'] + assert 0 < report['max_rss_bytes']['client'] < 256 * 2**20, report['max_rss_bytes'] + assert 0 < report['max_rss_bytes']['minio'] < args.minio_rss_mib * 2**20, report['max_rss_bytes'] + report['ok'] = True + except BaseException: + for name, stream in [('RUST', log), ('MINIO', minio_log)]: + stream.flush(); stream.seek(0) + output = stream.read().replace(SECRET, '[REDACTED]').replace(ACCESS, '[REDACTED]') + print(name + ' LOG:\n' + output[-12000:]) + raise + finally: + done.set() + if monitor.is_alive(): monitor.join(timeout=2) + if client is not None: client.close() + if server is not None: stop(server) + if minio is not None: + minio.terminate() + try: minio.wait(timeout=10) + except subprocess.TimeoutExpired: minio.kill(); minio.wait() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, default=Path('target/release/rust-xmpp')) + parser.add_argument('--minio', type=Path, default=os.environ.get('MINIO_BINARY')) + parser.add_argument('--minio-compat-patch', type=Path, help='record any test-only MinIO environment adapter diff in the report') + parser.add_argument('--size', type=int, default=10 * 2**30) + parser.add_argument('--temp-dir', type=Path) + parser.add_argument('--server-rss-mib', type=int, default=512) + parser.add_argument('--minio-rss-mib', type=int, default=2048) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + parser.error('--minio or MINIO_BINARY must name a genuine MinIO executable') if args.minio is None else None + args.binary, args.minio = args.binary.resolve(), args.minio.resolve() + assert args.size > PART and args.size >= 32 + report = {'ok': False} + try: run(args, report) + except Exception as error: + report['error'] = str(error) + raise + finally: + result = json.dumps(report, indent=2) + print(result, flush=True) + if args.report: args.report.write_text(result + '\n') diff --git a/tests/upload_multi_network.py b/tests/upload_multi_network.py new file mode 100644 index 0000000..cf9b6c0 --- /dev/null +++ b/tests/upload_multi_network.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Real TLS XMPP/HTTP multi-component upload isolation; no external services.""" +import argparse +from contextlib import contextmanager +import copy +import http.client +import json +from pathlib import Path +import socket +import subprocess +import tempfile +import time +from urllib.parse import urlsplit +import xml.etree.ElementTree as ET +from _wire_client import Client, BIN + +UPLOAD = 'urn:xmpp:http:upload:0' +DISCO = 'http://jabber.org/protocol/disco#info' + + +def free_port(): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + return probe.getsockname()[1] + + +def request(port, host, path, method='GET', body=None, authorization=None, extra_headers=None): + connection = http.client.HTTPConnection('127.0.0.1', port, timeout=5) + headers = {'Host': host, 'Connection': 'close'} + if authorization: + headers['Authorization'] = authorization + headers.update(extra_headers or {}) + try: + connection.request(method, path, body, headers) + response = connection.getresponse() + return response.status, response.read() + finally: + connection.close() + + +@contextmanager +def running(binary, config, root): + path = root / 'config.json' + path.write_text(json.dumps(config)) + with (root / 'server.log').open('w+') as log: + process = subprocess.Popen([binary, 'serve', str(path), '--experimental'], stdout=log, stderr=log) + try: + for _ in range(100): + assert process.poll() is None, 'server exited during startup' + try: + with socket.create_connection(('127.0.0.1', config['global']['http_ports'][0]), .1): + break + except OSError: + time.sleep(.05) + else: + raise AssertionError('HTTP listener startup timed out') + yield + except BaseException: + log.flush() + log.seek(0) + print(log.read()) + raise + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def slot(client, component, identifier, size=5, error=False): + client.send(f"") + result = client.iq(identifier, error=error) + if error: + return result + put = result.find(f'{{{UPLOAD}}}slot/{{{UPLOAD}}}put') + return urlsplit(put.get('url')), put.find(f'{{{UPLOAD}}}header').text + + +def exercise(binary, root, shared): + root.mkdir() + cert, key = root / 'cert.pem', root / 'key.pem' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes', '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True) + port, c2s = free_port(), free_port() + hosts = ['files.example.test', 'files.example.test'] if shared else ['a.example.test', 'b.example.test'] + mounts = ['/files/a', '/files/b'] if shared else ['/files', '/files'] + components = ['upload-a.localhost', 'upload-b.localhost'] + config = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'], 'c2s_ports': [c2s], 'http_ports': [port], 'https_ports': [], 's2s_ports': [], 'modules_enabled': ['tls', 'saslauth', 'disco', 'websocket', 'bosh', 'http_altconnect'], 'consider_bosh_secure': True, 'consider_websocket_secure': True, 'ssl': {'certificate': str(cert), 'key': str(key)}}, 'hosts': {'localhost': {'options': {'authentication': 'internal_hashed'}}}} + for index, component in enumerate(components): + config['hosts'][component] = {'component': 'http_file_share', 'options': {'http_host': hosts[index], 'http_paths': {'file_share': mounts[index]}, 'http_file_share_size_limit': 8 * (index + 1), 'http_file_share_daily_quota': 5, 'http_file_share_global_quota': 5, 'http_file_share_secret': f'component-secret-{index}'}} + path = root / 'config.json' + path.write_text(json.dumps(config)) + subprocess.run([binary, 'create-user', str(path), 'alice', '--password-stdin'], input='secret\n', text=True, check=True, capture_output=True) + slots = [] + with running(binary, config, root): + client = Client(c2s, cert, 'alice', 'secret') + try: + client.send(f"") + items = {node.get('jid') for node in client.iq('items').iter()} + assert set(components) <= items, items + for index, component in enumerate(components): + client.send(f"") + info = client.iq(f'info-{index}') + assert any(node.get('var') == UPLOAD for node in info.iter()) + fields = {node.get('var'): node.find('{jabber:x:data}value').text for node in info.iter('{jabber:x:data}field')} + assert fields['max-file-size'] == str(8 * (index + 1)), fields + url, token = slot(client, component, f'slot-{index}') + assert url.hostname == hosts[index] and url.path.startswith(mounts[index] + '/'), url + slots.append((url, token)) + slot(client, component, f'quota-{index}', size=1, error=True) + # A JWT issued by component A must not authorize B's valid slot. + assert request(port, hosts[1], slots[1][0].path, 'PUT', b'hello', slots[0][1])[0] == 403 + for index, (url, token) in enumerate(slots): + assert request(port, hosts[index], url.path, 'PUT', b'hello', token)[0] == 201 + assert request(port, hosts[index].upper() + f':{port}', url.path) == (200, b'hello') + assert request(port, hosts[index], url.path, 'PUT', b'hello', token)[0] == 409 + assert request(port, 'unknown.example.test', url.path)[0] == 404 + other = 1 - index + wrong_path = mounts[other] + url.path[len(mounts[index]):] + assert request(port, hosts[other], wrong_path)[0] == 404 + # Transport and discovery middleware continue working around uploads. + assert request(port, 'localhost', '/.well-known/host-meta')[0] == 200 + assert request(port, 'localhost', '/xmpp-websocket', extra_headers={'Connection': 'Upgrade', 'Upgrade': 'websocket', 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==', 'Sec-WebSocket-Protocol': 'xmpp'})[0] == 101 + assert request(port, 'localhost', '/http-bind', 'POST', b"")[0] == 200 + finally: + client.close() + with running(binary, config, root): + for index, (url, _) in enumerate(slots): + assert request(port, hosts[index], url.path) == (200, b'hello') + log_text = (root / "server.log").read_text() + assert '"route"="upload"' in log_text, "custom upload route missing from HTTP diagnostics" + assert all(token not in log_text for _, token in slots), "upload credentials leaked to logs" + if shared: + for second_path in (mounts[0], mounts[0] + '/nested', '/'): + ambiguous = copy.deepcopy(config) + ambiguous['hosts'][components[1]]['options']['http_paths']['file_share'] = second_path + path.write_text(json.dumps(ambiguous)) + result = subprocess.run([binary, 'serve', str(path), '--experimental'], capture_output=True, text=True, timeout=8) + assert result.returncode != 0 and 'ambiguous HTTP upload components' in result.stderr, result.stderr + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', default=str(BIN)) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix='xmpp-upload-multi-') as temporary: + for shared in (False, True): + exercise(str(Path(args.binary).resolve()), Path(temporary) / str(shared), shared) + print('PASS: multiple upload hosts/shared-host paths, discovery, quotas, JWT isolation, persistence, transport composition and conflict validation') + + +if __name__ == '__main__': + main() diff --git a/tests/upload_s3_network.py b/tests/upload_s3_network.py new file mode 100644 index 0000000..c0d8943 --- /dev/null +++ b/tests/upload_s3_network.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 +"""Real XMPP/HTTP upload integration against an independent S3 protocol fixture. + +Usage: python3 tests/upload_s3_network.py target/release/rust-xmpp +No external service or credentials are used. This validates the S3 HTTP contract +and independently recomputes SigV4; it is not an AWS or MinIO qualification test. +""" +import hashlib +import hmac +import http.client +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import re +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import threading +import time +from urllib.parse import parse_qs, quote, unquote, urlsplit +import xml.etree.ElementTree as ET + +from _wire_client import Client + +NS = 'urn:xmpp:http:upload:0' +ACCESS = 'fixture-access-not-production' +SECRET = 'fixture-secret-not-production' +TOKEN = 'fixture-session-not-production' +REGION = 'fixture-region-1' +PART = 8 * 1024 * 1024 + + +def free_port(): + with socket.socket() as stream: + stream.bind(('127.0.0.1', 0)) + return stream.getsockname()[1] + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +class Store(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self): + super().__init__(('127.0.0.1', 0), Handler) + self.objects = {} + self.uploads = {} + self.events = [] + self.errors = [] + self.fail = None + self.lock = threading.Lock() + + def verify(self, handler, payload): + authorization = handler.headers['Authorization'] + algorithm, fields = authorization.split(' ', 1) + assert algorithm == 'AWS4-HMAC-SHA256', 'wrong signature algorithm' + fields = dict(field.strip().split('=', 1) for field in fields.split(',')) + credential = fields['Credential'].split('/') + assert credential[0] == ACCESS and credential[2:] == [REGION, 's3', 'aws4_request'], 'wrong credential scope' + date = handler.headers['x-amz-date'] + assert credential[1] == date[:8], 'date mismatch' + assert handler.headers['x-amz-security-token'] == TOKEN, 'session token missing' + payload_hash = digest(payload) + assert handler.headers['x-amz-content-sha256'] == payload_hash, 'payload checksum mismatch' + names = fields['SignedHeaders'].split(';') + assert names == sorted(names), 'signed headers not sorted' + assert {'host', 'x-amz-date', 'x-amz-content-sha256', 'x-amz-security-token'} <= set(names), 'unsigned required header' + headers = ''.join(f'{name}:{" ".join(handler.headers[name].split())}\n' for name in names) + parsed = urlsplit(handler.path) + # AWS URI encoding preserves slash separators, but escapes all other + # non-unreserved bytes. Keys here include spaces and plus signs. + pairs = [] + for key, values in parse_qs(parsed.query, keep_blank_values=True).items(): + pairs.extend((quote(key, safe='-_.~'), quote(value, safe='-_.~')) for value in values) + query = '&'.join(f'{key}={value}' for key, value in sorted(pairs)) + canonical = '\n'.join((handler.command, parsed.path, query, headers, ';'.join(names), payload_hash)) + scope = '/'.join(credential[1:]) + string_to_sign = '\n'.join((algorithm, date, scope, digest(canonical.encode()))) + signing_key = ('AWS4' + SECRET).encode() + for value in (credential[1], REGION, 's3', 'aws4_request'): + signing_key = hmac.new(signing_key, value.encode(), hashlib.sha256).digest() + signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest() + assert hmac.compare_digest(signature, fields['Signature']), 'invalid SigV4 signature' + + +class Handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def log_message(self, *args): + pass + + def reply(self, status, payload=b'', **headers): + self.send_response(status) + self.send_header('Content-Length', str(len(payload))) + for name, value in headers.items(): + self.send_header(name.replace('_', '-'), value) + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(payload) + + def do_GET(self): self.dispatch() + def do_HEAD(self): self.dispatch() + def do_POST(self): self.dispatch() + def do_PUT(self): self.dispatch() + def do_DELETE(self): self.dispatch() + + def dispatch(self): + try: + payload = self.rfile.read(int(self.headers.get('Content-Length', '0'))) + self.server.verify(self, payload) + parsed = urlsplit(self.path) + query = parse_qs(parsed.query, keep_blank_values=True) + if self.command == 'GET' and 'uploads' in query: + assert parsed.path == '/api/fixture-bucket', 'list must address bucket root' + assert query.get('encoding-type') == ['url'], 'list must encode arbitrary XML keys' + prefix = query.get('prefix', [''])[0] + with self.server.lock: + self.server.events.append(('list', prefix, len(payload))) + entries = sorted((unquote(upload['key'].removeprefix('/api/fixture-bucket/')), uid) + for uid, upload in self.server.uploads.items() + if unquote(upload['key'].removeprefix('/api/fixture-bucket/')).startswith(prefix)) + marker = (query.get('key-marker', [''])[0], query.get('upload-id-marker', [''])[0]) + entries = [entry for entry in entries if entry > marker] + # Small pages force both marker fields to be exercised. + page, remaining = entries[:2], entries[2:] + root = ET.Element('ListMultipartUploadsResult') + for key, uid in page: + upload = ET.SubElement(root, 'Upload') + ET.SubElement(upload, 'Key').text = quote(key, safe='-_.~') + ET.SubElement(upload, 'UploadId').text = uid + ET.SubElement(root, 'IsTruncated').text = 'true' if remaining else 'false' + if remaining: + ET.SubElement(root, 'NextKeyMarker').text = quote(page[-1][0], safe='-_.~') + ET.SubElement(root, 'NextUploadIdMarker').text = page[-1][1] + return self.reply(200, ET.tostring(root)) + assert parsed.path.startswith('/api/fixture-bucket/space%20%2B/'), 'bucket/prefix escaping mismatch' + + key = parsed.path + with self.server.lock: + stage = ('init' if 'uploads' in query else 'part' if self.command == 'PUT' + else 'complete' if self.command == 'POST' else 'abort' if 'uploadId' in query + else self.command.lower()) + self.server.events.append((stage, key, len(payload))) + if stage == 'complete': + assert self.headers.get('If-None-Match') == '*', 'completion must prevent overwrite' + if self.server.fail == 'complete-malformed': + self.server.fail = None + return self.reply(200, b'InternalError') + if self.server.fail == stage: + self.server.fail = None + return self.reply(503, b'FixtureUnavailable') + if stage == 'init': + uid = f'upload+{len(self.server.events)}/=' + self.server.uploads[uid] = {'key': key, 'parts': {}} + return self.reply(200, f'{uid}'.encode()) + if stage == 'part': + upload = self.server.uploads[query['uploadId'][0]] + number = int(query['partNumber'][0]) + etag = '"' + hashlib.md5(payload).hexdigest() + '"' + upload['parts'][number] = (payload, etag) + return self.reply(200, ETag=etag) + if stage == 'complete': + upload = self.server.uploads[query['uploadId'][0]] + root = ET.fromstring(payload) + parts = list(root) + result = [] + for index, node in enumerate(parts, 1): + number = int(node.findtext('PartNumber')) + assert number == index, 'noncontiguous multipart completion' + data, etag = upload['parts'][number] + assert node.findtext('ETag') == etag, 'ETag mismatch' + if index < len(parts): + assert len(data) >= 5 * 1024 * 1024, 'nonfinal S3 part below 5 MiB' + result.append(data) + self.server.objects[key] = b''.join(result) + del self.server.uploads[query['uploadId'][0]] + if self.server.fail == 'complete-ambiguous': + self.server.fail = None + # The remote object exists, but the completion response + # gives the proxy no trustworthy success confirmation. + return self.reply(200, b'InternalError') + return self.reply(200, b'"complete"') + if stage == 'abort': + removed = self.server.uploads.pop(query['uploadId'][0], None) + return self.reply(204 if removed is not None else 404) + if self.command == 'DELETE': + self.server.objects.pop(key, None) + return self.reply(204) + data = self.server.objects.get(key) + if data is None: + return self.reply(404) + if self.headers.get('Range'): + start, end = self.headers['Range'].removeprefix('bytes=').split('-') + start, end = int(start), int(end) + return self.reply(206, data[start:end + 1], Content_Range=f'bytes {start}-{end}/{len(data)}') + return self.reply(200, data, Content_Type='application/octet-stream') + except (BrokenPipeError, ConnectionResetError): + # The crash recovery case deliberately kills the HTTP peer. + return + except BaseException as error: + self.server.errors.append(f'{type(error).__name__}: {error}') + self.reply(500, b'fixture validation failed') + + +def request(url, method='GET', body=None, headers=None): + parsed = urlsplit(url) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20) + try: + try: + connection.request(method, parsed.path, body=body, headers=headers or {}) + except (BrokenPipeError, ConnectionResetError): + # Authorization, replay, and backend preflight errors may produce + # a response before the large request body finishes transmitting. + pass + response = connection.getresponse() + return response.status, dict(response.getheaders()), response.read() + finally: + connection.close() + + +def slot(client, host, size, ident, error=False): + client.send(f"") + response = client.iq(ident, error=error) + if error: + return response + node = response.find(f'{{{NS}}}slot') + assert node is not None, ET.tostring(response) + put = node.find(f'{{{NS}}}put') + headers = {header.get('name'): header.text for header in put} + return put.get('url'), headers + + +def start(binary, config, port, log): + process = subprocess.Popen([binary, 'serve', str(config), '--experimental'], stdout=log, stderr=log) + for _ in range(100): + assert process.poll() is None, 'server exited during startup' + try: + with socket.create_connection(('127.0.0.1', port), timeout=.1): + return process + except OSError: + time.sleep(.05) + raise AssertionError('server startup timeout') + + +def stop(process): + process.terminate() + try: + assert process.wait(timeout=10) == 0, 'server shutdown failed' + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise + + +def wait_for(predicate, message): + for _ in range(100): + if predicate(): + return + time.sleep(.05) + raise AssertionError(message) + + +def run(binary): + with tempfile.TemporaryDirectory(prefix='xmpp-s3-network-') as directory, Store() as store: + root = Path(directory) + worker = threading.Thread(target=store.serve_forever, daemon=True) + worker.start() + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-keyout', str(root / 'key.pem'), '-out', str(root / 'cert.pem'), '-subj', + '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost'], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + c2s, http_port = free_port(), free_port() + size = PART + 123 + options = {'http_host': '127.0.0.1', 'http_file_share_backend': 's3', 'http_file_share_s3': { + 'endpoint': f'http://127.0.0.1:{store.server_address[1]}/api', + 'bucket': 'fixture-bucket', 'region': REGION, 'access_key': ACCESS, + 'secret_key': SECRET, 'session_token': TOKEN, 'key_prefix': 'space +'}, + 'http_file_share_base_url': f'http://127.0.0.1:{http_port}/s3', + 'http_paths': {'file_share': '/s3'}, 'http_file_share_size_limit': size, + 'http_file_share_daily_quota': size * 7, 'http_file_share_global_quota': size * 7, + 'http_file_share_expires_after': False} + config_data = {'global': {'data_path': str(root / 'data'), 'interfaces': ['127.0.0.1'], + 'c2s_ports': [c2s], 's2s_ports': [], 'http_ports': [http_port], 'https_ports': [], + 'modules_enabled': ['tls', 'saslauth', 'ping', 'disco']}, 'hosts': { + 'localhost': {'options': {'authentication': 'internal_hashed', 'password_hash': 'SHA-256', + 'ssl': {'key': str(root / 'key.pem'), 'certificate': str(root / 'cert.pem')}}}, + 's3.localhost': {'component': 'http_file_share', 'options': options}, + 'local.localhost': {'component': 'http_file_share', 'options': {'http_host': '127.0.0.1', + 'http_file_share_base_url': f'http://127.0.0.1:{http_port}/local', + 'http_paths': {'file_share': '/local'}}}}} + config = root / 'config.json' + config.write_text(json.dumps(config_data)) + subprocess.run([binary, 'create-user', str(config), 'alice', '--password-stdin'], + input='secret\n', text=True, check=True, capture_output=True) + process = None + client = None + with (root / 'server.log').open('w+') as log: + try: + process = start(binary, config, c2s, log) + client = Client(c2s, root / 'cert.pem', 'alice', 'secret') + data = b'A' * PART + b'B' * 123 + url, headers = slot(client, 's3.localhost', size, 'good') + # Send one part and fence on the fixture before sending the + # tail: the proxy must make progress without buffering the file. + parsed = urlsplit(url) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=20) + connection.putrequest('PUT', parsed.path) + for name, value in {**headers, 'Content-Length': str(size)}.items(): + connection.putheader(name, value) + connection.endheaders() + connection.send(data[:PART]) + wait_for(lambda: any(stage == 'part' for stage, _, _ in store.events), 'upload buffered the entire client body') + connection.send(data[PART:]) + response = connection.getresponse() + assert response.status == 201, (response.status, response.read()) + response.read() + connection.close() + assert [length for stage, _, length in store.events if stage == 'part'] == [PART, 123] + assert request(url)[2] == data + status, head, body = request(url, 'HEAD') + assert status == 200 and int(head.get('content-length', head.get('Content-Length', -1))) == size and body == b'' + etag = head.get('etag', head.get('ETag')) + assert etag, 'missing ETag' + count = len(store.events) + assert request(url, headers={'If-None-Match': etag})[0] == 304 + assert request(url, headers={'If-None-Match': 'W/' + etag})[0] == 304 + assert len(store.events) == count, 'conditional hit reached S3' + assert request(url, headers={'Range': 'bytes=1-2', 'If-Range': etag})[2] == b'AA' + status, _, body = request(url, headers={'Range': 'bytes=1-2', 'If-Range': '"stale"'}) + assert status == 200 and body == data + status, range_headers, body = request(url, headers={'Range': f'bytes={PART - 2}-{PART + 2}'}) + assert status == 206 and body == b'AABBB' + assert range_headers.get('content-range', range_headers.get('Content-Range')) == f'bytes {PART - 2}-{PART + 2}/{size}' + assert request(url, headers={'Range': 'bytes=-5'})[2] == b'BBBBB' + assert request(url, headers={'Range': f'bytes={PART}-'})[2] == b'B' * 123 + count = len(store.events) + assert request(url, headers={'Range': f'bytes={size}-'})[0] == 416 + assert len(store.events) == count, 'invalid range reached S3' + for stage, method in (('get', 'GET'), ('head', 'HEAD')): + store.fail = stage + assert request(url, method)[0] >= 500 + assert request(url)[2] == data + assert request(url, 'PUT', data, headers)[0] == 409, 'slot reused' + local_url, local_headers = slot(client, 'local.localhost', 4, 'local') + assert request(local_url, 'PUT', b'here', local_headers)[0] == 201 + assert request(local_url)[2] == b'here' + empty_url, empty_headers = slot(client, 's3.localhost', 0, 'empty') + assert request(empty_url, 'PUT', b'', empty_headers)[0] == 201 + assert request(empty_url)[2] == b'' + for stage in ('init', 'part', 'complete', 'complete-error'): + bad_url, bad_headers = slot(client, 's3.localhost', size, 'fail-' + stage) + store.fail = stage + status, _, response = request(bad_url, 'PUT', data, bad_headers) + assert status >= 500, (stage, status) + assert SECRET.encode() not in response and TOKEN.encode() not in response + assert request(bad_url)[0] in (404, 410), 'failed upload became readable' + wait_for(lambda: not store.uploads, 'failed multipart was not aborted') + assert request(local_url)[2] == b'here', 'S3 failure broke local backend' + # Failed slots remain retryable and only commit on success. + def retry_failed_slot(): + # Abort reconciliation briefly holds the same slot claim. + retry_status = request(bad_url, 'PUT', data, bad_headers)[0] + assert retry_status in (201, 409), (stage, retry_status) + return retry_status == 201 + wait_for(retry_failed_slot, 'failed slot remained unavailable: ' + stage) + ambiguous_url, ambiguous_headers = slot(client, 's3.localhost', size, 'ambiguous-complete') + store.fail = 'complete-ambiguous' + status, _, _ = request(ambiguous_url, 'PUT', data, ambiguous_headers) + assert status >= 500, 'ambiguous completion reported success' + wait_for(lambda: request(ambiguous_url)[0] == 200, 'committed object was not reconciled after ambiguous completion') + assert request(ambiguous_url)[2] == data + assert request(ambiguous_url, 'PUT', data, ambiguous_headers)[0] == 409, 'ambiguous completion allowed object overwrite' + assert not store.uploads + abort_url, abort_headers = slot(client, 's3.localhost', size, 'disconnect') + parsed = urlsplit(abort_url) + with socket.create_connection((parsed.hostname, parsed.port), timeout=10) as stream: + head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n' + head += ''.join(f'{key}: {value}\r\n' for key, value in abort_headers.items()) + '\r\n' + stream.sendall(head.encode() + data[:PART]) + wait_for(lambda: any(stage == 'part' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnect fixture did not receive first part') + wait_for(lambda: any(stage == 'abort' and key.endswith('/' + parsed.path.split('/')[-2]) for stage, key, _ in store.events), 'disconnected multipart was not aborted') + wait_for(lambda: not store.uploads, 'disconnected multipart leaked') + slot(client, 's3.localhost', size, 'quota', error=True) + client.close(); client = None + stop(process); process = None + options['http_file_share_daily_quota'] = size * 12 + options['http_file_share_global_quota'] = size * 12 + config.write_text(json.dumps(config_data)) + process = start(binary, config, c2s, log) + assert request(url)[2] == data, 'S3 metadata did not survive restart' + assert request(url, 'PUT', data, headers)[0] == 409, 'single-use state lost on restart' + client = Client(c2s, root / 'cert.pem', 'alice', 'secret') + crash_url, crash_headers = slot(client, 's3.localhost', size, 'crash') + parsed = urlsplit(crash_url) + crash_id = parsed.path.split('/')[-2] + stream = socket.create_connection((parsed.hostname, parsed.port), timeout=10) + head = f'PUT {parsed.path} HTTP/1.1\r\nHost: {parsed.netloc}\r\nContent-Length: {size}\r\n' + head += ''.join(f'{key}: {value}\r\n' for key, value in crash_headers.items()) + '\r\n' + stream.sendall(head.encode() + data[:PART]) + wait_for(lambda: any(stage == 'part' and key.endswith('/' + crash_id) for stage, key, _ in store.events), 'crash fixture did not receive first part') + metadata = root / 'data' / 's3%2elocalhost' / 'rust_uploads.json' + cached = json.loads(metadata.read_text()) + assert cached[crash_id].get('upload_id'), 'multipart identifier was not durable before part upload' + process.kill(); process.wait(timeout=10); process = None + stream.close() + client.close(); client = None + assert store.uploads, 'crash did not leave a remote multipart to recover' + # Model the earlier crash window: initiation reached S3, but its + # UploadId never reached the local journal. Include multiple + # owned uploads plus a foreign prefix match across list pages. + cached = json.loads(metadata.read_text()) + assert cached[crash_id].get('multipart_intent'), 'intent not durable before initiation' + cached[crash_id]['upload_id'] = None + metadata.write_text(json.dumps(cached)) + owned_key = '/api/fixture-bucket/space%20%2B/' + crash_id + foreign_uid = 'foreign-prefix-match' + with store.lock: + for uid in ('orphan-a', 'orphan-b', 'orphan-c'): + store.uploads[uid] = {'key': owned_key, 'parts': {}} + store.uploads[foreign_uid] = {'key': owned_key + '-foreign', 'parts': {}} + process = start(binary, config, c2s, log) + wait_for(lambda: set(store.uploads) == {foreign_uid}, 'startup did not abort only exact owned-key multiparts') + assert not any(stage == 'abort' and key == owned_key + '-foreign' for stage, key, _ in store.events), 'recovery aborted foreign prefix match' + with store.lock: + del store.uploads[foreign_uid] + wait_for(lambda: not json.loads(metadata.read_text())[crash_id].get('upload_id'), 'startup retained abandoned multipart identifier') + assert request(url)[2] == data + stop(process); process = None + # Model a remote completion followed by a crash before the + # local completion marker was persisted. Age exceeds the stale + # reservation timeout but remains within retention. + good_id = urlsplit(url).path.split('/')[-2] + cached = json.loads(metadata.read_text()) + old = int(time.time()) - 700 + cached[good_id].update(complete=False, upload_id='completed-before-crash', created=old) + metadata.write_text(json.dumps(cached)) + archive = metadata.parent / 'uploads.list' + lines = archive.read_text().splitlines(keepends=True) + changed = False + for index, line in enumerate(lines): + if good_id in line: + line, count = re.subn(r'(\["when"\]=)[0-9.]+', lambda match: match[1] + str(old) + '.0', line) + assert count == 1, 'failed to age authoritative upload timestamp' + lines[index] = line + changed = True + assert changed + archive.write_text(''.join(lines)) + archive.with_suffix('.lidx').unlink(missing_ok=True) + options['http_file_share_expires_after'] = 3600 + config.write_text(json.dumps(config_data)) + process = start(binary, config, c2s, log) + wait_for(lambda: json.loads(metadata.read_text()).get(good_id, {}).get('complete'), 'startup did not reconcile committed object before stale pruning') + assert request(url)[2] == data, 'committed object was deleted as an old reservation' + assert not json.loads(metadata.read_text())[good_id].get('upload_id') + # Unknown completion + HEAD404 quarantines only its own slot. + client = Client(c2s, root / 'cert.pem', 'alice', 'secret') + uncertain_url, uncertain_headers = slot(client, 's3.localhost', 1, 'uncertain') + uncertain_id = urlsplit(uncertain_url).path.split('/')[-2] + store.fail = 'complete-malformed' + assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] >= 500 + assert json.loads(metadata.read_text())[uncertain_id]['completion_attempted'] + client.close(); client = None + stop(process); process = None + process = start(binary, config, c2s, log) + assert request(uncertain_url, 'PUT', b'Q', uncertain_headers)[0] == 409 + retained = json.loads(metadata.read_text())[uncertain_id] + assert retained['completion_attempted'] and not retained['complete'] + client = Client(c2s, root / 'cert.pem', 'alice', 'secret') + independent_url, independent_headers = slot(client, 's3.localhost', 1, 'independent') + assert request(independent_url, 'PUT', b'I', independent_headers)[0] == 201 + # A late commit resolves through the next maintenance pass; + # requesting another slot also triggers that same pruning pass. + with store.lock: + store.objects['/api/fixture-bucket/space%20%2B/' + uncertain_id] = b'Q' + slot(client, 's3.localhost', 1, 'trigger-reconciliation') + wait_for(lambda: json.loads(metadata.read_text())[uncertain_id]['complete'], 'late completion was not reconciled') + assert request(uncertain_url)[2] == b'Q' + client.close(); client = None + stop(process); process = None + # Force expiry without slowing this test with a wall-clock wait. + options['http_file_share_expires_after'] = 0 + config.write_text(json.dumps(config_data)) + process = start(binary, config, c2s, log) + assert request(url)[0] in (404, 410) + wait_for(lambda: not store.objects, 'expired S3 objects were not deleted') + assert not store.errors, store.errors + log.flush(); log.seek(0) + output = log.read() + assert SECRET not in output and TOKEN not in output and ACCESS not in output, 'credentials leaked to logs' + print('PASS upload S3 protocol fixture: SigV4, multipart, abort, range, restart, expiry and backend isolation') + except BaseException: + log.flush(); log.seek(0) + output = log.read() + for credential in (SECRET, TOKEN, ACCESS): + output = output.replace(credential, '[REDACTED]') + print('SERVER LOG:\n' + output) + print('FIXTURE ERRORS:', store.errors) + raise + finally: + if client is not None: + client.close() + if process is not None: + stop(process) + store.shutdown() + worker.join(timeout=5) + + +if __name__ == '__main__': + run(str(Path(sys.argv[1] if len(sys.argv) > 1 else 'target/release/rust-xmpp').resolve()))