Reject a certificate whose verify callback throws

An exception from a verify callback unwound through the TLS library,
which is C: OpenSSL leaked the objects of the handshake, and a server
worker thread ended the process. It also left the handshake count that
update_certs_pem() waits on raised, so with a task queue that caught the
exception, every later update waited forever. Take the exception as a
rejection, and lower the count on every way out of the handshake.

httplib.h

@@ -18711,11 +18711,14 @@ inline bool SSLServer::process_and_close_socket(socket_t sock) {
 
   // Perform TLS accept handshake with timeout
   TlsError tls_err;
-  auto accepted = accept_nonblocking(session, sock, read_timeout_sec_,
+  auto accepted = false;
-                                     read_timeout_usec_, &tls_err);
   {
-    std::lock_guard<std::mutex> guard(ctx_mutex_);
+    auto handshake_left = detail::scope_exit([&] {
-    if (--handshakes_in_progress_ == 0) { ctx_cv_.notify_all(); }
+      std::lock_guard<std::mutex> guard(ctx_mutex_);
+      if (--handshakes_in_progress_ == 0) { ctx_cv_.notify_all(); }
+    });
+    accepted = accept_nonblocking(session, sock, read_timeout_sec_,
+                                  read_timeout_usec_, &tls_err);
   }
   if (!accepted) {
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
@@ -19230,6 +19233,19 @@ inline std::mutex &verify_callback_mutex() {
   return mutex;
 }
 
+// The TLS libraries are C, so an exception from a verify callback must not
+// unwind through them. It rejects the certificate instead.
+inline bool run_verify_callback(const VerifyCallback &callback,
+                                const VerifyContext &ctx) {
+#ifdef CPPHTTPLIB_NO_EXCEPTIONS
+  return callback(ctx);
+#else
+  try {
+    return callback(ctx);
+  } catch (...) { return false; }
+#endif
+}
+
 // Check if a string is an IPv4 address
 inline bool is_ipv4_address(const std::string &str) {
   int dots = 0;
@@ -19600,7 +19616,7 @@ inline int openssl_verify_callback(int preverify_ok, X509_STORE_CTX *ctx) {
   verify_ctx.error_string =
       (error != X509_V_OK) ? X509_verify_cert_error_string(error) : nullptr;
 
-  return (*callback)(verify_ctx) ? 1 : 0;
+  return run_verify_callback(*callback, verify_ctx) ? 1 : 0;
 }
 
 // X509_STORE_get0_objects is deprecated since OpenSSL 4.0 because it is not
@@ -20786,7 +20802,7 @@ inline int mbedtls_verify_callback(void *data, mbedtls_x509_crt *crt,
     verify_ctx.error_string = nullptr;
   }
 
-  bool accepted = session->verify_callback(verify_ctx);
+  bool accepted = run_verify_callback(session->verify_callback, verify_ctx);
 
   if (accepted) {
     *flags = 0; // Clear all error flags
@@ -22092,7 +22108,7 @@ inline int wolfssl_verify_callback(int preverify_ok,
     verify_ctx.error_string = nullptr;
   }
 
-  bool accepted = (*callback)(verify_ctx);
+  bool accepted = run_verify_callback(*callback, verify_ctx);
   return accepted ? 1 : 0;
 }
 

test/test.cc

@@ -4933,6 +4933,46 @@ TEST(SSLClientServerTest, VerifyCallbackKeepsClientCertificateRequired) {
   EXPECT_EQ("ok", res->body);
 }
 
+TEST(SSLClientServerTest, ThrowingVerifyCallbackRejectsTheCertificate) {
+  SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE, CLIENT_CA_CERT_FILE);
+  ASSERT_TRUE(svr.is_valid());
+  tls::set_verify_callback(svr.tls_context(),
+                           [](const tls::VerifyContext &) -> bool {
+                             throw std::runtime_error("verify");
+                           });
+  svr.Get("/test", [](const Request &, Response &res) {
+    res.set_content("ok", "text/plain");
+  });
+
+  thread t = thread([&]() { ASSERT_TRUE(svr.listen(HOST, PORT)); });
+  auto se = detail::scope_exit([&] {
+    svr.stop();
+    t.join();
+    ASSERT_FALSE(svr.is_running());
+  });
+  svr.wait_until_ready();
+
+  auto get = [] {
+    SSLClient cli(HOST, PORT, CLIENT_CERT_FILE, CLIENT_PRIVATE_KEY_FILE);
+    cli.enable_server_certificate_verification(false);
+    return cli.Get("/test");
+  };
+  EXPECT_FALSE(get());
+
+  // The handshake is over, so an update does not wait for it.
+  std::string cert_pem, key_pem;
+  read_file(SERVER_CERT_FILE, cert_pem);
+  read_file(SERVER_PRIVATE_KEY_FILE, key_pem);
+  EXPECT_TRUE(svr.update_certs_pem(cert_pem.c_str(), key_pem.c_str()));
+
+  tls::set_verify_callback(svr.tls_context(), [](const tls::VerifyContext &v) {
+    return v.preverify_ok;
+  });
+  auto res = get();
+  ASSERT_TRUE(res) << "Error: " << to_string(res.error());
+  EXPECT_EQ("ok", res->body);
+}
+
 #ifdef CPPHTTPLIB_OPENSSL_SUPPORT
 // A client that resumes its session gets through, but not once the client CA
 // has changed, as a resumed session does not show the certificate again.