DTLS support on macOS

GNUmakefile

@@ -191,6 +191,7 @@ MIN_VER = 10.15
 OBJS := $(OBJS) \
 	src/hid/hid.o \
 	src/unix/apple/image.o \
+	src/unix/apple/macosx/dtls.o \
 	src/unix/apple/macosx/hid.o \
 	src/unix/apple/macosx/gfx/metal-ctx.o
 
@@ -202,14 +203,17 @@ ifeq ($(ARCH), x86_64)
 FLAGS := $(FLAGS) -maes -mpclmul
 endif
 
+# macOS overrides this with a LibreSSL backed implementation above
+OBJS := $(OBJS) \
+	src/unix/apple/dtls.o
+
 endif
 
 OBJS := $(OBJS) \
 	src/unix/system.o \
 	src/unix/apple/audio.o \
 	src/unix/apple/base64.o \
 	src/unix/apple/crypto.o \
-	src/unix/apple/dtls.o \
 	src/unix/apple/request.o \
 	src/unix/apple/webview.o \
 	src/unix/apple/ws.o \

src/matoya.h

@@ -3553,7 +3553,7 @@ MTY_GetJNIEnv(void);
 //- #mbrief DTLS protocol wrapper.
 //- #mdetails This module performs no IO and acts as a DTLS engine that requires you
 //-   to feed it input and in turn will output data suitable for sending over a network.
-//- #msupport Windows Linux
+//- #msupport Windows macOS Linux
 
 #define MTY_FINGERPRINT_MAX 112 ///< Maximum size of the string set by MTY_CertGetFingerprint.
 
@@ -3573,13 +3573,13 @@ typedef bool (*MTY_DTLSWriteFunc)(const void *buf, size_t size, void *opaque);
 ///   negotiation.
 /// @returns On failure, NULL is returned. Call MTY_GetLog for details.\n\n
 ///   The returned MTY_Cert must be destroyed with MTY_CertDestroy.
-//- #support Windows Linux
+//- #support Windows macOS Linux
 MTY_EXPORT MTY_Cert *
 MTY_CertCreate(void);
 
 /// @brief Destroy an MTY_Cert.
 /// @param cert Passed by reference and set to NULL after being destroyed.
-//- #support Windows Linux
+//- #support Windows macOS Linux
 MTY_EXPORT void
 MTY_CertDestroy(MTY_Cert **cert);
 
@@ -3589,7 +3589,7 @@ MTY_CertDestroy(MTY_Cert **cert);
 /// @param fingerprint Output buffer to receive the fingerprint. This buffer can be
 ///   of size MTY_FINGERPRINT_MAX.
 /// @param size Size in bytes of `fingerprint`.
-//- #support Windows Linux
+//- #support Windows macOS Linux
 MTY_EXPORT void
 MTY_CertGetFingerprint(MTY_Cert *ctx, char *fingerprint, size_t size);
 

src/unix/apple/macosx/dl/libssl.h

@@ -0,0 +1,242 @@
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT License was not distributed with this file,
+// You can obtain one at https://spdx.org/licenses/MIT.html.
+
+#pragma once
+
+#include <stdio.h>
+#include <dlfcn.h>
+
+#include "sym.h"
+
+
+// Interface
+
+#define SSL_ERROR_WANT_READ                           2
+#define SSL_ERROR_WANT_WRITE                          3
+#define SSL_ERROR_ZERO_RETURN                         6
+
+#define SSL_CTRL_SET_MTU                              17
+#define SSL_CTRL_OPTIONS                              32
+
+#define SSL_VERIFY_PEER                               0x01
+#define SSL_VERIFY_FAIL_IF_NO_PEER_CERT               0x02
+#define SSL_OP_NO_QUERY_MTU                           0x00001000U
+#define SSL_OP_NO_TICKET                              0x00004000U
+#define SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION 0x00010000U
+
+#define DTLS1_2_VERSION                               0xFEFD
+
+#define MBSTRING_FLAG                                 0x1000
+#define MBSTRING_ASC                                  (MBSTRING_FLAG | 1)
+#define RSA_F4                                        0x10001L
+#define EVP_MAX_MD_SIZE                               64
+
+typedef struct rsa_st RSA;
+typedef struct bio_st BIO;
+typedef struct bignum_st BIGNUM;
+typedef struct x509_st X509;
+typedef struct X509_name_st X509_NAME;
+typedef struct x509_store_ctx_st X509_STORE_CTX;
+typedef struct evp_pkey_st EVP_PKEY;
+typedef struct bn_gencb_st BN_GENCB;
+typedef struct asn1_string_st ASN1_INTEGER;
+typedef struct asn1_string_st ASN1_TIME;
+typedef struct evp_md_st EVP_MD;
+typedef struct bio_method_st BIO_METHOD;
+
+typedef struct ssl_ctx_st SSL_CTX;
+typedef struct ssl_method_st SSL_METHOD;
+typedef struct ssl_st SSL;
+
+typedef int (*SSL_verify_cb)(int preverify_ok, X509_STORE_CTX *x509_ctx);
+
+static SSL *(*SSL_new)(SSL_CTX *ctx);
+static void (*SSL_free)(SSL *ssl);
+static int (*SSL_read)(SSL *ssl, void *buf, int num);
+static int (*SSL_write)(SSL *ssl, const void *buf, int num);
+static void (*SSL_set_verify)(SSL *s, int mode, SSL_verify_cb callback);
+static int (*SSL_get_error)(const SSL *s, int ret_code);
+static long (*SSL_ctrl)(SSL *ssl, int cmd, long larg, void *parg);
+static void (*SSL_set_bio)(SSL *s, BIO *rbio, BIO *wbio);
+static void (*SSL_set_connect_state)(SSL *s);
+static int (*SSL_do_handshake)(SSL *s);
+static int (*SSL_use_certificate)(SSL *ssl, X509 *x);
+static int (*SSL_use_RSAPrivateKey)(SSL *ssl, RSA *rsa);
+static X509 *(*SSL_get_peer_certificate)(const SSL *s);
+
+static const SSL_METHOD *(*DTLS_method)(void);
+static SSL_CTX *(*SSL_CTX_new)(const SSL_METHOD *meth);
+static void (*SSL_CTX_free)(SSL_CTX *ctx);
+static int (*SSL_CTX_set_max_proto_version)(SSL_CTX *ctx, uint16_t version);
+
+static BIO *(*BIO_new)(const BIO_METHOD *type);
+static int (*BIO_free)(BIO *a);
+static const BIO_METHOD *(*BIO_s_mem)(void);
+static int (*BIO_write)(BIO *b, const void *data, int len);
+static size_t (*BIO_ctrl_pending)(BIO *b);
+static int (*BIO_read)(BIO *b, void *data, int len);
+
+static void (*X509_free)(X509 *a);
+static X509 *(*X509_new)(void);
+static int (*X509_set_pubkey)(X509 *x, EVP_PKEY *pkey);
+static int (*X509_sign)(X509 *x, EVP_PKEY *pkey, const EVP_MD *md);
+static int (*X509_digest)(const X509 *data, const EVP_MD *type, unsigned char *md, unsigned int *len);
+static ASN1_TIME *(*X509_getm_notBefore)(const X509 *x);
+static ASN1_TIME *(*X509_getm_notAfter)(const X509 *x);
+static int (*X509_set_version)(X509 *x, long version);
+static int (*X509_set_issuer_name)(X509 *x, X509_NAME *name);
+static X509_NAME *(*X509_get_subject_name)(const X509 *a);
+static ASN1_INTEGER *(*X509_get_serialNumber)(X509 *x);
+static ASN1_TIME *(*X509_gmtime_adj)(ASN1_TIME *s, long adj);
+static int (*X509_NAME_add_entry_by_txt)(X509_NAME *name, const char *field, int type,
+	const unsigned char *bytes, int len, int loc, int set);
+
+static BIGNUM *(*BN_new)(void);
+static void (*BN_free)(BIGNUM *a);
+static int (*BN_set_word)(BIGNUM *a, unsigned long long w);
+
+static EVP_PKEY *(*EVP_PKEY_new)(void);
+static void (*EVP_PKEY_free)(EVP_PKEY *pkey);
+static const EVP_MD *(*EVP_sha256)(void);
+static int (*EVP_PKEY_set1_RSA)(EVP_PKEY *pkey, RSA *key);
+
+static RSA *(*RSA_new)(void);
+static void (*RSA_free)(RSA *r);
+static int (*RSA_generate_key_ex)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
+
+static int (*ASN1_INTEGER_set)(ASN1_INTEGER *a, long v);
+
+
+// Runtime open
+
+// macOS ships LibreSSL as versioned dylibs inside the dyld shared cache. The unversioned
+// libssl.dylib deliberately aborts the process ("is loading libcrypto in an unsafe way"),
+// so only the versioned soname may be opened. Several versions are present side by side
+// for binary compatibility, but the LibreSSL 2.x ones predate DTLS_method, so probe from
+// the newest soname down and take the first that can actually speak DTLS.
+#define LIBSSL_SO_MIN 48
+#define LIBSSL_SO_MAX 80
+
+static MTY_Atomic32 LIBSSL_LOCK;
+static MTY_SO *LIBSSL_SO;
+static bool LIBSSL_INIT;
+
+static void libssl_global_destroy_lockfree(void)
+{
+	MTY_SOUnload(&LIBSSL_SO);
+	LIBSSL_INIT = false;
+}
+
+static void __attribute__((destructor)) libssl_global_destroy(void)
+{
+	MTY_GlobalLock(&LIBSSL_LOCK);
+
+	libssl_global_destroy_lockfree();
+
+	MTY_GlobalUnlock(&LIBSSL_LOCK);
+}
+
+static MTY_SO *libssl_dlopen(void)
+{
+	for (int32_t v = LIBSSL_SO_MAX; v >= LIBSSL_SO_MIN; v--) {
+		char name[32];
+		snprintf(name, sizeof(name), "libssl.%d.dylib", v);
+
+		// Probed via dlopen directly rather than MTY_SOLoad to keep the misses out of the log
+		void *so = dlopen(name, RTLD_NOW | RTLD_LOCAL);
+		if (!so)
+			continue;
+
+		// libcrypto symbols resolve through the libssl handle via its dependency
+		if (dlsym(so, "DTLS_method") && dlsym(so, "X509_new"))
+			return so;
+
+		dlclose(so);
+	}
+
+	MTY_Log("'dlopen' failed to find a libssl dylib with DTLS support");
+
+	return NULL;
+}
+
+static bool libssl_global_init(void)
+{
+	MTY_GlobalLock(&LIBSSL_LOCK);
+
+	if (!LIBSSL_INIT) {
+		bool r = true;
+		LIBSSL_SO = libssl_dlopen();
+
+		if (!LIBSSL_SO) {
+			r = false;
+			goto except;
+		}
+
+		LOAD_SYM(LIBSSL_SO, SSL_new);
+		LOAD_SYM(LIBSSL_SO, SSL_free);
+		LOAD_SYM(LIBSSL_SO, SSL_read);
+		LOAD_SYM(LIBSSL_SO, SSL_write);
+		LOAD_SYM(LIBSSL_SO, SSL_set_verify);
+		LOAD_SYM(LIBSSL_SO, SSL_get_error);
+		LOAD_SYM(LIBSSL_SO, SSL_ctrl);
+		LOAD_SYM(LIBSSL_SO, SSL_set_bio);
+		LOAD_SYM(LIBSSL_SO, SSL_set_connect_state);
+		LOAD_SYM(LIBSSL_SO, SSL_do_handshake);
+		LOAD_SYM(LIBSSL_SO, SSL_use_certificate);
+		LOAD_SYM(LIBSSL_SO, SSL_use_RSAPrivateKey);
+		LOAD_SYM(LIBSSL_SO, SSL_get_peer_certificate);
+
+		LOAD_SYM(LIBSSL_SO, DTLS_method);
+		LOAD_SYM(LIBSSL_SO, SSL_CTX_new);
+		LOAD_SYM(LIBSSL_SO, SSL_CTX_free);
+		LOAD_SYM(LIBSSL_SO, SSL_CTX_set_max_proto_version);
+
+		LOAD_SYM(LIBSSL_SO, BIO_new);
+		LOAD_SYM(LIBSSL_SO, BIO_s_mem);
+		LOAD_SYM(LIBSSL_SO, BIO_write);
+		LOAD_SYM(LIBSSL_SO, BIO_ctrl_pending);
+		LOAD_SYM(LIBSSL_SO, BIO_read);
+		LOAD_SYM(LIBSSL_SO, BIO_free);
+
+		LOAD_SYM(LIBSSL_SO, X509_new);
+		LOAD_SYM(LIBSSL_SO, X509_free);
+		LOAD_SYM(LIBSSL_SO, X509_set_pubkey);
+		LOAD_SYM(LIBSSL_SO, X509_sign);
+		LOAD_SYM(LIBSSL_SO, X509_digest);
+		LOAD_SYM(LIBSSL_SO, X509_getm_notBefore);
+		LOAD_SYM(LIBSSL_SO, X509_getm_notAfter);
+		LOAD_SYM(LIBSSL_SO, X509_set_version);
+		LOAD_SYM(LIBSSL_SO, X509_set_issuer_name);
+		LOAD_SYM(LIBSSL_SO, X509_get_subject_name);
+		LOAD_SYM(LIBSSL_SO, X509_get_serialNumber);
+		LOAD_SYM(LIBSSL_SO, X509_gmtime_adj);
+		LOAD_SYM(LIBSSL_SO, X509_NAME_add_entry_by_txt);
+
+		LOAD_SYM(LIBSSL_SO, RSA_new);
+		LOAD_SYM(LIBSSL_SO, RSA_free);
+		LOAD_SYM(LIBSSL_SO, RSA_generate_key_ex);
+
+		LOAD_SYM(LIBSSL_SO, BN_new);
+		LOAD_SYM(LIBSSL_SO, BN_free);
+		LOAD_SYM(LIBSSL_SO, BN_set_word);
+
+		LOAD_SYM(LIBSSL_SO, EVP_PKEY_new);
+		LOAD_SYM(LIBSSL_SO, EVP_PKEY_free);
+		LOAD_SYM(LIBSSL_SO, EVP_sha256);
+		LOAD_SYM(LIBSSL_SO, EVP_PKEY_set1_RSA);
+
+		LOAD_SYM(LIBSSL_SO, ASN1_INTEGER_set);
+
+		except:
+
+		if (!r)
+			libssl_global_destroy_lockfree();
+
+		LIBSSL_INIT = r;
+	}
+
+	MTY_GlobalUnlock(&LIBSSL_LOCK);
+
+	return LIBSSL_INIT;
+}

src/unix/apple/macosx/dl/sym.h

@@ -0,0 +1,14 @@
+// This Source Code Form is subject to the terms of the MIT License.
+// If a copy of the MIT License was not distributed with this file,
+// You can obtain one at https://spdx.org/licenses/MIT.html.
+
+#pragma once
+
+#include "matoya.h"
+
+#define LOAD_SYM(so, name) \
+	name = MTY_SOGetSymbol(so, #name); \
+	if (!name) {r = false; goto except;}
+
+#define LOAD_SYM_OPT(so, name) \
+	name = MTY_SOGetSymbol(so, #name);