Page MenuHomeFreeBSD
Files F109205420

D48570.diff
No OneTemporary
Actions

D48570.diff
View Options

diff --git a/sys/rpc/rpcsec_tls.h b/sys/rpc/rpcsec_tls.h
--- a/sys/rpc/rpcsec_tls.h
+++ b/sys/rpc/rpcsec_tls.h
@@ -32,9 +32,6 @@
#define RPCTLS_SYSC_CLSOCKET 2
#define RPCTLS_SYSC_SRVSOCKET 5
-/* Max nprocs for SRV startup */
-#define RPCTLS_SRV_MAXNPROCS 16
-
/* System call used by the rpctlscd, rpctlssd daemons. */
int rpctls_syscall(int, const char *);
diff --git a/usr.sbin/rpc.tlsservd/Makefile b/usr.sbin/rpc.tlsservd/Makefile
--- a/usr.sbin/rpc.tlsservd/Makefile
+++ b/usr.sbin/rpc.tlsservd/Makefile
@@ -6,7 +6,7 @@
CFLAGS+= -I.
-LIBADD= ssl crypto util
+LIBADD= ssl crypto util pthread
CLEANFILES= rpctlssd_svc.c rpctlssd_xdr.c rpctlssd.h
diff --git a/usr.sbin/rpc.tlsservd/rpc.tlsservd.8 b/usr.sbin/rpc.tlsservd/rpc.tlsservd.8
--- a/usr.sbin/rpc.tlsservd/rpc.tlsservd.8
+++ b/usr.sbin/rpc.tlsservd/rpc.tlsservd.8
@@ -24,7 +24,7 @@
.\" SUCH DAMAGE.
.\"
.\" Modified from gssd.8 for rpc.tlsservd.8 by Rick Macklem.
-.Dd November 10, 2022
+.Dd January 25, 2025
.Dt RPC.TLSSERVD 8
.Os
.Sh NAME
@@ -39,7 +39,7 @@
.Op Fl h
.Op Fl l Ar CAfile
.Op Fl m
-.Op Fl N Ar num_servers
+.Op Fl N Ar max_threads
.Op Fl n Ar domain
.Op Fl p Ar CApath
.Op Fl r Ar CRLfile
@@ -236,16 +236,17 @@
that verifies.
See
.Xr exports 5 .
-.It Fl N Ar num_servers , Fl Fl numdaemons= Ns Ar num_servers
+.It Fl N Ar max_threads , Fl Fl maxthreads= Ns Ar max_threads
For a server with a large number of NFS-over-TLS client mounts,
this daemon might get overloaded after a reboot, when many
clients attempt to do a TLS handshake at the same time.
-This option may be used to specify that
-.Dq num_servers
-daemons are to be run instead of a single daemon.
-When this is done, the TLS handshakes are spread across the
-.Dq num_servers
-daemons in a round robin fashion to spread out the load.
+To speed up recovery after reboot, the daemon always processes a TLS handshake
+in a separate spawned thread.
+By default the maximum number of concurrent threads (and thus
+parallel handshakes) is limited to
+.Va 1/2
+of available CPUs on a system.
+This option may be used to override this default.
.It Fl n Ar domain , Fl Fl domain= Ns Ar domain
This option specifies what the
.Dq domain
diff --git a/usr.sbin/rpc.tlsservd/rpc.tlsservd.c b/usr.sbin/rpc.tlsservd/rpc.tlsservd.c
--- a/usr.sbin/rpc.tlsservd/rpc.tlsservd.c
+++ b/usr.sbin/rpc.tlsservd/rpc.tlsservd.c
@@ -4,6 +4,7 @@
* Copyright (c) 2008 Isilon Inc http://www.isilon.com/
* Authors: Doug Rabson <dfr@rabson.org>
* Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
+ * Copyright (c) 2025 Gleb Smirnoff <glebius@FreeBSD.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -32,27 +33,20 @@
* the server side of kernel RPC-over-TLS by Rick Macklem.
*/
-#include <sys/param.h>
#include <sys/types.h>
#include <sys/linker.h>
#include <sys/module.h>
#include <sys/queue.h>
-#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
-#include <sys/time.h>
-#include <sys/wait.h>
+#include <assert.h>
#include <err.h>
#include <getopt.h>
#include <libutil.h>
#include <netdb.h>
+#include <pthread.h>
#include <pwd.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
#include <stdbool.h>
-#include <string.h>
#include <unistd.h>
#include <rpc/rpc.h>
@@ -86,7 +80,14 @@
const char *rpctls_verify_capath = NULL;
char *rpctls_crlfile = NULL;
bool rpctls_gothup = false;
+
+static SVCXPRT *xprt;
+static pthread_key_t xidkey;
struct ssl_list rpctls_ssllist;
+static pthread_rwlock_t rpctls_rwlock;
+static u_int rpctls_nthreads = 0;
+static pthread_mutex_t rpctls_mtx;
+static pthread_cond_t rpctls_cv;
static struct pidfh *rpctls_pfh = NULL;
static bool rpctls_do_mutual = false;
@@ -98,9 +99,7 @@
static const char *rpctls_cnuseroid = "1.3.6.1.4.1.2238.1.1.1";
static const char *rpctls_ciphers = NULL;
static int rpctls_mintls = TLS1_3_VERSION;
-static int rpctls_procs = 1;
-static pid_t rpctls_workers[RPCTLS_SRV_MAXNPROCS - 1];
-static bool rpctls_im_a_worker = false;
+static u_int rpctls_maxthreads;
static void rpctls_cleanup_term(int sig);
static SSL_CTX *rpctls_setup_ssl(const char *certdir);
@@ -114,6 +113,8 @@
extern void rpctlssd_2(struct svc_req *rqstp, SVCXPRT *transp);
+static void *dummy_thread(void *v __unused) { return (NULL); }
+
static struct option longopts[] = {
{ "allowtls1_2", no_argument, NULL, '2' },
{ "ciphers", required_argument, NULL, 'C' },
@@ -122,7 +123,7 @@
{ "checkhost", no_argument, NULL, 'h' },
{ "verifylocs", required_argument, NULL, 'l' },
{ "mutualverf", no_argument, NULL, 'm' },
- { "numdaemons", required_argument, NULL, 'N' },
+ { "maxthreads", required_argument, NULL, 'N' },
{ "domain", required_argument, NULL, 'n' },
{ "verifydir", required_argument, NULL, 'p' },
{ "crl", required_argument, NULL, 'r' },
@@ -136,13 +137,13 @@
int
main(int argc, char **argv)
{
- int ch, i;
- SVCXPRT *xprt;
+ int ch;
char hostname[MAXHOSTNAMELEN + 2];
pid_t otherpid;
+ pthread_t tid;
bool tls_enable;
size_t tls_enable_len;
- sigset_t signew;
+ u_int ncpu;
/* Check that another rpctlssd isn't already running. */
rpctls_pfh = pidfile_open(_PATH_RPCTLSSDPID, 0600, &otherpid);
@@ -166,6 +167,8 @@
}
rpctls_verbose = false;
+ rpctls_maxthreads = (ncpu = (u_int)sysconf(_SC_NPROCESSORS_ONLN)) / 2;
+
while ((ch = getopt_long(argc, argv, "2C:D:dhl:N:n:mp:r:uvWw", longopts,
NULL)) != -1) {
switch (ch) {
@@ -191,11 +194,10 @@
rpctls_do_mutual = true;
break;
case 'N':
- rpctls_procs = atoi(optarg);
- if (rpctls_procs < 1 ||
- rpctls_procs > RPCTLS_SRV_MAXNPROCS)
- errx(1, "numdaemons/-N must be between 1 and "
- "%d", RPCTLS_SRV_MAXNPROCS);
+ rpctls_maxthreads = atoi(optarg);
+ if (rpctls_maxthreads < 1 || rpctls_maxthreads > ncpu)
+ errx(1, "maximum threads must be between 1 and "
+ "number of CPUs (%d)", ncpu);
break;
case 'n':
hostname[0] = '@';
@@ -233,7 +235,6 @@
"[-D/--certdir certdir] [-d/--debuglevel] "
"[-h/--checkhost] "
"[-l/--verifylocs CAfile] [-m/--mutualverf] "
- "[-N/--numdaemons num] "
"[-n/--domain domain_name] "
"[-p/--verifydir CApath] [-r/--crl CRLfile] "
"[-u/--certuser] [-v/--verbose] [-W/--multiwild] "
@@ -262,53 +263,20 @@
if (kldload("krpc") < 0 || modfind("krpc") < 0)
errx(1, "Kernel RPC is not available");
}
-
- for (i = 0; i < rpctls_procs - 1; i++)
- rpctls_workers[i] = -1;
-
- if (rpctls_debug_level == 0) {
- /*
- * Temporarily block SIGTERM and SIGCHLD, so workers[] can't
- * end up bogus.
- */
- sigemptyset(&signew);
- sigaddset(&signew, SIGTERM);
- sigaddset(&signew, SIGCHLD);
- sigprocmask(SIG_BLOCK, &signew, NULL);
-
- if (daemon(0, 0) != 0)
- err(1, "Can't daemonize");
- signal(SIGINT, SIG_IGN);
- signal(SIGQUIT, SIG_IGN);
- }
signal(SIGPIPE, SIG_IGN);
signal(SIGHUP, rpctls_huphandler);
signal(SIGTERM, rpctls_cleanup_term);
- signal(SIGCHLD, rpctls_cleanup_term);
+ if (rpctls_debug_level == 0 && daemon(0, 0) != 0)
+ err(1, "Can't daemonize");
pidfile_write(rpctls_pfh);
- if (rpctls_debug_level == 0) {
- /* Fork off the worker daemons. */
- for (i = 0; i < rpctls_procs - 1; i++) {
- rpctls_workers[i] = fork();
- if (rpctls_workers[i] == 0) {
- rpctls_im_a_worker = true;
- setproctitle("server");
- break;
- } else if (rpctls_workers[i] < 0) {
- syslog(LOG_ERR, "fork: %m");
- }
- }
-
- if (!rpctls_im_a_worker && rpctls_procs > 1)
- setproctitle("master");
- }
- sigemptyset(&signew);
- sigaddset(&signew, SIGTERM);
- sigaddset(&signew, SIGCHLD);
- sigprocmask(SIG_UNBLOCK, &signew, NULL);
-
+ /*
+ * XXX: Push libc internal state into threaded mode before creating
+ * the threaded svc_nl xprt.
+ */
+ (void)pthread_create(&tid, NULL, dummy_thread, NULL);
+ (void)pthread_join(tid, NULL);
if ((xprt = svc_nl_create("tlsserv")) == NULL) {
if (rpctls_debug_level == 0) {
syslog(LOG_ERR,
@@ -317,6 +285,8 @@
}
err(1, "Can't create transport for local rpctlssd socket");
}
+ if (!SVC_CONTROL(xprt, SVCNL_GET_XIDKEY, &xidkey))
+ err(1, "Failed to obtain pthread key for xid from svc_nl");
if (!svc_reg(xprt, RPCTLSSD, RPCTLSSDVERS, rpctlssd_2, NULL)) {
if (rpctls_debug_level == 0) {
syslog(LOG_ERR,
@@ -335,6 +305,9 @@
err(1, "Can't create SSL context");
}
rpctls_gothup = false;
+ pthread_rwlock_init(&rpctls_rwlock, NULL);
+ pthread_mutex_init(&rpctls_mtx, NULL);
+ pthread_cond_init(&rpctls_cv, NULL);
LIST_INIT(&rpctls_ssllist);
rpctls_svc_run();
@@ -352,24 +325,90 @@
return (TRUE);
}
+/*
+ * To parallelize SSL handshakes we will launch a thread per handshake. Thread
+ * creation/destruction shall be order(s) of magnitude cheaper than a crypto
+ * handshake, so we are not keeping a pool of workers here.
+ *
+ * Marrying rpc(3) and pthread(3):
+ *
+ * Normally the rpcgen(1) generated rpctlssd_V() calls rpctlssd_connect_V_svc(),
+ * and the latter processes the RPC all the way to the end and returns a TRUE
+ * value and populates the result. The generated code immediately calls
+ * svc_sendreply() transmitting the result back.
+ *
+ * We will make a private copy of arguments and return FALSE. Then it is our
+ * obligation to call svc_sendreply() once we do the work in the thread.
+ */
+
+static void * rpctlssd_connect_thread(void *);
+struct rpctlssd_connect_thread_ctx {
+ struct rpctlssd_connect_arg arg;
+ uint32_t xid;
+};
+
bool_t
rpctlssd_connect_2_svc(struct rpctlssd_connect_arg *argp,
- struct rpctlssd_connect_res *result, __unused struct svc_req *rqstp)
+ struct rpctlssd_connect_res *result __unused, struct svc_req *rqstp)
{
+ struct rpctlssd_connect_thread_ctx *ctx;
+ pthread_t tid;
+
+ assert(rqstp->rq_xprt == xprt);
+
+ ctx = malloc(sizeof(*ctx));
+ memcpy(&ctx->arg, argp, sizeof(ctx->arg));
+ ctx->xid = *(uint32_t *)pthread_getspecific(xidkey);
+
+ pthread_mutex_lock(&rpctls_mtx);
+ while (rpctls_nthreads >= rpctls_maxthreads)
+ pthread_cond_wait(&rpctls_cv, &rpctls_mtx);
+ rpctls_nthreads++;
+ pthread_mutex_unlock(&rpctls_mtx);
+
+ rpctls_verbose_out("rpctlsd_connect_svc: xid %u thread %u\n",
+ ctx->xid, rpctls_nthreads);
+
+ if (pthread_create(&tid, NULL, rpctlssd_connect_thread, ctx) != 0)
+ warn("failed to start handshake thread");
+
+ /* Intentionally, so that RPC generated code doesn't try to reply. */
+ return (FALSE);
+}
+
+static void *
+rpctlssd_connect_thread(void *v)
+{
+ struct rpctlssd_connect_thread_ctx *ctx = v;
+ struct rpctlssd_connect_res result;
+ uint64_t socookie;
int ngrps, s;
SSL *ssl;
uint32_t flags;
struct ssl_entry *newslp;
- uint32_t uid;
+ uint32_t xid, uid;
uint32_t *gidp;
X509 *cert;
- rpctls_verbose_out("rpctlsd_connect_svc: started\n");
- memset(result, 0, sizeof(*result));
+ socookie = ctx->arg.socookie;
+ xid = ctx->xid;
+ free(ctx);
+ ctx = NULL;
+ pthread_detach(pthread_self());
+
+ if (pthread_setspecific(xidkey, &xid) != 0) {
+ rpctls_verbose_out("rpctlssd_connect_svc: pthread_setspecific "
+ "failed\n");
+ goto out;
+ }
+
/* Get the socket fd from the kernel. */
- s = rpctls_syscall(RPCTLS_SYSC_SRVSOCKET, (char *)argp->socookie);
- if (s < 0)
- return (FALSE);
+ s = rpctls_syscall(RPCTLS_SYSC_SRVSOCKET, (char *)socookie);
+ if (s < 0) {
+ rpctls_verbose_out("rpctlssd_connect_svc: rpctls_syscall "
+ "accept failed\n");
+ goto out;
+ }
/* Do the server side of a TLS handshake. */
gidp = calloc(NGROUPS, sizeof(*gidp));
@@ -383,20 +422,24 @@
* to close off the socket upon handshake failure.
*/
close(s);
- return (FALSE);
+ goto out;
} else {
rpctls_verbose_out("rpctlssd_connect_svc: "
"succeeded flags=0x%x\n", flags);
- result->flags = flags;
- if ((flags & RPCTLS_FLAGS_CERTUSER) != 0) {
- result->uid = uid;
- result->gid.gid_len = ngrps;
- result->gid.gid_val = gidp;
- } else {
- result->uid = 0;
- result->gid.gid_len = 0;
- result->gid.gid_val = gidp;
- }
+ if ((flags & RPCTLS_FLAGS_CERTUSER) != 0)
+ result = (struct rpctlssd_connect_res){
+ .flags = flags,
+ .uid = uid,
+ .gid.gid_len = ngrps,
+ .gid.gid_val = gidp,
+ };
+ else
+ result = (struct rpctlssd_connect_res){
+ .flags = flags,
+ .uid = 0,
+ .gid.gid_len = 0,
+ .gid.gid_val = gidp,
+ };
}
/* Maintain list of all current SSL *'s */
@@ -404,10 +447,27 @@
newslp->ssl = ssl;
newslp->s = s;
newslp->shutoff = false;
- newslp->cookie = argp->socookie;
+ newslp->cookie = socookie;
newslp->cert = cert;
+ pthread_rwlock_wrlock(&rpctls_rwlock);
LIST_INSERT_HEAD(&rpctls_ssllist, newslp, next);
- return (TRUE);
+ pthread_rwlock_unlock(&rpctls_rwlock);
+
+ if (!svc_sendreply(xprt, (xdrproc_t)xdr_rpctlssd_connect_res, &result))
+ svcerr_systemerr(xprt);
+
+ free(result.gid.gid_val);
+ rpctls_verbose_out("rpctlsd_connect_svc: xid %u: thread finished\n",
+ xid);
+
+out:
+ pthread_mutex_lock(&rpctls_mtx);
+ if (rpctls_nthreads-- >= rpctls_maxthreads) {
+ pthread_mutex_unlock(&rpctls_mtx);
+ pthread_cond_signal(&rpctls_cv);
+ } else
+ pthread_mutex_unlock(&rpctls_mtx);
+ return (NULL);
}
bool_t
@@ -418,9 +478,11 @@
int ret;
char junk;
+ pthread_rwlock_rdlock(&rpctls_rwlock);
LIST_FOREACH(slp, &rpctls_ssllist, next)
if (slp->cookie == argp->socookie)
break;
+ pthread_rwlock_unlock(&rpctls_rwlock);
if (slp != NULL) {
rpctls_verbose_out("rpctlssd_handlerecord fd=%d\n",
@@ -455,14 +517,17 @@
struct ssl_entry *slp;
int ret;
+ pthread_rwlock_wrlock(&rpctls_rwlock);
LIST_FOREACH(slp, &rpctls_ssllist, next)
- if (slp->cookie == argp->socookie)
+ if (slp->cookie == argp->socookie) {
+ LIST_REMOVE(slp, next);
break;
+ }
+ pthread_rwlock_unlock(&rpctls_rwlock);
if (slp != NULL) {
rpctls_verbose_out("rpctlssd_disconnect fd=%d closed\n",
slp->s);
- LIST_REMOVE(slp, next);
if (!slp->shutoff) {
ret = SSL_get_shutdown(slp->ssl);
/*
@@ -490,15 +555,9 @@
}
int
-rpctlssd_2_freeresult(__unused SVCXPRT *transp, xdrproc_t xdr_result,
- caddr_t result)
+rpctlssd_2_freeresult(__unused SVCXPRT *transp, xdrproc_t xdr_result __unused,
+ caddr_t result __unused)
{
- rpctlssd_connect_res *res;
-
- if (xdr_result == (xdrproc_t)xdr_rpctlssd_connect_res) {
- res = (rpctlssd_connect_res *)(void *)result;
- free(res->gid.gid_val);
- }
return (TRUE);
}
@@ -506,36 +565,14 @@
* cleanup_term() called via SIGTERM (or SIGCHLD if a child dies).
*/
static void
-rpctls_cleanup_term(int sig)
+rpctls_cleanup_term(int sig __unused)
{
struct ssl_entry *slp;
- int i, cnt;
- if (rpctls_im_a_worker && sig == SIGCHLD)
- return;
LIST_FOREACH(slp, &rpctls_ssllist, next)
shutdown(slp->s, SHUT_RD);
SSL_CTX_free(rpctls_ctx);
EVP_cleanup();
-
- if (rpctls_im_a_worker)
- exit(0);
-
- /* I'm the server, so terminate the workers. */
- cnt = 0;
- for (i = 0; i < rpctls_procs - 1; i++) {
- if (rpctls_workers[i] != -1) {
- cnt++;
- kill(rpctls_workers[i], SIGTERM);
- }
- }
-
- /*
- * Wait for them to die.
- */
- for (i = 0; i < cnt; i++)
- wait3(NULL, 0, NULL);
-
pidfile_remove(rpctls_pfh);
exit(0);

File Metadata

Mime Type
text/plain
Expires
Mon, Feb 3, 1:16 AM (1 h, 19 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
16421485
Default Alt Text
D48570.diff (14 KB)

Event Timeline