diff --git a/daemon/daemon.c b/daemon/daemon.c index ea4e83e70..51dd51de3 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -79,6 +79,7 @@ #include "util/tcp_conn_limit.h" #include "util/edns.h" #include "services/listen_dnsport.h" +#include "services/outside_network.h" #include "services/cache/rrset.h" #include "services/cache/infra.h" #include "services/localzone.h" @@ -813,6 +814,10 @@ daemon_create_workers(struct daemon* daemon) fatal_exit("out of memory during daemon init"); numport = daemon_get_shufport(daemon, shufport); verbose(VERB_ALGO, "total of %d outgoing ports available", numport); + if(!(daemon->shared_ports = shared_ports_create(daemon->cfg->out_ifs, + daemon->cfg->num_out_ifs, daemon->cfg->do_ip4, + daemon->cfg->do_ip6, shufport, numport))) + fatal_exit("could not setup shared ports: out of memory"); #ifdef HAVE_NGTCP2 if (cfg_has_quic(daemon->cfg)) { @@ -843,10 +848,7 @@ daemon_create_workers(struct daemon* daemon) #endif } for(i=0; inum; i++) { - if(!(daemon->workers[i] = worker_create(daemon, i, - shufport+numport*i/daemon->num, - numport*(i+1)/daemon->num - numport*i/daemon->num))) - /* the above is not ports/numthr, due to rounding */ + if(!(daemon->workers[i] = worker_create(daemon, i))) fatal_exit("could not create worker"); } /* create per-worker alloc caches if not reusing existing ones. */ @@ -1204,6 +1206,8 @@ daemon_cleanup(struct daemon* daemon) if(!daemon->reuse_cache || daemon->need_to_exit) daemon_clear_allocs(daemon); daemon->num = 0; + shared_ports_delete(daemon->shared_ports); + daemon->shared_ports = NULL; #ifdef USE_DNSTAP dt_delete(daemon->dtenv); daemon->dtenv = NULL; diff --git a/daemon/daemon.h b/daemon/daemon.h index 20386d7fc..e6f099629 100644 --- a/daemon/daemon.h +++ b/daemon/daemon.h @@ -62,6 +62,7 @@ struct doq_table; struct cookie_secrets; struct fast_reload_thread; struct fast_reload_printq; +struct shared_ports; #include "dnstap/dnstap_config.h" #ifdef USE_DNSTAP @@ -97,6 +98,8 @@ struct daemon { int rc_port; /** listening ports for remote control */ struct listen_port* rc_ports; + /** the shared ports structure, with random ports numbers. */ + struct shared_ports* shared_ports; /** remote control connections management (for first worker) */ struct daemon_remote* rc; /** ssl context for listening to dnstcp over ssl */ diff --git a/daemon/worker.c b/daemon/worker.c index a5dd9bc02..aa72f4e25 100644 --- a/daemon/worker.c +++ b/daemon/worker.c @@ -2225,23 +2225,16 @@ void worker_probe_timer_cb(void* arg) } struct worker* -worker_create(struct daemon* daemon, int id, int* ports, int n) +worker_create(struct daemon* daemon, int id) { unsigned int seed; struct worker* worker = (struct worker*)calloc(1, sizeof(struct worker)); if(!worker) return NULL; - worker->numports = n; - worker->ports = (int*)memdup(ports, sizeof(int)*n); - if(!worker->ports) { - free(worker); - return NULL; - } worker->daemon = daemon; worker->thread_num = id; if(!(worker->cmd = tube_create())) { - free(worker->ports); free(worker); return NULL; } @@ -2249,7 +2242,6 @@ worker_create(struct daemon* daemon, int id, int* ports, int n) if(!(worker->rndstate = ub_initstate(daemon->rand))) { log_err("could not init random numbers."); tube_delete(worker->cmd); - free(worker->ports); free(worker); return NULL; } @@ -2348,14 +2340,14 @@ worker_init(struct worker* worker, struct config_file *cfg, cfg->out_ifs, cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, cfg->do_tcp?cfg->outgoing_num_tcp:0, cfg->ip_dscp, worker->daemon->env->infra_cache, worker->rndstate, - cfg->use_caps_bits_for_id, worker->ports, worker->numports, + cfg->use_caps_bits_for_id, cfg->unwanted_threshold, cfg->outgoing_tcp_mss, &worker_alloc_cleanup, worker, cfg->do_udp || cfg->udp_upstream_without_downstream, worker->daemon->connect_dot_sslctx, cfg->delay_close, cfg->tls_use_sni, dtenv, cfg->udp_connect, cfg->max_reuse_tcp_queries, cfg->tcp_reuse_timeout, - cfg->tcp_auth_query_timeout); + cfg->tcp_auth_query_timeout, worker->daemon->shared_ports); if(!worker->back) { log_err("could not create outgoing sockets"); worker_delete(worker); @@ -2506,7 +2498,6 @@ worker_delete(struct worker* worker) tube_delete(worker->cmd); comm_timer_delete(worker->stat_timer); comm_timer_delete(worker->env.probe_timer); - free(worker->ports); if(worker->thread_num == 0) { #ifdef UB_ON_WINDOWS wsvc_desetup_worker(worker); diff --git a/daemon/worker.h b/daemon/worker.h index b7bb52fd7..37f3728ef 100644 --- a/daemon/worker.h +++ b/daemon/worker.h @@ -104,10 +104,6 @@ struct worker { struct listen_dnsport* front; /** the backside outside network interface to the auth servers */ struct outside_network* back; - /** ports to be used by this worker. */ - int* ports; - /** number of ports for this worker */ - int numports; /** the signal handler */ struct comm_signal* comsig; /** commpoint to listen to commands. */ @@ -146,11 +142,9 @@ struct worker { * with backpointers only. Use worker_init on it later. * @param daemon: the daemon that this worker thread is part of. * @param id: the thread number from 0.. numthreads-1. - * @param ports: the ports it is allowed to use, array. - * @param n: the number of ports. * @return: the new worker or NULL on alloc failure. */ -struct worker* worker_create(struct daemon* daemon, int id, int* ports, int n); +struct worker* worker_create(struct daemon* daemon, int id); /** * Initialize worker. diff --git a/libunbound/libworker.c b/libunbound/libworker.c index 6e7244c03..d70527f59 100644 --- a/libunbound/libworker.c +++ b/libunbound/libworker.c @@ -105,6 +105,7 @@ libworker_delete_env(struct libworker* w) SSL_CTX_free(w->sslctx); #endif outside_network_delete(w->back); + shared_ports_delete(w->shared_ports); } /** delete libworker struct */ @@ -219,17 +220,25 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb) libworker_delete(w); return NULL; } + if(!(w->shared_ports = shared_ports_create(cfg->out_ifs, + cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, ports, numports))) { + if(!w->is_bg || w->is_bg_thread) { + lock_basic_unlock(&ctx->cfglock); + } + libworker_delete(w); + return NULL; + } w->back = outside_network_create(w->base, cfg->msg_buffer_size, (size_t)cfg->outgoing_num_ports, cfg->out_ifs, cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, cfg->do_tcp?cfg->outgoing_num_tcp:0, cfg->ip_dscp, w->env->infra_cache, w->env->rnd, cfg->use_caps_bits_for_id, - ports, numports, cfg->unwanted_threshold, + cfg->unwanted_threshold, cfg->outgoing_tcp_mss, &libworker_alloc_cleanup, w, cfg->do_udp || cfg->udp_upstream_without_downstream, w->sslctx, cfg->delay_close, cfg->tls_use_sni, NULL, cfg->udp_connect, cfg->max_reuse_tcp_queries, cfg->tcp_reuse_timeout, - cfg->tcp_auth_query_timeout); + cfg->tcp_auth_query_timeout, w->shared_ports); w->env->outnet = w->back; if(!w->is_bg || w->is_bg_thread) { lock_basic_unlock(&ctx->cfglock); diff --git a/libunbound/libworker.h b/libunbound/libworker.h index 42aa5bae3..f527cb093 100644 --- a/libunbound/libworker.h +++ b/libunbound/libworker.h @@ -60,6 +60,7 @@ struct tube; struct sldns_buffer; struct ub_event_base; struct query_info; +struct shared_ports; /** * The library-worker status structure @@ -84,6 +85,8 @@ struct libworker { struct comm_base* base; /** the backside outside network interface to the auth servers */ struct outside_network* back; + /** shared ports structure */ + struct shared_ports* shared_ports; /** random() table for this worker. */ struct ub_randstate* rndstate; /** sslcontext for SSL wrapped DNS over TCP queries */ diff --git a/services/outside_network.c b/services/outside_network.c index 8034ff60b..4533217ae 100644 --- a/services/outside_network.c +++ b/services/outside_network.c @@ -1480,7 +1480,7 @@ portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc) pif = pc->pif; log_assert(pif->inuse > 0); #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_ports[pif->avail_total - pif->inuse] = pc->number; + shared_ports_return_port(outnet->shared_ports, pif->shpif, pc->number); #endif pif->inuse--; pif->out[pc->index] = pif->out[pif->inuse]; @@ -1694,19 +1694,19 @@ create_pending_tcp(struct outside_network* outnet, size_t bufsize) } /** setup an outgoing interface, ready address */ -static int setup_if(struct port_if* pif, const char* addrstr, - int* avail, int numavail, size_t numfd) +static int setup_if(struct port_if* pif, const char* addrstr, size_t numfd, + struct shared_ports* shp) { -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_total = numavail; - pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int)); - if(!pif->avail_ports) - return 0; -#endif if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) && !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen, &pif->pfxlen)) return 0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + pif->shpif = shared_ports_find_if(shp, &pif->addr, pif->addrlen, + pif->pfxlen); +#else + (void)shp; +#endif pif->maxout = (int)numfd; pif->inuse = 0; pif->out = (struct port_comm**)calloc(numfd, @@ -1720,12 +1720,12 @@ struct outside_network* outside_network_create(struct comm_base *base, size_t bufsize, size_t num_ports, char** ifs, int num_ifs, int do_ip4, int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra, - struct ub_randstate* rnd, int use_caps_for_id, int* availports, - int numavailports, size_t unwanted_threshold, int tcp_mss, + struct ub_randstate* rnd, int use_caps_for_id, + size_t unwanted_threshold, int tcp_mss, void (*unwanted_action)(void*), void* unwanted_param, int do_udp, void* sslctx, int delayclose, int tls_use_sni, struct dt_env* dtenv, int udp_connect, int max_reuse_tcp_queries, int tcp_reuse_timeout, - int tcp_auth_query_timeout) + int tcp_auth_query_timeout, struct shared_ports* shared_ports) { struct outside_network* outnet = (struct outside_network*) calloc(1, sizeof(struct outside_network)); @@ -1760,6 +1760,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, outnet->do_udp = do_udp; outnet->tcp_mss = tcp_mss; outnet->ip_dscp = dscp; + outnet->shared_ports = shared_ports; #ifndef S_SPLINT_S if(delayclose) { outnet->delayclose = 1; @@ -1770,7 +1771,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, if(udp_connect) { outnet->udp_connect = 1; } - if(numavailports == 0 || num_ports == 0) { + if(num_ports == 0) { log_err("no outgoing ports available"); outside_network_delete(outnet); return NULL; @@ -1831,13 +1832,13 @@ outside_network_create(struct comm_base *base, size_t bufsize, /* allocate interfaces */ if(num_ifs == 0) { if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0", - availports, numavailports, num_ports)) { + num_ports, outnet->shared_ports)) { log_err("malloc failed"); outside_network_delete(outnet); return NULL; } if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::", - availports, numavailports, num_ports)) { + num_ports, outnet->shared_ports)) { log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1848,7 +1849,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, for(i=0; iip6_ifs[done_6], ifs[i], - availports, numavailports, num_ports)){ + num_ports, outnet->shared_ports)){ log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1857,7 +1858,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, } if(!str_is_ip6(ifs[i]) && do_ip4) { if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i], - availports, numavailports, num_ports)){ + num_ports, outnet->shared_ports)){ log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1935,9 +1936,6 @@ outside_network_delete(struct outside_network* outnet) comm_point_delete(pc->cp); free(pc); } -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - free(outnet->ip4_ifs[i].avail_ports); -#endif free(outnet->ip4_ifs[i].out); } free(outnet->ip4_ifs); @@ -1951,9 +1949,6 @@ outside_network_delete(struct outside_network* outnet) comm_point_delete(pc->cp); free(pc); } -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - free(outnet->ip6_ifs[i].avail_ports); -#endif free(outnet->ip6_ifs[i].out); } free(outnet->ip6_ifs); @@ -2163,7 +2158,10 @@ static int select_ifport(struct outside_network* outnet, struct pending* pend, int num_if, struct port_if* ifs) { - int my_if, my_port, fd, portno, inuse, tries=0; + int my_if, fd, portno, inuse, tries=0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int reused; +#endif struct port_if* pif; /* randomly select interface and port */ if(num_if == 0) { @@ -2177,37 +2175,35 @@ select_ifport(struct outside_network* outnet, struct pending* pend, my_if = ub_random_max(outnet->rnd, num_if); pif = &ifs[my_if]; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - if(outnet->udp_connect) { - /* if we connect() we cannot reuse fds for a port */ - if(pif->inuse >= pif->avail_total) { - tries++; - if(tries < MAX_PORT_RETRY) - continue; - log_err("failed to find an open port, drop msg"); - return 0; - } - my_port = pif->inuse + ub_random_max(outnet->rnd, - pif->avail_total - pif->inuse); - } else { - my_port = ub_random_max(outnet->rnd, pif->avail_total); - if(my_port < pif->inuse) { - /* port already open */ - pend->pc = pif->out[my_port]; - verbose(VERB_ALGO, "using UDP if=%d port=%d", - my_if, pend->pc->number); - break; - } + if(!shared_ports_fetch_random(outnet->shared_ports, + pif->shpif, outnet->rnd, outnet->udp_connect, + pif->inuse, &portno, &reused)) { + tries++; + if(tries < MAX_PORT_RETRY) + continue; + log_err("failed to find an open port, drop msg"); + return 0; + } + if(reused) { + /* port already open */ + log_assert(portno < pif->inuse); + pend->pc = pif->out[portno]; + verbose(VERB_ALGO, "using UDP if=%d port=%d", + my_if, pend->pc->number); + break; } - /* try to open new port, if fails, loop to try again */ - log_assert(pif->inuse < pif->maxout); - portno = pif->avail_ports[my_port - pif->inuse]; #else - my_port = portno = 0; + portno = 0; #endif + /* try to open new port, if fails, loop to try again */ fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen, portno, &inuse, outnet->rnd, outnet->ip_dscp); if(fd == -1 && !inuse) { /* nonrecoverable error making socket */ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port(outnet->shared_ports, + pif->shpif, portno); +#endif return 0; } if(fd != -1) { @@ -2224,6 +2220,11 @@ select_ifport(struct outside_network* outnet, struct pending* pend, pend->addrlen); } sock_close(fd); +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port( + outnet->shared_ports, + pif->shpif, portno); +#endif return 0; } } @@ -2241,14 +2242,14 @@ select_ifport(struct outside_network* outnet, struct pending* pend, /* grab port in interface */ pif->out[pif->inuse] = pend->pc; -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_ports[my_port - pif->inuse] = - pif->avail_ports[pif->avail_total-pif->inuse-1]; -#endif pif->inuse++; break; } /* failed, already in use */ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port(outnet->shared_ports, pif->shpif, + portno); +#endif verbose(VERB_QUERY, "port %d in use, trying another", portno); tries++; if(tries == MAX_PORT_RETRY) { @@ -3630,13 +3631,16 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, { struct sockaddr_storage* addr; socklen_t addrlen; - int i, try, pnum, dscp; + int i, try, dscp; struct port_if* pif; /* create fd */ dscp = outnet->ip_dscp; for(try = 0; try<1000; try++) { int port = 0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int reused = 0; +#endif int freebind = 0; int noproto = 0; int inuse = 0; @@ -3665,16 +3669,18 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, addr = &pif->addr; addrlen = pif->addrlen; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pnum = ub_random_max(outnet->rnd, pif->avail_total); - if(pnum < pif->inuse) { - /* port already open */ - port = pif->out[pnum]->number; - } else { - /* unused ports in start part of array */ - port = pif->avail_ports[pnum - pif->inuse]; + if(!shared_ports_fetch_random(outnet->shared_ports, + pif->shpif, outnet->rnd, 0, pif->inuse, + &port, &reused)) { + /* try again, perhaps another interface. */ + continue; + } + if(reused) { + log_assert(port < pif->inuse); + port = pif->out[port]->number; } #else - pnum = port = 0; + port = 0; #endif if(addr_is_ip6(to_addr, to_addrlen)) { struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr; @@ -3689,6 +3695,14 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, (struct sockaddr*)addr, addrlen, 1, &inuse, &noproto, 0, 0, 0, NULL, 0, freebind, 0, dscp); } +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + if(!reused) { + /* Return the port to the pool, since the caller does + * not keep track of it, also have done fd, and bind. */ + shared_ports_return_port(outnet->shared_ports, + pif->shpif, port); + } +#endif if(fd != -1) { return fd; } @@ -3919,11 +3933,7 @@ if_get_mem(struct port_if* pif) { size_t s; int i; - s = sizeof(*pif) + -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - sizeof(int)*pif->avail_total + -#endif - sizeof(struct port_comm*)*pif->maxout; + s = sizeof(*pif) + sizeof(struct port_comm*)*pif->maxout; for(i=0; iinuse; i++) s += sizeof(*pif->out[i]) + comm_point_get_mem(pif->out[i]->cp); @@ -4011,3 +4021,237 @@ serviced_get_mem(struct serviced_query* sq) return s; } +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Setup shared port interface */ +static int shared_ports_setup_if(struct shared_ports_if* shpif, char* str, + int* availports, int numavailports) +{ + shpif->avail_ports = (int*)memdup(availports, + (size_t)numavailports*sizeof(int)); + if(!shpif->avail_ports) + return 0; + shpif->avail_total = numavailports; + shpif->inuse = 0; + shpif->pfxlen = 0; + if(!ipstrtoaddr(str, UNBOUND_DNS_PORT, &shpif->addr, &shpif->addrlen) && + !netblockstrtoaddr(str, UNBOUND_DNS_PORT, &shpif->addr, + &shpif->addrlen, &shpif->pfxlen)) + return 0; + return 1; +} +#endif + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Allocate shared ports interfaces */ +static int shared_ports_alloc_ifs(struct shared_ports* shp, char** ifs, + int num_ifs, int do_ip4, int do_ip6, int* availports, + int numavailports) +{ +#ifndef INET6 + do_ip6 = 0; +#endif + calc_num46(ifs, num_ifs, do_ip4, do_ip6, + &shp->num_ip4, &shp->num_ip6); + if(shp->num_ip4 != 0) { + if(!(shp->ip4_ifs = (struct shared_ports_if*)calloc( + (size_t)shp->num_ip4, + sizeof(struct shared_ports_if)))) + return 0; + } + if(shp->num_ip6 != 0) { + if(!(shp->ip6_ifs = (struct shared_ports_if*)calloc( + (size_t)shp->num_ip6, + sizeof(struct shared_ports_if)))) + return 0; + } + if(num_ifs == 0) { + if(do_ip4 && !shared_ports_setup_if(&shp->ip4_ifs[0], + "0.0.0.0", availports, numavailports)) + return 0; + if(do_ip6 && !shared_ports_setup_if(&shp->ip6_ifs[0], + "::", availports, numavailports)) + return 0; + } else { + size_t done_4 = 0, done_6 = 0; + int i; + for(i=0; iip6_ifs[done_6], + ifs[i], availports, numavailports)) + return 0; + done_6++; + } + if(!str_is_ip6(ifs[i]) && do_ip4) { + if(!shared_ports_setup_if(&shp->ip4_ifs[done_4], + ifs[i], availports, numavailports)) + return 0; + done_4++; + } + } + } + return 1; +} +#endif + +struct shared_ports* shared_ports_create(char** ifs, int num_ifs, int do_ip4, + int do_ip6, int* availports, int numavailports) +{ + struct shared_ports* shp = calloc(1, sizeof(*shp)); + if(!shp) { + log_err("malloc failed"); + return NULL; + } + lock_basic_init(&shp->lock); + lock_protect(&shp->lock, shp, sizeof(*shp)); + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + /* Allocate interfaces */ + if(!shared_ports_alloc_ifs(shp, ifs, num_ifs, do_ip4, do_ip6, + availports, numavailports)) { + log_err("malloc failed"); + shared_ports_delete(shp); + return NULL; + } +#else + (void)ifs; (void)num_ifs; (void)do_ip4; (void)do_ip6; + (void)availports; (void)numavailports; +#endif + return shp; +} + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Delete shared ports interface structure */ +static void shared_ports_if_delete(struct shared_ports_if* shpif) +{ + if(!shpif) + return; + free(shpif->avail_ports); +} +#endif + +void shared_ports_delete(struct shared_ports* shp) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int i; +#endif + if(!shp) + return; + lock_basic_destroy(&shp->lock); +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + for(i=0; inum_ip4; i++) { + shared_ports_if_delete(&shp->ip4_ifs[i]); + } + free(shp->ip4_ifs); + for(i=0; inum_ip6; i++) { + shared_ports_if_delete(&shp->ip6_ifs[i]); + } + free(shp->ip6_ifs); +#endif + free(shp); +} + +struct shared_ports_if* shared_ports_find_if(struct shared_ports* shp, + struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + struct shared_ports_if* ret, *ifs = NULL; + int i, num_ifs = 0; + lock_basic_lock(&shp->lock); + if(addr_is_ip6(addr, addrlen)) { + ifs = shp->ip6_ifs; + num_ifs = shp->num_ip6; + } else { + ifs = shp->ip4_ifs; + num_ifs = shp->num_ip4; + } + for(i=0; ilock); + return ret; + } + } + lock_basic_unlock(&shp->lock); + return NULL; +#else + (void)shp; (void)addr; (void)addrlen; (void)pfxlen; + return NULL; +#endif +} + +int shared_ports_fetch_random(struct shared_ports* shp, + struct shared_ports_if* shpif, struct ub_randstate* rnd, + int udp_connect, int reusenum, int* port, int* reused) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int portno = 0, my_port = 0; + if(!shpif) + return 0; + lock_basic_lock(&shp->lock); + if(udp_connect) { + /* if we connect() we cannot reuse fds for a port. */ + if(shpif->inuse >= shpif->avail_total) { + lock_basic_unlock(&shp->lock); + return 0; + } + my_port = ub_random_max(rnd, + shpif->avail_total - shpif->inuse); + } else { + /* select from free ports and open ports on this thread. */ + if(shpif->inuse >= shpif->avail_total) { + lock_basic_unlock(&shp->lock); + if(reusenum == 0) { + return 0; + } + my_port = ub_random_max(rnd, reusenum); + *port = my_port; + *reused = 1; + return 1; + } + my_port = ub_random_max(rnd, shpif->avail_total - shpif->inuse + + reusenum); + if(my_port < reusenum) { + /* port already open */ + lock_basic_unlock(&shp->lock); + *port = my_port; + *reused = 1; + return 1; + } + my_port -= reusenum; + } + log_assert(shpif->inuse < shpif->avail_total); + log_assert(my_port >= 0 && my_port < shpif->avail_total); + portno = shpif->avail_ports[my_port]; + shpif->avail_ports[my_port] = + shpif->avail_ports[shpif->avail_total-shpif->inuse-1]; + shpif->inuse++; + lock_basic_unlock(&shp->lock); + *port = portno; + *reused = 0; + return 1; +#else + (void)shp; (void)shpif; (void)rnd; (void)udp_connect; + (void)reusenum; + *port = 0; + *reused = 0; + return 1; +#endif +} + +void shared_ports_return_port(struct shared_ports* shp, + struct shared_ports_if* shpif, int port) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + if(!shpif) + return; + lock_basic_lock(&shp->lock); + log_assert(shpif->inuse > 0); + shpif->avail_ports[shpif->avail_total - shpif->inuse] = port; + shpif->inuse--; + lock_basic_unlock(&shp->lock); +#else + (void)shp; (void)shpif; (void)port; +#endif +} diff --git a/services/outside_network.h b/services/outside_network.h index e30ce92eb..573f3cb02 100644 --- a/services/outside_network.h +++ b/services/outside_network.h @@ -70,6 +70,8 @@ struct module_env; struct module_qstate; struct query_info; struct config_file; +struct shared_ports; +struct shared_ports_if; /** * Send queries to outside servers and wait for answers from servers. @@ -119,6 +121,9 @@ struct outside_network { int udp_connect; /** number of udp packets sent. */ size_t num_udp_outgoing; + /** the shared ports structure, with random ports numbers. + * This is a reference to the member in the daemon structure. */ + struct shared_ports* shared_ports; /** array of outgoing IP4 interfaces */ struct port_if* ip4_ifs; @@ -211,11 +216,8 @@ struct port_if { int pfxlen; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - /** the available ports array. These are unused. - * Only the first total-inuse part is filled. */ - int* avail_ports; - /** the total number of available ports (size of the array) */ - int avail_total; + /** the shared port numbers for this interface. */ + struct shared_ports_if* shpif; #endif /** array of the commpoints currently in use. @@ -245,6 +247,42 @@ struct port_comm { struct comm_point* cp; }; +/** + * Shared ports, the list of ports shared across threads + */ +struct shared_ports { + /** mutex on the ports */ + lock_basic_type lock; + /** array of IP4 interfaces */ + struct shared_ports_if* ip4_ifs; + /** number of outgoing IP4 interfaces */ + int num_ip4; + /** array of IP6 interfaces */ + struct shared_ports_if* ip6_ifs; + /** number of outgoing IP6 interfaces */ + int num_ip6; +}; + +/** + * Shared ports for an interface. + */ +struct shared_ports_if { + /** address ready to allocate new socket (except port no). */ + struct sockaddr_storage addr; + /** length of addr field */ + socklen_t addrlen; + /** if a netblock, the prefix */ + int pfxlen; + + /** the available ports array. These are unused. + * Only the first total-inuse part is filled. */ + int* avail_ports; + /** the total number of available ports (size of the array) */ + int avail_total; + /** the number in use. */ + int inuse; +}; + /** * Reuse TCP connection, still open can be used again. */ @@ -551,8 +589,6 @@ struct serviced_query { * @param infra: pointer to infra cached used for serviced queries. * @param rnd: stored to create random numbers for serviced queries. * @param use_caps_for_id: enable to use 0x20 bits to encode id randomness. - * @param availports: array of available ports. - * @param numavailports: number of available ports in array. * @param unwanted_threshold: when to take defensive action. * @param unwanted_action: the action to take. * @param unwanted_param: user parameter to action. @@ -567,17 +603,18 @@ struct serviced_query { * @param max_reuse_tcp_queries: max number of queries on a reuse connection. * @param tcp_reuse_timeout: timeout for REUSE entries in milliseconds. * @param tcp_auth_query_timeout: timeout in milliseconds for TCP queries to auth servers. + * @param shared_ports: the shared_ports structure. * @return: the new structure (with no pending answers) or NULL on error. */ struct outside_network* outside_network_create(struct comm_base* base, size_t bufsize, size_t num_ports, char** ifs, int num_ifs, int do_ip4, int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra, - struct ub_randstate* rnd, int use_caps_for_id, int* availports, - int numavailports, size_t unwanted_threshold, int tcp_mss, + struct ub_randstate* rnd, int use_caps_for_id, + size_t unwanted_threshold, int tcp_mss, void (*unwanted_action)(void*), void* unwanted_param, int do_udp, void* sslctx, int delayclose, int tls_use_sni, struct dt_env *dtenv, int udp_connect, int max_reuse_tcp_queries, int tcp_reuse_timeout, - int tcp_auth_query_timeout); + int tcp_auth_query_timeout, struct shared_ports* shared_ports); /** * Delete outside_network structure. @@ -819,6 +856,54 @@ struct comm_point* outnet_comm_point_for_http(struct outside_network* outnet, /** connect tcp connection to addr, 0 on failure */ int outnet_tcp_connect(int s, struct sockaddr_storage* addr, socklen_t addrlen); +/** + * Create new shared ports structure. + * @param ifs: interface names (or NULL for default interface). + * These interfaces must be able to access all authoritative servers. + * @param num_ifs: number of names in array ifs. + * @param do_ip4: service IP4. + * @param do_ip6: service IP6. + * @param availports: array of available ports. + * @param numavailports: number of available ports in array. + * @return new, or NULL on failure. + */ +struct shared_ports* shared_ports_create(char** ifs, int num_ifs, int do_ip4, + int do_ip6, int* availports, int numavailports); + +/** + * Delete shared ports structure. + * @param shp: shared ports structure. + */ +void shared_ports_delete(struct shared_ports* shp); + +/** Find interface in shared ports. */ +struct shared_ports_if* shared_ports_find_if(struct shared_ports* shp, + struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen); + +/** + * Get a shared port from the list of random ports. + * @param shp: shared ports structure. + * @param shpif: the shared ports interface. + * @param rnd: used to make random numbers. + * @param udp_connect: set to true if no reuse is possible. + * @param reusenum: number of ports that can be reused (already open). + * @param port: the port number is returned. + * @param reused: if the port numer is reused, returned. + * @return false on failure. That can mean no more free ports to use. + */ +int shared_ports_fetch_random(struct shared_ports* shp, + struct shared_ports_if* shpif, struct ub_randstate* rnd, + int udp_connect, int reusenum, int* port, int* reused); + +/** + * Return a shared port to the list of random ports. + * @param shp: shared ports structure. + * @param shpif: the shared ports interface. + * @param port: port number to return to be used again. + */ +void shared_ports_return_port(struct shared_ports* shp, + struct shared_ports_if* shpif, int port); + /** callback for incoming udp answers from the network */ int outnet_udp_cb(struct comm_point* c, void* arg, int error, struct comm_reply *reply_info); diff --git a/testcode/fake_event.c b/testcode/fake_event.c index ce439edd1..6b6ab3731 100644 --- a/testcode/fake_event.c +++ b/testcode/fake_event.c @@ -1126,15 +1126,16 @@ outside_network_create(struct comm_base* base, size_t bufsize, int ATTR_UNUSED(dscp), struct infra_cache* infra, struct ub_randstate* ATTR_UNUSED(rnd), - int ATTR_UNUSED(use_caps_for_id), int* ATTR_UNUSED(availports), - int ATTR_UNUSED(numavailports), size_t ATTR_UNUSED(unwanted_threshold), + int ATTR_UNUSED(use_caps_for_id), + size_t ATTR_UNUSED(unwanted_threshold), int ATTR_UNUSED(outgoing_tcp_mss), void (*unwanted_action)(void*), void* ATTR_UNUSED(unwanted_param), int ATTR_UNUSED(do_udp), void* ATTR_UNUSED(sslctx), int ATTR_UNUSED(delayclose), int ATTR_UNUSED(tls_use_sni), struct dt_env* ATTR_UNUSED(dtenv), int ATTR_UNUSED(udp_connect), int ATTR_UNUSED(max_reuse_tcp_queries), int ATTR_UNUSED(tcp_reuse_timeout), - int ATTR_UNUSED(tcp_auth_query_timeout)) + int ATTR_UNUSED(tcp_auth_query_timeout), + struct shared_ports* ATTR_UNUSED(shared_ports)) { struct replay_runtime* runtime = (struct replay_runtime*)base; struct outside_network* outnet = calloc(1, @@ -1980,6 +1981,20 @@ int outnet_tcp_connect(int ATTR_UNUSED(s), struct sockaddr_storage* ATTR_UNUSED( return 0; } +struct shared_ports* shared_ports_create(char** ATTR_UNUSED(ifs), + int ATTR_UNUSED(num_ifs), int ATTR_UNUSED(do_ip4), + int ATTR_UNUSED(do_ip6), int* ATTR_UNUSED(availports), + int ATTR_UNUSED(numavailports)) +{ + return calloc(1, sizeof(struct shared_ports)); +} + +void shared_ports_delete(struct shared_ports* shp) +{ + if(!shp) return; + free(shp); +} + int tcp_req_info_add_meshstate(struct tcp_req_info* ATTR_UNUSED(req), struct mesh_area* ATTR_UNUSED(mesh), struct mesh_state* ATTR_UNUSED(m)) { diff --git a/testcode/unittcpreuse.c b/testcode/unittcpreuse.c index 5f45a4b45..ce62e3325 100644 --- a/testcode/unittcpreuse.c +++ b/testcode/unittcpreuse.c @@ -41,6 +41,7 @@ #include "config.h" #include "testcode/unitmain.h" #include "util/log.h" +#include "util/net_help.h" #include "util/random.h" #include "services/outside_network.h" @@ -479,6 +480,278 @@ static void reuse_write_wait_test(void) check_reuse_write_wait_removal(1, &reuse, store, 0, 1); } +static void shared_port_test_ifs(void) +{ + struct shared_ports* shp; + struct shared_ports_if* shpif; + char* ifs[] = {"1.2.3.4", "1.2.3.5", "::1:2", "::1:3"}; + int availports[] = {1, 2, 3, 4}; + struct sockaddr_storage addr; + socklen_t addrlen; + + shp = shared_ports_create(ifs, 4, 1, 1, availports, 4); + unit_assert(shp); + + if(!ipstrtoaddr("1.2.3.4", UNBOUND_DNS_PORT, &addr, &addrlen)) + log_err("could not parse"); + shpif = shared_ports_find_if(shp, &addr, addrlen, 0); + unit_assert(shpif); + + if(!ipstrtoaddr("1.2.3.5", UNBOUND_DNS_PORT, &addr, &addrlen)) + log_err("could not parse"); + shpif = shared_ports_find_if(shp, &addr, addrlen, 0); + unit_assert(shpif); + + if(!ipstrtoaddr("::1:2", UNBOUND_DNS_PORT, &addr, &addrlen)) + log_err("could not parse"); + shpif = shared_ports_find_if(shp, &addr, addrlen, 0); + unit_assert(shpif); + + if(!ipstrtoaddr("::1:3", UNBOUND_DNS_PORT, &addr, &addrlen)) + log_err("could not parse"); + shpif = shared_ports_find_if(shp, &addr, addrlen, 0); + unit_assert(shpif); + + shared_ports_delete(shp); +} + +/** See if a port is on the shared_ports ports list */ +static int +pif_list_contains(struct shared_ports_if* shpif, int item) +{ + int i; + unit_assert(shpif->inuse >= 0 && shpif->inuse <= shpif->avail_total); + for(i=0; i< shpif->avail_total - shpif->inuse; i++) { + if(shpif->avail_ports[i] == item) + return 1; + } + return 0; +} + +/** See if a number of ports are on the shared_ports list */ +static int +pif_list_contains_items(struct shared_ports_if* shpif, int item1, + int item2, int item3, int item4) +{ + if(item1 != -1 && !pif_list_contains(shpif, item1)) + return 0; + if(item2 != -1 && !pif_list_contains(shpif, item2)) + return 0; + if(item3 != -1 && !pif_list_contains(shpif, item3)) + return 0; + if(item4 != -1 && !pif_list_contains(shpif, item4)) + return 0; + return 1; +} + +static void shared_port_test_port(void) +{ + struct shared_ports* shp; + struct shared_ports_if* shpif; + char* ifs[] = {"1.2.3.4", "1.2.3.5"}; + int availports[] = {1, 2, 3, 4}; + struct sockaddr_storage addr; + socklen_t addrlen; + int p1, p2, p3, reused; + struct ub_randstate* rnd; + + rnd = ub_initstate(NULL); + unit_assert(rnd); + + shp = shared_ports_create(ifs, 2, 1, 1, availports, 4); + unit_assert(shp); + + if(!ipstrtoaddr("1.2.3.4", UNBOUND_DNS_PORT, &addr, &addrlen)) + log_err("could not parse"); + shpif = shared_ports_find_if(shp, &addr, addrlen, 0); + unit_assert(shpif); + + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 0); + unit_assert(pif_list_contains_items(shpif, 1, 2, 3, 4)); + + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + unit_assert(!pif_list_contains(shpif, p1)); + if(p1 != 1) unit_assert(pif_list_contains(shpif, 1)); + if(p1 != 2) unit_assert(pif_list_contains(shpif, 2)); + if(p1 != 3) unit_assert(pif_list_contains(shpif, 3)); + if(p1 != 4) unit_assert(pif_list_contains(shpif, 4)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 1); + + shared_ports_return_port(shp, shpif, p1); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 0); + unit_assert(pif_list_contains_items(shpif, 1, 2, 3, 4)); + + /* pick up two items */ + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p2, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p2 != 0); + unit_assert(!pif_list_contains(shpif, p1)); + unit_assert(!pif_list_contains(shpif, p2)); + if(p1 != 1 && p2 != 1) unit_assert(pif_list_contains(shpif, 1)); + if(p1 != 2 && p2 != 2) unit_assert(pif_list_contains(shpif, 2)); + if(p1 != 3 && p2 != 3) unit_assert(pif_list_contains(shpif, 3)); + if(p1 != 4 && p2 != 4) unit_assert(pif_list_contains(shpif, 4)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 2); + + shared_ports_return_port(shp, shpif, p1); + unit_assert(pif_list_contains(shpif, p1)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 1); + + shared_ports_return_port(shp, shpif, p2); + unit_assert(pif_list_contains(shpif, p2)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 0); + unit_assert(pif_list_contains_items(shpif, 1, 2, 3, 4)); + + /* pick up three items */ + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p2, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p2 != 0); + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p3, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p3 != 0); + unit_assert(!pif_list_contains(shpif, p1)); + unit_assert(!pif_list_contains(shpif, p2)); + unit_assert(!pif_list_contains(shpif, p3)); + if(p1 != 1 && p2 != 1 && p3 != 1) + unit_assert(pif_list_contains(shpif, 1)); + if(p1 != 2 && p2 != 2 && p3 != 2) + unit_assert(pif_list_contains(shpif, 2)); + if(p1 != 3 && p2 != 3 && p3 != 3) + unit_assert(pif_list_contains(shpif, 3)); + if(p1 != 4 && p2 != 4 && p3 != 4) + unit_assert(pif_list_contains(shpif, 4)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 3); + + shared_ports_return_port(shp, shpif, p1); + unit_assert(pif_list_contains(shpif, p1)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 2); + + shared_ports_return_port(shp, shpif, p2); + unit_assert(pif_list_contains(shpif, p2)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 1); + + shared_ports_return_port(shp, shpif, p3); + unit_assert(pif_list_contains(shpif, p3)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 0); + unit_assert(pif_list_contains_items(shpif, 1, 2, 3, 4)); + + /* pick up all four items */ + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0, 0, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 0); + unit_assert(p1 != 0); + unit_assert(!pif_list_contains(shpif, 1)); + unit_assert(!pif_list_contains(shpif, 2)); + unit_assert(!pif_list_contains(shpif, 3)); + unit_assert(!pif_list_contains(shpif, 4)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 4); + + /* more fetches fail, it is fully inuse. */ + unit_assert(!shared_ports_fetch_random(shp, shpif, rnd, 0, 0, &p2, + &reused)); + unit_assert(!shared_ports_fetch_random(shp, shpif, rnd, 0, 0, &p3, + &reused)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 4); + + /* reuse is then always the case */ + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0 /* can reuse */, 4 /* reusenum */, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 1); + unit_assert(p1 >= 0 && p1 < 4 /* reusenum */); + + if(!shared_ports_fetch_random(shp, shpif, rnd, + 0 /* can reuse */, 4 /* reusenum */, &p1, &reused)) { + unit_assert(0); /* should succeed */ + } + unit_assert(reused == 1); + unit_assert(p1 >= 0 && p1 < 4 /* reusenum */); + + /* return all the ports */ + shared_ports_return_port(shp, shpif, 1); + unit_assert(pif_list_contains(shpif, 1)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 3); + shared_ports_return_port(shp, shpif, 2); + unit_assert(pif_list_contains(shpif, 2)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 2); + shared_ports_return_port(shp, shpif, 3); + unit_assert(pif_list_contains(shpif, 3)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 1); + shared_ports_return_port(shp, shpif, 4); + unit_assert(pif_list_contains(shpif, 4)); + unit_assert(shpif->avail_total == 4); + unit_assert(shpif->inuse == 0); + unit_assert(pif_list_contains_items(shpif, 1, 2, 3, 4)); + + shared_ports_delete(shp); + ub_randfree(rnd); +} + void tcpreuse_test(void) { unit_show_feature("tcp_reuse"); @@ -486,4 +759,7 @@ void tcpreuse_test(void) tcp_reuse_tree_list_test(); waiting_tcp_list_test(); reuse_write_wait_test(); + unit_show_feature("shared_ports"); + shared_port_test_ifs(); + shared_port_test_port(); }