diff --git a/daemon/daemon.c b/daemon/daemon.c index ea4e83e70..51dd51de3 100644 --- a/daemon/daemon.c +++ b/daemon/daemon.c @@ -79,6 +79,7 @@ #include "util/tcp_conn_limit.h" #include "util/edns.h" #include "services/listen_dnsport.h" +#include "services/outside_network.h" #include "services/cache/rrset.h" #include "services/cache/infra.h" #include "services/localzone.h" @@ -813,6 +814,10 @@ daemon_create_workers(struct daemon* daemon) fatal_exit("out of memory during daemon init"); numport = daemon_get_shufport(daemon, shufport); verbose(VERB_ALGO, "total of %d outgoing ports available", numport); + if(!(daemon->shared_ports = shared_ports_create(daemon->cfg->out_ifs, + daemon->cfg->num_out_ifs, daemon->cfg->do_ip4, + daemon->cfg->do_ip6, shufport, numport))) + fatal_exit("could not setup shared ports: out of memory"); #ifdef HAVE_NGTCP2 if (cfg_has_quic(daemon->cfg)) { @@ -843,10 +848,7 @@ daemon_create_workers(struct daemon* daemon) #endif } for(i=0; inum; i++) { - if(!(daemon->workers[i] = worker_create(daemon, i, - shufport+numport*i/daemon->num, - numport*(i+1)/daemon->num - numport*i/daemon->num))) - /* the above is not ports/numthr, due to rounding */ + if(!(daemon->workers[i] = worker_create(daemon, i))) fatal_exit("could not create worker"); } /* create per-worker alloc caches if not reusing existing ones. */ @@ -1204,6 +1206,8 @@ daemon_cleanup(struct daemon* daemon) if(!daemon->reuse_cache || daemon->need_to_exit) daemon_clear_allocs(daemon); daemon->num = 0; + shared_ports_delete(daemon->shared_ports); + daemon->shared_ports = NULL; #ifdef USE_DNSTAP dt_delete(daemon->dtenv); daemon->dtenv = NULL; diff --git a/daemon/daemon.h b/daemon/daemon.h index 20386d7fc..e6f099629 100644 --- a/daemon/daemon.h +++ b/daemon/daemon.h @@ -62,6 +62,7 @@ struct doq_table; struct cookie_secrets; struct fast_reload_thread; struct fast_reload_printq; +struct shared_ports; #include "dnstap/dnstap_config.h" #ifdef USE_DNSTAP @@ -97,6 +98,8 @@ struct daemon { int rc_port; /** listening ports for remote control */ struct listen_port* rc_ports; + /** the shared ports structure, with random ports numbers. */ + struct shared_ports* shared_ports; /** remote control connections management (for first worker) */ struct daemon_remote* rc; /** ssl context for listening to dnstcp over ssl */ diff --git a/daemon/worker.c b/daemon/worker.c index a5dd9bc02..aa72f4e25 100644 --- a/daemon/worker.c +++ b/daemon/worker.c @@ -2225,23 +2225,16 @@ void worker_probe_timer_cb(void* arg) } struct worker* -worker_create(struct daemon* daemon, int id, int* ports, int n) +worker_create(struct daemon* daemon, int id) { unsigned int seed; struct worker* worker = (struct worker*)calloc(1, sizeof(struct worker)); if(!worker) return NULL; - worker->numports = n; - worker->ports = (int*)memdup(ports, sizeof(int)*n); - if(!worker->ports) { - free(worker); - return NULL; - } worker->daemon = daemon; worker->thread_num = id; if(!(worker->cmd = tube_create())) { - free(worker->ports); free(worker); return NULL; } @@ -2249,7 +2242,6 @@ worker_create(struct daemon* daemon, int id, int* ports, int n) if(!(worker->rndstate = ub_initstate(daemon->rand))) { log_err("could not init random numbers."); tube_delete(worker->cmd); - free(worker->ports); free(worker); return NULL; } @@ -2348,14 +2340,14 @@ worker_init(struct worker* worker, struct config_file *cfg, cfg->out_ifs, cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, cfg->do_tcp?cfg->outgoing_num_tcp:0, cfg->ip_dscp, worker->daemon->env->infra_cache, worker->rndstate, - cfg->use_caps_bits_for_id, worker->ports, worker->numports, + cfg->use_caps_bits_for_id, cfg->unwanted_threshold, cfg->outgoing_tcp_mss, &worker_alloc_cleanup, worker, cfg->do_udp || cfg->udp_upstream_without_downstream, worker->daemon->connect_dot_sslctx, cfg->delay_close, cfg->tls_use_sni, dtenv, cfg->udp_connect, cfg->max_reuse_tcp_queries, cfg->tcp_reuse_timeout, - cfg->tcp_auth_query_timeout); + cfg->tcp_auth_query_timeout, worker->daemon->shared_ports); if(!worker->back) { log_err("could not create outgoing sockets"); worker_delete(worker); @@ -2506,7 +2498,6 @@ worker_delete(struct worker* worker) tube_delete(worker->cmd); comm_timer_delete(worker->stat_timer); comm_timer_delete(worker->env.probe_timer); - free(worker->ports); if(worker->thread_num == 0) { #ifdef UB_ON_WINDOWS wsvc_desetup_worker(worker); diff --git a/daemon/worker.h b/daemon/worker.h index b7bb52fd7..37f3728ef 100644 --- a/daemon/worker.h +++ b/daemon/worker.h @@ -104,10 +104,6 @@ struct worker { struct listen_dnsport* front; /** the backside outside network interface to the auth servers */ struct outside_network* back; - /** ports to be used by this worker. */ - int* ports; - /** number of ports for this worker */ - int numports; /** the signal handler */ struct comm_signal* comsig; /** commpoint to listen to commands. */ @@ -146,11 +142,9 @@ struct worker { * with backpointers only. Use worker_init on it later. * @param daemon: the daemon that this worker thread is part of. * @param id: the thread number from 0.. numthreads-1. - * @param ports: the ports it is allowed to use, array. - * @param n: the number of ports. * @return: the new worker or NULL on alloc failure. */ -struct worker* worker_create(struct daemon* daemon, int id, int* ports, int n); +struct worker* worker_create(struct daemon* daemon, int id); /** * Initialize worker. diff --git a/libunbound/libworker.c b/libunbound/libworker.c index 6e7244c03..d70527f59 100644 --- a/libunbound/libworker.c +++ b/libunbound/libworker.c @@ -105,6 +105,7 @@ libworker_delete_env(struct libworker* w) SSL_CTX_free(w->sslctx); #endif outside_network_delete(w->back); + shared_ports_delete(w->shared_ports); } /** delete libworker struct */ @@ -219,17 +220,25 @@ libworker_setup(struct ub_ctx* ctx, int is_bg, struct ub_event_base* eb) libworker_delete(w); return NULL; } + if(!(w->shared_ports = shared_ports_create(cfg->out_ifs, + cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, ports, numports))) { + if(!w->is_bg || w->is_bg_thread) { + lock_basic_unlock(&ctx->cfglock); + } + libworker_delete(w); + return NULL; + } w->back = outside_network_create(w->base, cfg->msg_buffer_size, (size_t)cfg->outgoing_num_ports, cfg->out_ifs, cfg->num_out_ifs, cfg->do_ip4, cfg->do_ip6, cfg->do_tcp?cfg->outgoing_num_tcp:0, cfg->ip_dscp, w->env->infra_cache, w->env->rnd, cfg->use_caps_bits_for_id, - ports, numports, cfg->unwanted_threshold, + cfg->unwanted_threshold, cfg->outgoing_tcp_mss, &libworker_alloc_cleanup, w, cfg->do_udp || cfg->udp_upstream_without_downstream, w->sslctx, cfg->delay_close, cfg->tls_use_sni, NULL, cfg->udp_connect, cfg->max_reuse_tcp_queries, cfg->tcp_reuse_timeout, - cfg->tcp_auth_query_timeout); + cfg->tcp_auth_query_timeout, w->shared_ports); w->env->outnet = w->back; if(!w->is_bg || w->is_bg_thread) { lock_basic_unlock(&ctx->cfglock); diff --git a/libunbound/libworker.h b/libunbound/libworker.h index 42aa5bae3..f527cb093 100644 --- a/libunbound/libworker.h +++ b/libunbound/libworker.h @@ -60,6 +60,7 @@ struct tube; struct sldns_buffer; struct ub_event_base; struct query_info; +struct shared_ports; /** * The library-worker status structure @@ -84,6 +85,8 @@ struct libworker { struct comm_base* base; /** the backside outside network interface to the auth servers */ struct outside_network* back; + /** shared ports structure */ + struct shared_ports* shared_ports; /** random() table for this worker. */ struct ub_randstate* rndstate; /** sslcontext for SSL wrapped DNS over TCP queries */ diff --git a/services/outside_network.c b/services/outside_network.c index 8034ff60b..4533217ae 100644 --- a/services/outside_network.c +++ b/services/outside_network.c @@ -1480,7 +1480,7 @@ portcomm_loweruse(struct outside_network* outnet, struct port_comm* pc) pif = pc->pif; log_assert(pif->inuse > 0); #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_ports[pif->avail_total - pif->inuse] = pc->number; + shared_ports_return_port(outnet->shared_ports, pif->shpif, pc->number); #endif pif->inuse--; pif->out[pc->index] = pif->out[pif->inuse]; @@ -1694,19 +1694,19 @@ create_pending_tcp(struct outside_network* outnet, size_t bufsize) } /** setup an outgoing interface, ready address */ -static int setup_if(struct port_if* pif, const char* addrstr, - int* avail, int numavail, size_t numfd) +static int setup_if(struct port_if* pif, const char* addrstr, size_t numfd, + struct shared_ports* shp) { -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_total = numavail; - pif->avail_ports = (int*)memdup(avail, (size_t)numavail*sizeof(int)); - if(!pif->avail_ports) - return 0; -#endif if(!ipstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen) && !netblockstrtoaddr(addrstr, UNBOUND_DNS_PORT, &pif->addr, &pif->addrlen, &pif->pfxlen)) return 0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + pif->shpif = shared_ports_find_if(shp, &pif->addr, pif->addrlen, + pif->pfxlen); +#else + (void)shp; +#endif pif->maxout = (int)numfd; pif->inuse = 0; pif->out = (struct port_comm**)calloc(numfd, @@ -1720,12 +1720,12 @@ struct outside_network* outside_network_create(struct comm_base *base, size_t bufsize, size_t num_ports, char** ifs, int num_ifs, int do_ip4, int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra, - struct ub_randstate* rnd, int use_caps_for_id, int* availports, - int numavailports, size_t unwanted_threshold, int tcp_mss, + struct ub_randstate* rnd, int use_caps_for_id, + size_t unwanted_threshold, int tcp_mss, void (*unwanted_action)(void*), void* unwanted_param, int do_udp, void* sslctx, int delayclose, int tls_use_sni, struct dt_env* dtenv, int udp_connect, int max_reuse_tcp_queries, int tcp_reuse_timeout, - int tcp_auth_query_timeout) + int tcp_auth_query_timeout, struct shared_ports* shared_ports) { struct outside_network* outnet = (struct outside_network*) calloc(1, sizeof(struct outside_network)); @@ -1760,6 +1760,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, outnet->do_udp = do_udp; outnet->tcp_mss = tcp_mss; outnet->ip_dscp = dscp; + outnet->shared_ports = shared_ports; #ifndef S_SPLINT_S if(delayclose) { outnet->delayclose = 1; @@ -1770,7 +1771,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, if(udp_connect) { outnet->udp_connect = 1; } - if(numavailports == 0 || num_ports == 0) { + if(num_ports == 0) { log_err("no outgoing ports available"); outside_network_delete(outnet); return NULL; @@ -1831,13 +1832,13 @@ outside_network_create(struct comm_base *base, size_t bufsize, /* allocate interfaces */ if(num_ifs == 0) { if(do_ip4 && !setup_if(&outnet->ip4_ifs[0], "0.0.0.0", - availports, numavailports, num_ports)) { + num_ports, outnet->shared_ports)) { log_err("malloc failed"); outside_network_delete(outnet); return NULL; } if(do_ip6 && !setup_if(&outnet->ip6_ifs[0], "::", - availports, numavailports, num_ports)) { + num_ports, outnet->shared_ports)) { log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1848,7 +1849,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, for(i=0; iip6_ifs[done_6], ifs[i], - availports, numavailports, num_ports)){ + num_ports, outnet->shared_ports)){ log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1857,7 +1858,7 @@ outside_network_create(struct comm_base *base, size_t bufsize, } if(!str_is_ip6(ifs[i]) && do_ip4) { if(!setup_if(&outnet->ip4_ifs[done_4], ifs[i], - availports, numavailports, num_ports)){ + num_ports, outnet->shared_ports)){ log_err("malloc failed"); outside_network_delete(outnet); return NULL; @@ -1935,9 +1936,6 @@ outside_network_delete(struct outside_network* outnet) comm_point_delete(pc->cp); free(pc); } -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - free(outnet->ip4_ifs[i].avail_ports); -#endif free(outnet->ip4_ifs[i].out); } free(outnet->ip4_ifs); @@ -1951,9 +1949,6 @@ outside_network_delete(struct outside_network* outnet) comm_point_delete(pc->cp); free(pc); } -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - free(outnet->ip6_ifs[i].avail_ports); -#endif free(outnet->ip6_ifs[i].out); } free(outnet->ip6_ifs); @@ -2163,7 +2158,10 @@ static int select_ifport(struct outside_network* outnet, struct pending* pend, int num_if, struct port_if* ifs) { - int my_if, my_port, fd, portno, inuse, tries=0; + int my_if, fd, portno, inuse, tries=0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int reused; +#endif struct port_if* pif; /* randomly select interface and port */ if(num_if == 0) { @@ -2177,37 +2175,35 @@ select_ifport(struct outside_network* outnet, struct pending* pend, my_if = ub_random_max(outnet->rnd, num_if); pif = &ifs[my_if]; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - if(outnet->udp_connect) { - /* if we connect() we cannot reuse fds for a port */ - if(pif->inuse >= pif->avail_total) { - tries++; - if(tries < MAX_PORT_RETRY) - continue; - log_err("failed to find an open port, drop msg"); - return 0; - } - my_port = pif->inuse + ub_random_max(outnet->rnd, - pif->avail_total - pif->inuse); - } else { - my_port = ub_random_max(outnet->rnd, pif->avail_total); - if(my_port < pif->inuse) { - /* port already open */ - pend->pc = pif->out[my_port]; - verbose(VERB_ALGO, "using UDP if=%d port=%d", - my_if, pend->pc->number); - break; - } + if(!shared_ports_fetch_random(outnet->shared_ports, + pif->shpif, outnet->rnd, outnet->udp_connect, + pif->inuse, &portno, &reused)) { + tries++; + if(tries < MAX_PORT_RETRY) + continue; + log_err("failed to find an open port, drop msg"); + return 0; + } + if(reused) { + /* port already open */ + log_assert(portno < pif->inuse); + pend->pc = pif->out[portno]; + verbose(VERB_ALGO, "using UDP if=%d port=%d", + my_if, pend->pc->number); + break; } - /* try to open new port, if fails, loop to try again */ - log_assert(pif->inuse < pif->maxout); - portno = pif->avail_ports[my_port - pif->inuse]; #else - my_port = portno = 0; + portno = 0; #endif + /* try to open new port, if fails, loop to try again */ fd = udp_sockport(&pif->addr, pif->addrlen, pif->pfxlen, portno, &inuse, outnet->rnd, outnet->ip_dscp); if(fd == -1 && !inuse) { /* nonrecoverable error making socket */ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port(outnet->shared_ports, + pif->shpif, portno); +#endif return 0; } if(fd != -1) { @@ -2224,6 +2220,11 @@ select_ifport(struct outside_network* outnet, struct pending* pend, pend->addrlen); } sock_close(fd); +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port( + outnet->shared_ports, + pif->shpif, portno); +#endif return 0; } } @@ -2241,14 +2242,14 @@ select_ifport(struct outside_network* outnet, struct pending* pend, /* grab port in interface */ pif->out[pif->inuse] = pend->pc; -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pif->avail_ports[my_port - pif->inuse] = - pif->avail_ports[pif->avail_total-pif->inuse-1]; -#endif pif->inuse++; break; } /* failed, already in use */ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + shared_ports_return_port(outnet->shared_ports, pif->shpif, + portno); +#endif verbose(VERB_QUERY, "port %d in use, trying another", portno); tries++; if(tries == MAX_PORT_RETRY) { @@ -3630,13 +3631,16 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, { struct sockaddr_storage* addr; socklen_t addrlen; - int i, try, pnum, dscp; + int i, try, dscp; struct port_if* pif; /* create fd */ dscp = outnet->ip_dscp; for(try = 0; try<1000; try++) { int port = 0; +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int reused = 0; +#endif int freebind = 0; int noproto = 0; int inuse = 0; @@ -3665,16 +3669,18 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, addr = &pif->addr; addrlen = pif->addrlen; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - pnum = ub_random_max(outnet->rnd, pif->avail_total); - if(pnum < pif->inuse) { - /* port already open */ - port = pif->out[pnum]->number; - } else { - /* unused ports in start part of array */ - port = pif->avail_ports[pnum - pif->inuse]; + if(!shared_ports_fetch_random(outnet->shared_ports, + pif->shpif, outnet->rnd, 0, pif->inuse, + &port, &reused)) { + /* try again, perhaps another interface. */ + continue; + } + if(reused) { + log_assert(port < pif->inuse); + port = pif->out[port]->number; } #else - pnum = port = 0; + port = 0; #endif if(addr_is_ip6(to_addr, to_addrlen)) { struct sockaddr_in6 sa = *(struct sockaddr_in6*)addr; @@ -3689,6 +3695,14 @@ fd_for_dest(struct outside_network* outnet, struct sockaddr_storage* to_addr, (struct sockaddr*)addr, addrlen, 1, &inuse, &noproto, 0, 0, 0, NULL, 0, freebind, 0, dscp); } +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + if(!reused) { + /* Return the port to the pool, since the caller does + * not keep track of it, also have done fd, and bind. */ + shared_ports_return_port(outnet->shared_ports, + pif->shpif, port); + } +#endif if(fd != -1) { return fd; } @@ -3919,11 +3933,7 @@ if_get_mem(struct port_if* pif) { size_t s; int i; - s = sizeof(*pif) + -#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - sizeof(int)*pif->avail_total + -#endif - sizeof(struct port_comm*)*pif->maxout; + s = sizeof(*pif) + sizeof(struct port_comm*)*pif->maxout; for(i=0; iinuse; i++) s += sizeof(*pif->out[i]) + comm_point_get_mem(pif->out[i]->cp); @@ -4011,3 +4021,237 @@ serviced_get_mem(struct serviced_query* sq) return s; } +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Setup shared port interface */ +static int shared_ports_setup_if(struct shared_ports_if* shpif, char* str, + int* availports, int numavailports) +{ + shpif->avail_ports = (int*)memdup(availports, + (size_t)numavailports*sizeof(int)); + if(!shpif->avail_ports) + return 0; + shpif->avail_total = numavailports; + shpif->inuse = 0; + shpif->pfxlen = 0; + if(!ipstrtoaddr(str, UNBOUND_DNS_PORT, &shpif->addr, &shpif->addrlen) && + !netblockstrtoaddr(str, UNBOUND_DNS_PORT, &shpif->addr, + &shpif->addrlen, &shpif->pfxlen)) + return 0; + return 1; +} +#endif + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Allocate shared ports interfaces */ +static int shared_ports_alloc_ifs(struct shared_ports* shp, char** ifs, + int num_ifs, int do_ip4, int do_ip6, int* availports, + int numavailports) +{ +#ifndef INET6 + do_ip6 = 0; +#endif + calc_num46(ifs, num_ifs, do_ip4, do_ip6, + &shp->num_ip4, &shp->num_ip6); + if(shp->num_ip4 != 0) { + if(!(shp->ip4_ifs = (struct shared_ports_if*)calloc( + (size_t)shp->num_ip4, + sizeof(struct shared_ports_if)))) + return 0; + } + if(shp->num_ip6 != 0) { + if(!(shp->ip6_ifs = (struct shared_ports_if*)calloc( + (size_t)shp->num_ip6, + sizeof(struct shared_ports_if)))) + return 0; + } + if(num_ifs == 0) { + if(do_ip4 && !shared_ports_setup_if(&shp->ip4_ifs[0], + "0.0.0.0", availports, numavailports)) + return 0; + if(do_ip6 && !shared_ports_setup_if(&shp->ip6_ifs[0], + "::", availports, numavailports)) + return 0; + } else { + size_t done_4 = 0, done_6 = 0; + int i; + for(i=0; iip6_ifs[done_6], + ifs[i], availports, numavailports)) + return 0; + done_6++; + } + if(!str_is_ip6(ifs[i]) && do_ip4) { + if(!shared_ports_setup_if(&shp->ip4_ifs[done_4], + ifs[i], availports, numavailports)) + return 0; + done_4++; + } + } + } + return 1; +} +#endif + +struct shared_ports* shared_ports_create(char** ifs, int num_ifs, int do_ip4, + int do_ip6, int* availports, int numavailports) +{ + struct shared_ports* shp = calloc(1, sizeof(*shp)); + if(!shp) { + log_err("malloc failed"); + return NULL; + } + lock_basic_init(&shp->lock); + lock_protect(&shp->lock, shp, sizeof(*shp)); + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + /* Allocate interfaces */ + if(!shared_ports_alloc_ifs(shp, ifs, num_ifs, do_ip4, do_ip6, + availports, numavailports)) { + log_err("malloc failed"); + shared_ports_delete(shp); + return NULL; + } +#else + (void)ifs; (void)num_ifs; (void)do_ip4; (void)do_ip6; + (void)availports; (void)numavailports; +#endif + return shp; +} + +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION +/** Delete shared ports interface structure */ +static void shared_ports_if_delete(struct shared_ports_if* shpif) +{ + if(!shpif) + return; + free(shpif->avail_ports); +} +#endif + +void shared_ports_delete(struct shared_ports* shp) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int i; +#endif + if(!shp) + return; + lock_basic_destroy(&shp->lock); +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + for(i=0; inum_ip4; i++) { + shared_ports_if_delete(&shp->ip4_ifs[i]); + } + free(shp->ip4_ifs); + for(i=0; inum_ip6; i++) { + shared_ports_if_delete(&shp->ip6_ifs[i]); + } + free(shp->ip6_ifs); +#endif + free(shp); +} + +struct shared_ports_if* shared_ports_find_if(struct shared_ports* shp, + struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + struct shared_ports_if* ret, *ifs = NULL; + int i, num_ifs = 0; + lock_basic_lock(&shp->lock); + if(addr_is_ip6(addr, addrlen)) { + ifs = shp->ip6_ifs; + num_ifs = shp->num_ip6; + } else { + ifs = shp->ip4_ifs; + num_ifs = shp->num_ip4; + } + for(i=0; ilock); + return ret; + } + } + lock_basic_unlock(&shp->lock); + return NULL; +#else + (void)shp; (void)addr; (void)addrlen; (void)pfxlen; + return NULL; +#endif +} + +int shared_ports_fetch_random(struct shared_ports* shp, + struct shared_ports_if* shpif, struct ub_randstate* rnd, + int udp_connect, int reusenum, int* port, int* reused) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + int portno = 0, my_port = 0; + if(!shpif) + return 0; + lock_basic_lock(&shp->lock); + if(udp_connect) { + /* if we connect() we cannot reuse fds for a port. */ + if(shpif->inuse >= shpif->avail_total) { + lock_basic_unlock(&shp->lock); + return 0; + } + my_port = ub_random_max(rnd, + shpif->avail_total - shpif->inuse); + } else { + /* select from free ports and open ports on this thread. */ + if(shpif->inuse >= shpif->avail_total) { + lock_basic_unlock(&shp->lock); + if(reusenum == 0) { + return 0; + } + my_port = ub_random_max(rnd, reusenum); + *port = my_port; + *reused = 1; + return 1; + } + my_port = ub_random_max(rnd, shpif->avail_total - shpif->inuse + + reusenum); + if(my_port < reusenum) { + /* port already open */ + lock_basic_unlock(&shp->lock); + *port = my_port; + *reused = 1; + return 1; + } + my_port -= reusenum; + } + log_assert(shpif->inuse < shpif->avail_total); + log_assert(my_port >= 0 && my_port < shpif->avail_total); + portno = shpif->avail_ports[my_port]; + shpif->avail_ports[my_port] = + shpif->avail_ports[shpif->avail_total-shpif->inuse-1]; + shpif->inuse++; + lock_basic_unlock(&shp->lock); + *port = portno; + *reused = 0; + return 1; +#else + (void)shp; (void)shpif; (void)rnd; (void)udp_connect; + (void)reusenum; + *port = 0; + *reused = 0; + return 1; +#endif +} + +void shared_ports_return_port(struct shared_ports* shp, + struct shared_ports_if* shpif, int port) +{ +#ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION + if(!shpif) + return; + lock_basic_lock(&shp->lock); + log_assert(shpif->inuse > 0); + shpif->avail_ports[shpif->avail_total - shpif->inuse] = port; + shpif->inuse--; + lock_basic_unlock(&shp->lock); +#else + (void)shp; (void)shpif; (void)port; +#endif +} diff --git a/services/outside_network.h b/services/outside_network.h index e30ce92eb..573f3cb02 100644 --- a/services/outside_network.h +++ b/services/outside_network.h @@ -70,6 +70,8 @@ struct module_env; struct module_qstate; struct query_info; struct config_file; +struct shared_ports; +struct shared_ports_if; /** * Send queries to outside servers and wait for answers from servers. @@ -119,6 +121,9 @@ struct outside_network { int udp_connect; /** number of udp packets sent. */ size_t num_udp_outgoing; + /** the shared ports structure, with random ports numbers. + * This is a reference to the member in the daemon structure. */ + struct shared_ports* shared_ports; /** array of outgoing IP4 interfaces */ struct port_if* ip4_ifs; @@ -211,11 +216,8 @@ struct port_if { int pfxlen; #ifndef DISABLE_EXPLICIT_PORT_RANDOMISATION - /** the available ports array. These are unused. - * Only the first total-inuse part is filled. */ - int* avail_ports; - /** the total number of available ports (size of the array) */ - int avail_total; + /** the shared port numbers for this interface. */ + struct shared_ports_if* shpif; #endif /** array of the commpoints currently in use. @@ -245,6 +247,42 @@ struct port_comm { struct comm_point* cp; }; +/** + * Shared ports, the list of ports shared across threads + */ +struct shared_ports { + /** mutex on the ports */ + lock_basic_type lock; + /** array of IP4 interfaces */ + struct shared_ports_if* ip4_ifs; + /** number of outgoing IP4 interfaces */ + int num_ip4; + /** array of IP6 interfaces */ + struct shared_ports_if* ip6_ifs; + /** number of outgoing IP6 interfaces */ + int num_ip6; +}; + +/** + * Shared ports for an interface. + */ +struct shared_ports_if { + /** address ready to allocate new socket (except port no). */ + struct sockaddr_storage addr; + /** length of addr field */ + socklen_t addrlen; + /** if a netblock, the prefix */ + int pfxlen; + + /** the available ports array. These are unused. + * Only the first total-inuse part is filled. */ + int* avail_ports; + /** the total number of available ports (size of the array) */ + int avail_total; + /** the number in use. */ + int inuse; +}; + /** * Reuse TCP connection, still open can be used again. */ @@ -551,8 +589,6 @@ struct serviced_query { * @param infra: pointer to infra cached used for serviced queries. * @param rnd: stored to create random numbers for serviced queries. * @param use_caps_for_id: enable to use 0x20 bits to encode id randomness. - * @param availports: array of available ports. - * @param numavailports: number of available ports in array. * @param unwanted_threshold: when to take defensive action. * @param unwanted_action: the action to take. * @param unwanted_param: user parameter to action. @@ -567,17 +603,18 @@ struct serviced_query { * @param max_reuse_tcp_queries: max number of queries on a reuse connection. * @param tcp_reuse_timeout: timeout for REUSE entries in milliseconds. * @param tcp_auth_query_timeout: timeout in milliseconds for TCP queries to auth servers. + * @param shared_ports: the shared_ports structure. * @return: the new structure (with no pending answers) or NULL on error. */ struct outside_network* outside_network_create(struct comm_base* base, size_t bufsize, size_t num_ports, char** ifs, int num_ifs, int do_ip4, int do_ip6, size_t num_tcp, int dscp, struct infra_cache* infra, - struct ub_randstate* rnd, int use_caps_for_id, int* availports, - int numavailports, size_t unwanted_threshold, int tcp_mss, + struct ub_randstate* rnd, int use_caps_for_id, + size_t unwanted_threshold, int tcp_mss, void (*unwanted_action)(void*), void* unwanted_param, int do_udp, void* sslctx, int delayclose, int tls_use_sni, struct dt_env *dtenv, int udp_connect, int max_reuse_tcp_queries, int tcp_reuse_timeout, - int tcp_auth_query_timeout); + int tcp_auth_query_timeout, struct shared_ports* shared_ports); /** * Delete outside_network structure. @@ -819,6 +856,54 @@ struct comm_point* outnet_comm_point_for_http(struct outside_network* outnet, /** connect tcp connection to addr, 0 on failure */ int outnet_tcp_connect(int s, struct sockaddr_storage* addr, socklen_t addrlen); +/** + * Create new shared ports structure. + * @param ifs: interface names (or NULL for default interface). + * These interfaces must be able to access all authoritative servers. + * @param num_ifs: number of names in array ifs. + * @param do_ip4: service IP4. + * @param do_ip6: service IP6. + * @param availports: array of available ports. + * @param numavailports: number of available ports in array. + * @return new, or NULL on failure. + */ +struct shared_ports* shared_ports_create(char** ifs, int num_ifs, int do_ip4, + int do_ip6, int* availports, int numavailports); + +/** + * Delete shared ports structure. + * @param shp: shared ports structure. + */ +void shared_ports_delete(struct shared_ports* shp); + +/** Find interface in shared ports. */ +struct shared_ports_if* shared_ports_find_if(struct shared_ports* shp, + struct sockaddr_storage* addr, socklen_t addrlen, int pfxlen); + +/** + * Get a shared port from the list of random ports. + * @param shp: shared ports structure. + * @param shpif: the shared ports interface. + * @param rnd: used to make random numbers. + * @param udp_connect: set to true if no reuse is possible. + * @param reusenum: number of ports that can be reused (already open). + * @param port: the port number is returned. + * @param reused: if the port numer is reused, returned. + * @return false on failure. That can mean no more free ports to use. + */ +int shared_ports_fetch_random(struct shared_ports* shp, + struct shared_ports_if* shpif, struct ub_randstate* rnd, + int udp_connect, int reusenum, int* port, int* reused); + +/** + * Return a shared port to the list of random ports. + * @param shp: shared ports structure. + * @param shpif: the shared ports interface. + * @param port: port number to return to be used again. + */ +void shared_ports_return_port(struct shared_ports* shp, + struct shared_ports_if* shpif, int port); + /** callback for incoming udp answers from the network */ int outnet_udp_cb(struct comm_point* c, void* arg, int error, struct comm_reply *reply_info);