From 3f567e4949badb4d13e66f5688fc00fc10ab0a0a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Thu, 28 May 2020 15:29:19 +0200 Subject: [PATCH] REORG: include: split hathreads into haproxy/thread.h and haproxy/thread-t.h This splits the hathreads.h file into types+macros and functions. Given that most users of this file used to include it only to get the definition of THREAD_LOCAL and MAXTHREADS, the bare minimum was placed into thread-t.h (i.e. types and macros). All the thread management was left to haproxy/thread.h. It's worth noting the drop of the trailing "s" in the name, to remove the permanent confusion that arises between this one and the system implementation (no "s") and the makefile's option (no "s"). For consistency, src/hathreads.c was also renamed thread.c. A number of files were updated to only include thread-t which is the one they really needed. Some future improvements are possible like replacing empty inlined functions with macros for the thread-less case, as building at -O0 disables inlining and causes these ones to be emitted. But this really is cosmetic. --- MAINTAINERS | 2 +- Makefile | 2 +- include/common/memory.h | 2 +- include/common/regex.h | 2 +- include/common/time.h | 2 +- include/haproxy/list.h | 2 +- include/haproxy/thread-t.h | 171 ++++++++++ .../{common/hathreads.h => haproxy/thread.h} | 307 ++++++------------ include/proto/log.h | 2 +- include/proto/protocol.h | 2 +- include/proto/signal.h | 2 +- include/proto/task.h | 2 +- include/types/backend.h | 2 +- include/types/dict.h | 2 +- include/types/dns.h | 2 +- include/types/global.h | 2 +- include/types/listener.h | 2 +- include/types/log.h | 2 +- include/types/proxy.h | 2 +- include/types/server.h | 4 +- include/types/spoe.h | 2 +- include/types/ssl_sock.h | 2 +- include/types/vars.h | 2 +- src/51d.c | 2 +- src/activity.c | 2 +- src/auth.c | 2 +- src/cfgparse.c | 2 +- src/checks.c | 2 +- src/compression.c | 2 +- src/debug.c | 2 +- src/ev_epoll.c | 2 +- src/ev_evports.c | 2 +- src/ev_kqueue.c | 2 +- src/ev_poll.c | 2 +- src/ev_select.c | 2 +- src/flt_spoe.c | 2 +- src/haproxy.c | 2 +- src/hlua.c | 2 +- src/memory.c | 2 +- src/peers.c | 2 +- src/pipe.c | 2 +- src/queue.c | 2 +- src/ring.c | 2 +- src/stream.c | 2 +- src/{hathreads.c => thread.c} | 4 +- src/time.c | 2 +- src/wdt.c | 2 +- 47 files changed, 320 insertions(+), 252 deletions(-) create mode 100644 include/haproxy/thread-t.h rename include/{common/hathreads.h => haproxy/thread.h} (73%) rename src/{hathreads.c => thread.c} (99%) diff --git a/MAINTAINERS b/MAINTAINERS index 95b017521..f17c6480a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -77,7 +77,7 @@ Files: src/flt_spoe.c, include/*/spoe.h, contrib/spoa_example, doc/SPOE.txt Multi-threading Maintainer: Christopher Faulet Maintainer: Emeric Brun -Files: src/hathreads.c, include/*/hathreads.h +Files: src/thread.c, include/*/thread.h Note: every change around the locking or synchronization point will require approval from one of these maintainers. Problems which only appear when nbthread is greater than 1 and which disappear otherwise are also diff --git a/Makefile b/Makefile index ed736119b..f7155d02a 100644 --- a/Makefile +++ b/Makefile @@ -805,7 +805,7 @@ OBJS = src/mux_h2.o src/stream.o src/mux_fcgi.o src/cfgparse-listen.o \ src/htx.o src/memory.o src/applet.o src/channel.o src/signal.o \ src/lb_fwrr.o src/ev_select.o src/sink.o src/http_conv.o \ src/proto_sockpair.o src/mworker-prog.o src/activity.o src/lb_fwlc.o \ - src/http.o src/lb_fas.o src/uri_auth.o src/hathreads.o src/regex.o \ + src/http.o src/lb_fas.o src/uri_auth.o src/thread.o src/regex.o \ src/auth.o src/buffer.o src/compression.o src/proto_udp.o src/lb_map.o \ src/chunk.o src/wdt.o src/hpack-dec.o src/action.o src/xxhash.o \ src/pipe.o src/shctx.o src/hpack-tbl.o src/http_acl.o src/sha1.o \ diff --git a/include/common/memory.h b/include/common/memory.h index d80b4868c..3858ae4eb 100644 --- a/include/common/memory.h +++ b/include/common/memory.h @@ -30,7 +30,7 @@ #include #include -#include +#include /* On architectures supporting threads and double-word CAS, we can implement * lock-less memory pools. This isn't supported for debugging modes however. diff --git a/include/common/regex.h b/include/common/regex.h index d106b8690..79e54ff12 100644 --- a/include/common/regex.h +++ b/include/common/regex.h @@ -26,7 +26,7 @@ #include #include -#include +#include #ifdef USE_PCRE #include diff --git a/include/common/time.h b/include/common/time.h index 2a4ec0afc..d6863d42c 100644 --- a/include/common/time.h +++ b/include/common/time.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include /* eternity when exprimed in timeval */ diff --git a/include/haproxy/list.h b/include/haproxy/list.h index 0281aa1a4..a35502c67 100644 --- a/include/haproxy/list.h +++ b/include/haproxy/list.h @@ -24,7 +24,7 @@ #include #include -#include +#include /* First undefine some macros which happen to also be defined on OpenBSD, * in sys/queue.h, used by sys/event.h diff --git a/include/haproxy/thread-t.h b/include/haproxy/thread-t.h new file mode 100644 index 000000000..d93324492 --- /dev/null +++ b/include/haproxy/thread-t.h @@ -0,0 +1,171 @@ +/* + * include/haproxy/thread-t.h + * Definitions and types for thread support. + * + * Copyright (C) 2017 Christopher Faulet - cfaulet@haproxy.com + * Copyright (C) 2020 Willy Tarreau - w@1wt.eu + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, version 2.1 + * exclusively. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _HAPROXY_THREAD_T_H +#define _HAPROXY_THREAD_T_H + +#ifdef USE_THREAD +#include +#endif +#include + + +/* Note: this file mainly contains 3 sections: + * - one used solely when USE_THREAD is *not* set + * - one used solely when USE_THREAD is set + * - a common one. + */ + +#ifndef USE_THREAD + +/********************** THREADS DISABLED ************************/ + +#define THREAD_LOCAL /* empty */ +#define MAX_THREADS 1 +#define MAX_THREADS_MASK 1 + +/* These macros allow to make some struct fields or local variables optional */ +#define __decl_hathreads(decl) +#define __decl_spinlock(lock) +#define __decl_aligned_spinlock(lock) +#define __decl_rwlock(lock) +#define __decl_aligned_rwlock(lock) + +#else /* !USE_THREAD */ + +/********************** THREADS ENABLED ************************/ + +#define THREAD_LOCAL __thread + +#ifndef MAX_THREADS +#define MAX_THREADS LONGBITS +#endif + +#define MAX_THREADS_MASK (~0UL >> (LONGBITS - MAX_THREADS)) + +#define __decl_hathreads(decl) decl + +/* declare a self-initializing spinlock */ +#define __decl_spinlock(lock) \ + HA_SPINLOCK_T (lock); \ + INITCALL1(STG_LOCK, ha_spin_init, &(lock)) + +/* declare a self-initializing spinlock, aligned on a cache line */ +#define __decl_aligned_spinlock(lock) \ + HA_SPINLOCK_T (lock) __attribute__((aligned(64))); \ + INITCALL1(STG_LOCK, ha_spin_init, &(lock)) + +/* declare a self-initializing rwlock */ +#define __decl_rwlock(lock) \ + HA_RWLOCK_T (lock); \ + INITCALL1(STG_LOCK, ha_rwlock_init, &(lock)) + +/* declare a self-initializing rwlock, aligned on a cache line */ +#define __decl_aligned_rwlock(lock) \ + HA_RWLOCK_T (lock) __attribute__((aligned(64))); \ + INITCALL1(STG_LOCK, ha_rwlock_init, &(lock)) + +#endif /* USE_THREAD */ + + +/*** Common parts below ***/ + +/* thread info flags, for ha_thread_info[].flags */ +#define TI_FL_STUCK 0x00000001 + +/* This structure describes all the per-thread info we need. When threads are + * disabled, it contains the same info for the single running thread (except + * the pthread identifier which does not exist). + */ +struct thread_info { + __decl_hathreads(pthread_t pthread); + clockid_t clock_id; + timer_t wd_timer; /* valid timer or TIMER_INVALID if not set */ + uint64_t prev_cpu_time; /* previous per thread CPU time */ + uint64_t prev_mono_time; /* previous system wide monotonic time */ + unsigned int idle_pct; /* idle to total ratio over last sample (percent) */ + unsigned int flags; /* thread info flags, TI_FL_* */ + /* pad to cache line (64B) */ + char __pad[0]; /* unused except to check remaining room */ + char __end[0] __attribute__((aligned(64))); +}; + +/* storage types used by spinlocks and RW locks */ +#define __HA_SPINLOCK_T unsigned long +#define __HA_RWLOCK_T unsigned long + + +/* When thread debugging is enabled, we remap HA_SPINLOCK_T and HA_RWLOCK_T to + * complex structures which embed debugging info. + */ +#if !defined(DEBUG_THREAD) && !defined(DEBUG_FULL) + +#define HA_SPINLOCK_T __HA_SPINLOCK_T +#define HA_RWLOCK_T __HA_RWLOCK_T + +#else /* !DEBUG_THREAD */ + +#define HA_SPINLOCK_T struct ha_spinlock +#define HA_RWLOCK_T struct ha_rwlock + +/* Debugging information that is only used when thread debugging is enabled */ + +struct lock_stat { + uint64_t nsec_wait_for_write; + uint64_t nsec_wait_for_read; + uint64_t num_write_locked; + uint64_t num_write_unlocked; + uint64_t num_read_locked; + uint64_t num_read_unlocked; +}; + +struct ha_spinlock { + __HA_SPINLOCK_T lock; + struct { + unsigned long owner; /* a bit is set to 1 << tid for the lock owner */ + unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */ + struct { + const char *function; + const char *file; + int line; + } last_location; /* location of the last owner */ + } info; +}; + +struct ha_rwlock { + __HA_RWLOCK_T lock; + struct { + unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */ + unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */ + unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */ + unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */ + struct { + const char *function; + const char *file; + int line; + } last_location; /* location of the last write owner */ + } info; +}; + +#endif /* DEBUG_THREAD */ + +#endif /* _HAPROXY_THREAD_T_H */ diff --git a/include/common/hathreads.h b/include/haproxy/thread.h similarity index 73% rename from include/common/hathreads.h rename to include/haproxy/thread.h index 6146998f0..1afd95848 100644 --- a/include/common/hathreads.h +++ b/include/haproxy/thread.h @@ -1,8 +1,9 @@ /* - * include/common/hathreads.h - * definitions, macros and inline functions about threads. + * include/haproxy/thread.h + * definitions, macros and inline functions used by threads. * - * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com + * Copyright (C) 2017 Christopher Faulet - cfaulet@haproxy.com + * Copyright (C) 2020 Willy Tarreau - w@1wt.eu * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -19,79 +20,66 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef _COMMON_HATHREADS_H -#define _COMMON_HATHREADS_H +#ifndef _HAPROXY_THREAD_H +#define _HAPROXY_THREAD_H #include #include #ifdef _POSIX_PRIORITY_SCHEDULING #include #endif + #include #include +#include -/* Note about all_threads_mask : - * - this variable is comprised between 1 and LONGBITS. - * - with threads support disabled, this symbol is defined as constant 1UL. - * - with threads enabled, it contains the mask of enabled threads. Thus if - * only one thread is enabled, it equals 1. +/* Note: this file mainly contains 5 sections: + * - a small common part, which also corresponds to the common API + * - one used solely when USE_THREAD is *not* set + * - one used solely when USE_THREAD is set + * - one used solely when USE_THREAD is set WITHOUT debugging + * - one used solely when USE_THREAD is set WITH debugging + * */ -/* thread info flags, for ha_thread_info[].flags */ -#define TI_FL_STUCK 0x00000001 + +/* Generic exports */ +int parse_nbthread(const char *arg, char **err); +int thread_get_default_count(); +extern int thread_cpus_enabled_at_boot; +extern struct thread_info ha_thread_info[MAX_THREADS]; +extern THREAD_LOCAL struct thread_info *ti; /* thread_info for the current thread */ #ifndef USE_THREAD -#define THREAD_LOCAL /* empty */ -#define MAX_THREADS 1 -#define MAX_THREADS_MASK 1 +/********************** THREADS DISABLED ************************/ /* Only way found to replace variables with constants that are optimized away * at build time. */ enum { all_threads_mask = 1UL }; enum { threads_harmless_mask = 0 }; -enum { threads_want_rdv_mask = 0 }; enum { threads_sync_mask = 0 }; +enum { threads_want_rdv_mask = 0 }; enum { tid_bit = 1UL }; enum { tid = 0 }; -extern struct thread_info { - clockid_t clock_id; - timer_t wd_timer; /* valid timer or TIMER_INVALID if not set */ - uint64_t prev_cpu_time; /* previous per thread CPU time */ - uint64_t prev_mono_time; /* previous system wide monotonic time */ - unsigned int idle_pct; /* idle to total ratio over last sample (percent) */ - unsigned int flags; /* thread info flags, TI_FL_* */ - /* pad to cache line (64B) */ - char __pad[0]; /* unused except to check remaining room */ - char __end[0] __attribute__((aligned(64))); -} ha_thread_info[MAX_THREADS]; +#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0) +#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0) +#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0) +#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; }) +#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0) -extern THREAD_LOCAL struct thread_info *ti; /* thread_info for the current thread */ - -#define __decl_hathreads(decl) -#define __decl_spinlock(lock) -#define __decl_aligned_spinlock(lock) -#define __decl_rwlock(lock) -#define __decl_aligned_rwlock(lock) - -#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0) -#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0) -#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0) -#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; }) -#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0) - -#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0) -#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0) -#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0) #define HA_RWLOCK_TRYWRLOCK(lbl, l) ({ 0; }) -#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0) -#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0) #define HA_RWLOCK_TRYRDLOCK(lbl, l) ({ 0; }) -#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0) +#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0) #define ha_sigmask(how, set, oldset) sigprocmask(how, set, oldset) @@ -149,71 +137,30 @@ static inline unsigned long thread_isolated() return 1; } -#else /* USE_THREAD */ +#else /* !USE_THREAD */ + +/********************** THREADS ENABLED ************************/ #include #include #include -#include #include -#define THREAD_LOCAL __thread - -#ifndef MAX_THREADS -#define MAX_THREADS LONGBITS -#endif - -#define MAX_THREADS_MASK (~0UL >> (LONGBITS - MAX_THREADS)) - -#define __decl_hathreads(decl) decl - -/* declare a self-initializing spinlock */ -#define __decl_spinlock(lock) \ - HA_SPINLOCK_T (lock); \ - INITCALL1(STG_LOCK, ha_spin_init, &(lock)) - -/* declare a self-initializing spinlock, aligned on a cache line */ -#define __decl_aligned_spinlock(lock) \ - HA_SPINLOCK_T (lock) __attribute__((aligned(64))); \ - INITCALL1(STG_LOCK, ha_spin_init, &(lock)) - -/* declare a self-initializing rwlock */ -#define __decl_rwlock(lock) \ - HA_RWLOCK_T (lock); \ - INITCALL1(STG_LOCK, ha_rwlock_init, &(lock)) - -/* declare a self-initializing rwlock, aligned on a cache line */ -#define __decl_aligned_rwlock(lock) \ - HA_RWLOCK_T (lock) __attribute__((aligned(64))); \ - INITCALL1(STG_LOCK, ha_rwlock_init, &(lock)) - void thread_harmless_till_end(); void thread_isolate(); void thread_release(); void thread_sync_release(); void ha_tkill(unsigned int thr, int sig); void ha_tkillall(int sig); +void ha_spin_init(HA_SPINLOCK_T *l); +void ha_rwlock_init(HA_RWLOCK_T *l); -extern struct thread_info { - pthread_t pthread; - clockid_t clock_id; - timer_t wd_timer; /* valid timer or TIMER_INVALID if not set */ - uint64_t prev_cpu_time; /* previous per thread CPU time */ - uint64_t prev_mono_time; /* previous system wide monotonic time */ - unsigned int idle_pct; /* idle to total ratio over last sample (percent) */ - unsigned int flags; /* thread info flags, TI_FL_* */ - /* pad to cache line (64B) */ - char __pad[0]; /* unused except to check remaining room */ - char __end[0] __attribute__((aligned(64))); -} ha_thread_info[MAX_THREADS]; - -extern THREAD_LOCAL unsigned int tid; /* The thread id */ -extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */ -extern THREAD_LOCAL struct thread_info *ti; /* thread_info for the current thread */ extern volatile unsigned long all_threads_mask; -extern volatile unsigned long threads_want_rdv_mask; extern volatile unsigned long threads_harmless_mask; extern volatile unsigned long threads_sync_mask; +extern volatile unsigned long threads_want_rdv_mask; +extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */ +extern THREAD_LOCAL unsigned int tid; /* The thread id */ /* explanation for threads_want_rdv_mask, threads_harmless_mask, and * threads_sync_mask : @@ -324,9 +271,63 @@ static inline unsigned long thread_isolated() } -#if defined(DEBUG_THREAD) || defined(DEBUG_FULL) +#if !defined(DEBUG_THREAD) && !defined(DEBUG_FULL) -/* WARNING!!! if you update this enum, please also keep lock_label() up to date below */ +/* Thread debugging is DISABLED, these are the regular locking functions */ + +#define HA_SPIN_INIT(l) ({ (*l) = 0; }) +#define HA_SPIN_DESTROY(l) ({ (*l) = 0; }) +#define HA_SPIN_LOCK(lbl, l) pl_take_s(l) +#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l) +#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l) + +#define HA_RWLOCK_INIT(l) ({ (*l) = 0; }) +#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; }) +#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l) +#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l) +#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l) +#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l) +#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l) +#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l) + +#else /* !defined(DEBUG_THREAD) && !defined(DEBUG_FULL) */ + +/* Thread debugging is ENABLED, these are the instrumented functions */ + +#define __SPIN_INIT(l) ({ (*l) = 0; }) +#define __SPIN_DESTROY(l) ({ (*l) = 0; }) +#define __SPIN_LOCK(l) pl_take_s(l) +#define __SPIN_TRYLOCK(l) !pl_try_s(l) +#define __SPIN_UNLOCK(l) pl_drop_s(l) + +#define __RWLOCK_INIT(l) ({ (*l) = 0; }) +#define __RWLOCK_DESTROY(l) ({ (*l) = 0; }) +#define __RWLOCK_WRLOCK(l) pl_take_w(l) +#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l) +#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l) +#define __RWLOCK_RDLOCK(l) pl_take_r(l) +#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l) +#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l) + +#define HA_SPIN_INIT(l) __spin_init(l) +#define HA_SPIN_DESTROY(l) __spin_destroy(l) + +#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__) +#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__) +#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__) + +#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l)) +#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l)) +#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__) +#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__) +#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__) +#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l) +#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l) +#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l) + +/* WARNING!!! if you update this enum, please also keep lock_label() up to date + * below. + */ enum lock_label { TASK_RQ_LOCK, TASK_WQ_LOCK, @@ -365,84 +366,9 @@ enum lock_label { OTHER_LOCK, LOCK_LABELS }; -struct lock_stat { - uint64_t nsec_wait_for_write; - uint64_t nsec_wait_for_read; - uint64_t num_write_locked; - uint64_t num_write_unlocked; - uint64_t num_read_locked; - uint64_t num_read_unlocked; -}; extern struct lock_stat lock_stats[LOCK_LABELS]; -#define __HA_SPINLOCK_T unsigned long - -#define __SPIN_INIT(l) ({ (*l) = 0; }) -#define __SPIN_DESTROY(l) ({ (*l) = 0; }) -#define __SPIN_LOCK(l) pl_take_s(l) -#define __SPIN_TRYLOCK(l) !pl_try_s(l) -#define __SPIN_UNLOCK(l) pl_drop_s(l) - -#define __HA_RWLOCK_T unsigned long - -#define __RWLOCK_INIT(l) ({ (*l) = 0; }) -#define __RWLOCK_DESTROY(l) ({ (*l) = 0; }) -#define __RWLOCK_WRLOCK(l) pl_take_w(l) -#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l) -#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l) -#define __RWLOCK_RDLOCK(l) pl_take_r(l) -#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l) -#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l) - -#define HA_SPINLOCK_T struct ha_spinlock - -#define HA_SPIN_INIT(l) __spin_init(l) -#define HA_SPIN_DESTROY(l) __spin_destroy(l) - -#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__) -#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__) -#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__) - -#define HA_RWLOCK_T struct ha_rwlock - -#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l)) -#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l)) -#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__) -#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__) -#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__) -#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l) -#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l) -#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l) - -struct ha_spinlock { - __HA_SPINLOCK_T lock; - struct { - unsigned long owner; /* a bit is set to 1 << tid for the lock owner */ - unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */ - struct { - const char *function; - const char *file; - int line; - } last_location; /* location of the last owner */ - } info; -}; - -struct ha_rwlock { - __HA_RWLOCK_T lock; - struct { - unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */ - unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */ - unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */ - unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */ - struct { - const char *function; - const char *file; - int line; - } last_location; /* location of the last write owner */ - } info; -}; - static inline const char *lock_label(enum lock_label label) { switch (label) { @@ -519,7 +445,8 @@ static inline void show_lock_stats() /* Following functions are used to collect some stats about locks. We wrap * pthread functions to known how much time we wait in a lock. */ -static uint64_t nsec_now(void) { +static uint64_t nsec_now(void) +{ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -772,38 +699,8 @@ static inline void __spin_unlock(enum lock_label lbl, struct ha_spinlock *l, HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1); } -#else /* DEBUG_THREAD */ - -#define HA_SPINLOCK_T unsigned long - -#define HA_SPIN_INIT(l) ({ (*l) = 0; }) -#define HA_SPIN_DESTROY(l) ({ (*l) = 0; }) -#define HA_SPIN_LOCK(lbl, l) pl_take_s(l) -#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l) -#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l) - -#define HA_RWLOCK_T unsigned long - -#define HA_RWLOCK_INIT(l) ({ (*l) = 0; }) -#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; }) -#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l) -#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l) -#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l) -#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l) -#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l) -#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l) - #endif /* DEBUG_THREAD */ - -void ha_spin_init(HA_SPINLOCK_T *l); -void ha_rwlock_init(HA_RWLOCK_T *l); - #endif /* USE_THREAD */ -extern int thread_cpus_enabled_at_boot; - -int parse_nbthread(const char *arg, char **err); -int thread_get_default_count(); - -#endif /* _COMMON_HATHREADS_H */ +#endif /* _HAPROXY_THREAD_H */ diff --git a/include/proto/log.h b/include/proto/log.h index a84d61263..ef792c81a 100644 --- a/include/proto/log.h +++ b/include/proto/log.h @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include diff --git a/include/proto/protocol.h b/include/proto/protocol.h index f25f77f0a..ee9e7e0c4 100644 --- a/include/proto/protocol.h +++ b/include/proto/protocol.h @@ -23,7 +23,7 @@ #define _PROTO_PROTOCOL_H #include -#include +#include #include extern struct protocol *__protocol_by_family[AF_CUST_MAX]; diff --git a/include/proto/signal.h b/include/proto/signal.h index ef2892a1c..558fbc20a 100644 --- a/include/proto/signal.h +++ b/include/proto/signal.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include diff --git a/include/proto/task.h b/include/proto/task.h index 944ada50a..668cee3f4 100644 --- a/include/proto/task.h +++ b/include/proto/task.h @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/types/backend.h b/include/types/backend.h index 3150ecf1b..679805879 100644 --- a/include/types/backend.h +++ b/include/types/backend.h @@ -23,7 +23,7 @@ #define _TYPES_BACKEND_H #include -#include +#include #include #include diff --git a/include/types/dict.h b/include/types/dict.h index 30953f6ce..980781692 100644 --- a/include/types/dict.h +++ b/include/types/dict.h @@ -1,7 +1,7 @@ #ifndef _TYPES_DICT_H #define _TYPES_DICT_H -#include +#include #include struct dict_entry { diff --git a/include/types/dns.h b/include/types/dns.h index 84a82190a..0a8bf72db 100644 --- a/include/types/dns.h +++ b/include/types/dns.h @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/include/types/global.h b/include/types/global.h index cb9ff4b24..b0beeb1b6 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/include/types/listener.h b/include/types/listener.h index 431420940..50e208ed9 100644 --- a/include/types/listener.h +++ b/include/types/listener.h @@ -31,7 +31,7 @@ #endif #include -#include +#include #include #include diff --git a/include/types/log.h b/include/types/log.h index 301c2bf33..ae7ce0b2f 100644 --- a/include/types/log.h +++ b/include/types/log.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/types/proxy.h b/include/types/proxy.h index f7bdd30ff..aa1ad3583 100644 --- a/include/types/proxy.h +++ b/include/types/proxy.h @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/types/server.h b/include/types/server.h index 1737f8069..0bc5319df 100644 --- a/include/types/server.h +++ b/include/types/server.h @@ -26,8 +26,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/include/types/spoe.h b/include/types/spoe.h index 85f8b430d..562e9f21f 100644 --- a/include/types/spoe.h +++ b/include/types/spoe.h @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include diff --git a/include/types/ssl_sock.h b/include/types/ssl_sock.h index d62e4ddf9..c8a068173 100644 --- a/include/types/ssl_sock.h +++ b/include/types/ssl_sock.h @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include diff --git a/include/types/vars.h b/include/types/vars.h index 91fcf0727..e7d71d34d 100644 --- a/include/types/vars.h +++ b/include/types/vars.h @@ -2,7 +2,7 @@ #define _TYPES_VARS_H #include -#include +#include #include diff --git a/src/51d.c b/src/51d.c index ceef3559a..6d3e86f10 100644 --- a/src/51d.c +++ b/src/51d.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/activity.c b/src/activity.c index 6da8f40e9..e9a66fce1 100644 --- a/src/activity.c +++ b/src/activity.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/auth.c b/src/auth.c index 6570ecafe..4decf84c9 100644 --- a/src/auth.c +++ b/src/auth.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/cfgparse.c b/src/cfgparse.c index da76aa99e..f4de78a9b 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/checks.c b/src/checks.c index 5dbbecae4..92f1e4a20 100644 --- a/src/checks.c +++ b/src/checks.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/compression.c b/src/compression.c index 5797a0646..a63eeef14 100644 --- a/src/compression.c +++ b/src/compression.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include diff --git a/src/debug.c b/src/debug.c index cf77c5b71..4017232ce 100644 --- a/src/debug.c +++ b/src/debug.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/ev_epoll.c b/src/ev_epoll.c index 91670afd3..132d4876a 100644 --- a/src/ev_epoll.c +++ b/src/ev_epoll.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/ev_evports.c b/src/ev_evports.c index c0005e198..e440f1544 100644 --- a/src/ev_evports.c +++ b/src/ev_evports.c @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include diff --git a/src/ev_kqueue.c b/src/ev_kqueue.c index e9821bdf4..5b14d51fd 100644 --- a/src/ev_kqueue.c +++ b/src/ev_kqueue.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include diff --git a/src/ev_poll.c b/src/ev_poll.c index 0e6dc9445..c3c52f0db 100644 --- a/src/ev_poll.c +++ b/src/ev_poll.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include diff --git a/src/ev_select.c b/src/ev_select.c index 9dd93b887..3b02bed40 100644 --- a/src/ev_select.c +++ b/src/ev_select.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include diff --git a/src/flt_spoe.c b/src/flt_spoe.c index 9bd6ab10a..7021e556d 100644 --- a/src/flt_spoe.c +++ b/src/flt_spoe.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/src/haproxy.c b/src/haproxy.c index a1dff0c38..b1853cf1b 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -95,7 +95,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/hlua.c b/src/hlua.c index 15c34cd32..84fcf8b03 100644 --- a/src/hlua.c +++ b/src/hlua.c @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/src/memory.c b/src/memory.c index 2caf13d9c..0bca71278 100644 --- a/src/memory.c +++ b/src/memory.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/peers.c b/src/peers.c index e53076081..6d5a5de61 100644 --- a/src/peers.c +++ b/src/peers.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/pipe.c b/src/pipe.c index b08882087..8c828214a 100644 --- a/src/pipe.c +++ b/src/pipe.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/src/queue.c b/src/queue.c index 3c5007687..f2407ed67 100644 --- a/src/queue.c +++ b/src/queue.c @@ -72,7 +72,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/ring.c b/src/ring.c index d10ebd75f..ee72bb157 100644 --- a/src/ring.c +++ b/src/ring.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/stream.c b/src/stream.c index 0c030379c..7b5620f97 100644 --- a/src/stream.c +++ b/src/stream.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/hathreads.c b/src/thread.c similarity index 99% rename from src/hathreads.c rename to src/thread.c index 14c565897..47c5ab9dd 100644 --- a/src/hathreads.c +++ b/src/thread.c @@ -24,7 +24,7 @@ #endif #include -#include +#include #include #include #include @@ -190,7 +190,7 @@ static int thread_cpus_enabled() } __attribute__((constructor)) -static void __hathreads_init(void) +static void __thread_init(void) { char *ptr = NULL; diff --git a/src/time.c b/src/time.c index 5c8ed4be2..86c246987 100644 --- a/src/time.c +++ b/src/time.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include THREAD_LOCAL unsigned int ms_left_scaled; /* milliseconds left for current second (0..2^32-1) */ THREAD_LOCAL unsigned int now_ms; /* internal date in milliseconds (may wrap) */ diff --git a/src/wdt.c b/src/wdt.c index 2d03b4a1f..86de28640 100644 --- a/src/wdt.c +++ b/src/wdt.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include