diff --git a/Makefile b/Makefile index 10be7cc39..e2c4d17a1 100644 --- a/Makefile +++ b/Makefile @@ -737,7 +737,13 @@ ifneq ($(USE_51DEGREES),) OPTIONS_OBJS += $(51DEGREES_LIB)/../cityhash/city.o OPTIONS_OBJS += $(51DEGREES_LIB)/51Degrees.o OPTIONS_OBJS += src/51d.o -OPTIONS_CFLAGS += -DUSE_51DEGREES -DFIFTYONEDEGREES_NO_THREADING $(if $(51DEGREES_INC),-I$(51DEGREES_INC)) +OPTIONS_CFLAGS += -DUSE_51DEGREES $(if $(51DEGREES_INC),-I$(51DEGREES_INC)) +ifeq ($(USE_THREAD),) +OPTIONS_CFLAGS += -DFIFTYONEDEGREES_NO_THREADING +else +OPTIONS_OBJS += $(51DEGREES_LIB)/../threading.o +endif + BUILD_OPTIONS += $(call ignore_implicit,USE_51DEGREES) OPTIONS_LDFLAGS += $(if $(51DEGREES_LIB),-L$(51DEGREES_LIB)) -lm endif diff --git a/include/common/hathreads.h b/include/common/hathreads.h index 6f788769f..e4603a5f9 100644 --- a/include/common/hathreads.h +++ b/include/common/hathreads.h @@ -416,6 +416,7 @@ enum lock_label { START_LOCK, TLSKEYS_REF_LOCK, AUTH_LOCK, + OTHER_LOCK, LOCK_LABELS }; struct lock_stat { @@ -532,6 +533,7 @@ static inline const char *lock_label(enum lock_label label) case START_LOCK: return "START"; case TLSKEYS_REF_LOCK: return "TLSKEYS_REF"; case AUTH_LOCK: return "AUTH"; + case OTHER_LOCK: return "OTHER"; case LOCK_LABELS: break; /* keep compiler happy */ }; /* only way to come here is consecutive to an internal bug */ diff --git a/src/51d.c b/src/51d.c index a09d13968..701709553 100644 --- a/src/51d.c +++ b/src/51d.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,8 @@ struct _51d_property_names { #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED static struct lru64_head *_51d_lru_tree = NULL; static unsigned long long _51d_lru_seed; + +__decl_spinlock(_51d_lru_lock); #endif static struct { @@ -37,7 +40,9 @@ static struct { #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED int32_t *header_offsets; /* offsets to the HTTP header name string. */ +#ifdef FIFTYONEDEGREES_NO_THREADING fiftyoneDegreesDeviceOffsets device_offsets; /* Memory used for device offsets. */ +#endif #endif int cache_size; } global_51degrees = { @@ -201,7 +206,9 @@ static void _51d_insert_cache_entry(struct sample *smp, struct lru64 *lru, void* memcpy(cache_entry->area, smp->data.u.str.area, smp->data.u.str.data); cache_entry->area[smp->data.u.str.data] = 0; cache_entry->data = smp->data.u.str.data; + HA_SPIN_LOCK(OTHER_LOCK, &_51d_lru_lock); lru64_commit(lru, cache_entry, domain, 0, _51d_lru_free); + HA_SPIN_UNLOCK(OTHER_LOCK, &_51d_lru_lock); } /* Retrieves the data from the cache and sets the sample data to this string. @@ -250,13 +257,19 @@ static void _51d_set_headers(struct sample *smp, fiftyoneDegreesWorkset *ws) #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED -static void _51d_set_device_offsets(struct sample *smp) +static void _51d_init_device_offsets(fiftyoneDegreesDeviceOffsets *offsets) { + int i; + for (i = 0; i < global_51degrees.data_set.uniqueHttpHeaders.count; i++) { + offsets->firstOffset[i].userAgent = NULL; + } +} + +static void _51d_set_device_offsets(struct sample *smp, fiftyoneDegreesDeviceOffsets *offsets) { struct hdr_idx *idx; struct hdr_ctx ctx; const struct http_msg *msg; int index; - fiftyoneDegreesDeviceOffsets *offsets = &global_51degrees.device_offsets; idx = &smp->strm->txn->hdr_idx; msg = &smp->strm->txn->req; @@ -306,13 +319,11 @@ static void _51d_process_match(const struct arg *args, struct sample *smp, fifty char *methodName; #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED -static void _51d_process_match(const struct arg *args, struct sample *smp) +static void _51d_process_match(const struct arg *args, struct sample *smp, fiftyoneDegreesDeviceOffsets *offsets) { char valuesBuffer[1024]; const char **requiredProperties = fiftyoneDegreesGetRequiredPropertiesNames(&global_51degrees.data_set); int requiredPropertiesCount = fiftyoneDegreesGetRequiredPropertiesCount(&global_51degrees.data_set); - fiftyoneDegreesDeviceOffsets *deviceOffsets = &global_51degrees.device_offsets; - #endif char no_data[] = "NoData"; /* response when no data could be found */ @@ -362,7 +373,7 @@ static void _51d_process_match(const struct arg *args, struct sample *smp) for (j = 0; j < requiredPropertiesCount; j++) { property_name = requiredProperties[j]; if (strcmp(property_name, args[i].data.str.area) == 0 && - fiftyoneDegreesGetValueFromOffsets(&global_51degrees.data_set, deviceOffsets, j, valuesBuffer, 1024) > 0) { + fiftyoneDegreesGetValueFromOffsets(&global_51degrees.data_set, offsets, j, valuesBuffer, 1024) > 0) { found = 1; chunk_appendf(temp, "%s", valuesBuffer); break; @@ -392,6 +403,9 @@ static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw fiftyoneDegreesWorkset* ws; /* workset for detection */ struct lru64 *lru = NULL; #endif +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED + fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */ +#endif /* Needed to ensure that the HTTP message has been fully received when * used with TCP operation. Not required for HTTP operation. @@ -421,13 +435,18 @@ static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw /* Check the cache to see if there's results for these headers already. */ if (_51d_lru_tree) { + HA_SPIN_LOCK(OTHER_LOCK, &_51d_lru_lock); + lru = lru64_get(_51d_req_hash(args, ws), _51d_lru_tree, (void*)args, 0); + if (lru && lru->domain) { fiftyoneDegreesWorksetPoolRelease(global_51degrees.pool, ws); _51d_retrieve_cache_entry(smp, lru); + HA_SPIN_UNLOCK(OTHER_LOCK, &_51d_lru_lock); return 1; } + HA_SPIN_UNLOCK(OTHER_LOCK, &_51d_lru_lock); } fiftyoneDegreesMatchForHttpHeaders(ws); @@ -437,12 +456,22 @@ static int _51d_fetch(const struct arg *args, struct sample *smp, const char *kw #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +#ifndef FIFTYONEDEGREES_NO_THREADING + offsets = fiftyoneDegreesCreateDeviceOffsets(&global_51degrees.data_set); + _51d_init_device_offsets(offsets); +#else + offsets = &global_51degrees.device_offsets; +#endif /* Trie is very fast so all the headers can be passed in and the result * returned faster than the hashing algorithm process. */ - _51d_set_device_offsets(smp); - _51d_process_match(args, smp); + _51d_set_device_offsets(smp, offsets); + _51d_process_match(args, smp, offsets); + +#ifndef FIFTYONEDEGREES_NO_THREADING + fiftyoneDegreesFreeDeviceOffsets(offsets); +#endif #endif @@ -461,7 +490,9 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) fiftyoneDegreesWorkset* ws; /* workset for detection */ struct lru64 *lru = NULL; #endif - +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED + fiftyoneDegreesDeviceOffsets *offsets; /* Offsets for detection */ +#endif /* Flags the sample to show it uses constant memory*/ smp->flags |= SMP_F_CONST; @@ -471,12 +502,15 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) if (_51d_lru_tree) { unsigned long long seed = _51d_lru_seed ^ (long)args; + HA_SPIN_LOCK(OTHER_LOCK, &_51d_lru_lock); lru = lru64_get(XXH64(smp->data.u.str.area, smp->data.u.str.data, seed), _51d_lru_tree, (void*)args, 0); if (lru && lru->domain) { _51d_retrieve_cache_entry(smp, lru); + HA_SPIN_UNLOCK(OTHER_LOCK, &_51d_lru_lock); return 1; } + HA_SPIN_UNLOCK(OTHER_LOCK, &_51d_lru_lock); } /* Create workset. This will later contain detection results. */ @@ -497,10 +531,17 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) _51d_process_match(args, smp, ws); #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED - global_51degrees.device_offsets.firstOffset->deviceOffset = fiftyoneDegreesGetDeviceOffset(&global_51degrees.data_set, - smp->data.u.str.area); - global_51degrees.device_offsets.size = 1; - _51d_process_match(args, smp); +#ifndef FIFTYONEDEGREES_NO_THREADING + offsets = fiftyoneDegreesCreateDeviceOffsets(&global_51degrees.data_set); + _51d_init_device_offsets(offsets); +#else + offsets = &global_51degrees.device_offsets; +#endif + + offsets->firstOffset->deviceOffset = fiftyoneDegreesGetDeviceOffset(&global_51degrees.data_set, + smp->data.u.str.area); + offsets->size = 1; + _51d_process_match(args, smp, offsets); #endif #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED @@ -509,6 +550,12 @@ static int _51d_conv(const struct arg *args, struct sample *smp, void *private) _51d_insert_cache_entry(smp, lru, (void*)args); #endif +#ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +#ifndef FIFTYONEDEGREES_NO_THREADING + fiftyoneDegreesFreeDeviceOffsets(offsets); +#endif +#endif + return 1; } @@ -535,8 +582,11 @@ void _51d_init_http_headers() int index = 0; fiftyoneDegreesDataSet *ds = &global_51degrees.data_set; global_51degrees.header_count = fiftyoneDegreesGetHttpHeaderCount(ds); +#ifdef FIFTYONEDEGREES_NO_THREADING global_51degrees.device_offsets.firstOffset = malloc( global_51degrees.header_count * sizeof(fiftyoneDegreesDeviceOffset)); + _51d_init_device_offsets(&global_51degrees.device_offsets); +#endif global_51degrees.header_names = malloc(global_51degrees.header_count * sizeof(struct buffer)); global_51degrees.header_offsets = malloc(global_51degrees.header_count * sizeof(int32_t)); for (index = 0; index < global_51degrees.header_count; index++) { @@ -562,8 +612,8 @@ static int init_51degrees(void) if (!global_51degrees.data_file_path) return 0; - if (global.nbthread > 1) { - ha_alert("51Degrees: multithreading is not supported for now.\n"); + if (global.nbthread < 1) { + ha_alert("51Degrees: The thread count cannot be zero or negative.\n"); return (ERR_FATAL | ERR_ALERT); } @@ -586,10 +636,7 @@ static int init_51degrees(void) switch (_51d_dataset_status) { case DATA_SET_INIT_STATUS_SUCCESS: #ifdef FIFTYONEDEGREES_H_PATTERN_INCLUDED - /* only 1 workset in the pool because HAProxy is currently single threaded - * this value should be set to the number of threads in future versions. - */ - global_51degrees.pool = fiftyoneDegreesWorksetPoolCreate(&global_51degrees.data_set, NULL, 1); + global_51degrees.pool = fiftyoneDegreesWorksetPoolCreate(&global_51degrees.data_set, NULL, global.nbthread); #endif _51d_init_http_headers(); break; @@ -658,7 +705,9 @@ static void deinit_51degrees(void) fiftyoneDegreesWorksetPoolFree(global_51degrees.pool); #endif #ifdef FIFTYONEDEGREES_H_TRIE_INCLUDED +#ifdef FIFTYONEDEGREES_NO_THREADING free(global_51degrees.device_offsets.firstOffset); +#endif free(global_51degrees.header_offsets); #endif fiftyoneDegreesDataSetFree(&global_51degrees.data_set);