MEDIUM: ssl: manage shared cache by blocks for huge sessions.

Sessions using client certs are huge (more than 1 kB) and do not fit
in session cache, or require a huge cache.

In this new implementation sshcachesize set a number of available blocks
instead a number of available sessions.

Each block is large enough (128 bytes) to store a simple session (without
client certs).

Huge sessions will take multiple blocks depending on client certificate size.

Note: some unused code for session sync with remote peers was temporarily
      removed.
This commit is contained in:
Emeric Brun 2012-11-28 18:47:52 +01:00 committed by Willy Tarreau
parent dc979f2492
commit af9619da3e
3 changed files with 289 additions and 175 deletions

View File

@ -878,14 +878,16 @@ tune.sndbuf.server <number>
notifying haproxy again.
tune.ssl.cachesize <number>
Sets the size of the global SSL session cache, in number of sessions. Each
entry uses approximately 600 bytes of memory. The default value may be forced
at build time, otherwise defaults to 20000. When the cache is full, the most
idle entries are purged and reassigned. Higher values reduce the occurrence
of such a purge, hence the number of CPU-intensive SSL handshakes by ensuring
that all users keep their session as long as possible. All entries are pre-
allocated upon startup and are shared between all processes if "nbproc" is
greater than 1.
Sets the size of the global SSL session cache, in a number of blocks. A block
is large enough to contain an encoded session without peer certificate.
An encoded session with peer certificate is stored in multiple blocks
depending on the size of the peer certificate. A block use approximatively
200 bytes of memory. The default value may be forced at build time, otherwise
defaults to 20000. When the cache is full, the most idle entries are purged
and reassigned. Higher values reduce the occurrence of such a purge, hence
the number of CPU-intensive SSL handshakes by ensuring that all users keep
their session as long as possible. All entries are pre-allocated upon startup
and are shared between all processes if "nbproc" is greater than 1.
tune.ssl.lifetime <timeout>
Sets how long a cached SSL session may remain valid. This time is expressed

View File

@ -16,13 +16,12 @@
#include <openssl/ssl.h>
#include <stdint.h>
#ifndef SHSESS_MAX_FOOTER_LEN
#define SHSESS_MAX_FOOTER_LEN sizeof(uint32_t) \
+ EVP_MAX_MD_SIZE
#ifndef SHSESS_BLOCK_MIN_SIZE
#define SHSESS_BLOCK_MIN_SIZE 128
#endif
#ifndef SHSESS_MAX_DATA_LEN
#define SHSESS_MAX_DATA_LEN 512
#define SHSESS_MAX_DATA_LEN 4096
#endif
#ifndef SHCTX_DEFAULT_SIZE
@ -33,37 +32,15 @@
#define SHCTX_APPNAME "haproxy"
#endif
#define SHSESS_MAX_ENCODED_LEN SSL_MAX_SSL_SESSION_ID_LENGTH \
+ SHSESS_MAX_DATA_LEN \
+ SHSESS_MAX_FOOTER_LEN
/* Callback called on a new session event:
* session contains the sessionid zeros padded to SSL_MAX_SSL_SESSION_ID_LENGTH
* followed by ASN1 session encoding.
* len is set to SSL_MAX_SSL_SESSION_ID_LENGTH + ASN1 session length
* len is always less than SSL_MAX_SSL_SESSION_ID_LENGTH + SHSESS_MAX_DATA_LEN.
* Remaining Bytes from len to SHSESS_MAX_ENCODED_LEN can be used to add a footer.
* cdate is the creation date timestamp.
*/
void shsess_set_new_cbk(void (*func)(unsigned char *session, unsigned int len, long cdate));
/* Add a session into the cache,
* session contains the sessionid zeros padded to SSL_MAX_SSL_SESSION_ID_LENGTH
* followed by ASN1 session encoding.
* len is set to SSL_MAX_SSL_SESSION_ID_LENGTH + ASN1 data length.
* if len greater than SHSESS_MAX_ENCODED_LEN, session is not added.
* if cdate not 0, on get events session creation date will be reset to cdate */
void shctx_sess_add(const unsigned char *session, unsigned int session_len, long cdate);
/* Allocate shared memory context.
* size is maximum cached sessions.
* if set less or equal to 0, SHCTX_DEFAULT_SIZE is used.
* set use_shared_memory to 1 to use a mapped shared memory insteed
* of private. (ignored if compiled whith USE_PRIVATE_CACHE=1)
* Returns: -1 on alloc failure, size if it performs context alloc,
* and 0 if cache is already allocated */
* <size> is the number of allocated blocks into cache (default 128 bytes)
* A block is large enough to contain a classic session (without client cert)
* If <size> is set less or equal to 0, SHCTX_DEFAULT_SIZE is used.
* Set <use_shared_memory> to 1 to use a mapped shared memory instead
* of private. (ignored if compiled with USE_PRIVATE_CACHE=1).
* Returns: -1 on alloc failure, <size> if it performs context alloc,
* and 0 if cache is already allocated.
*/
int shared_context_init(int size, int use_shared_memory);
/* Set shared cache callbacks on an ssl context.

View File

@ -24,20 +24,39 @@
#include <pthread.h>
#endif /* USE_SYSCALL_FUTEX */
#endif
#include <arpa/inet.h>
#include "ebmbtree.h"
#include "proto/shctx.h"
struct shsess_packet_hdr {
unsigned int eol;
unsigned char final:1;
unsigned char seq:7;
unsigned char id[SSL_MAX_SSL_SESSION_ID_LENGTH];
};
struct shsess_packet {
unsigned char version;
unsigned char sig[SHA_DIGEST_LENGTH];
struct shsess_packet_hdr hdr;
unsigned char data[0];
};
struct shared_session {
struct ebmb_node key;
unsigned char key_data[SSL_MAX_SSL_SESSION_ID_LENGTH];
long c_date;
int data_len;
unsigned char data[SHSESS_MAX_DATA_LEN];
struct shared_session *p;
struct shared_session *n;
unsigned char data[SHSESS_BLOCK_MIN_SIZE];
};
struct shared_block {
union {
struct shared_session session;
unsigned char data[sizeof(struct shared_session)];
} data;
short int data_len;
struct shared_block *p;
struct shared_block *n;
};
struct shared_context {
#ifndef USE_PRIVATE_CACHE
@ -47,8 +66,11 @@ struct shared_context {
pthread_mutex_t mutex;
#endif
#endif
struct shared_session active;
struct shared_session free;
struct shsess_packet_hdr upd;
unsigned char data[SHSESS_MAX_DATA_LEN];
short int data_len;
struct shared_block active;
struct shared_block free;
};
/* Static shared context */
@ -57,9 +79,6 @@ static struct shared_context *shctx = NULL;
static int use_shared_mem = 0;
#endif
/* Callbacks */
static void (*shared_session_new_cbk)(unsigned char *session, unsigned int session_len, long cdate);
/* Lock functions */
#ifdef USE_PRIVATE_CACHE
#define shared_context_lock()
@ -156,41 +175,171 @@ static inline void _shared_context_unlock(void)
/* List Macros */
#define shsess_unset(s) (s)->n->p = (s)->p; \
#define shblock_unset(s) (s)->n->p = (s)->p; \
(s)->p->n = (s)->n;
#define shsess_set_free(s) shsess_unset(s) \
(s)->p = &shctx->free; \
(s)->n = shctx->free.n; \
shctx->free.n->p = s; \
shctx->free.n = s;
#define shblock_set_free(s) shblock_unset(s) \
(s)->n = &shctx->free; \
(s)->p = shctx->free.p; \
shctx->free.p->n = s; \
shctx->free.p = s;
#define shsess_set_active(s) shsess_unset(s) \
(s)->p = &shctx->active; \
(s)->n = shctx->active.n; \
shctx->active.n->p = s; \
shctx->active.n = s;
#define shblock_set_active(s) shblock_unset(s) \
(s)->n = &shctx->active; \
(s)->p = shctx->active.p; \
shctx->active.p->n = s; \
shctx->active.p = s;
#define shsess_get_next() (shctx->free.p == &shctx->free) ? \
shctx->active.p : shctx->free.p;
/* Tree Macros */
#define shsess_tree_delete(s) ebmb_delete(&(s)->key);
#define shsess_tree_insert(s) (struct shared_session *)ebmb_insert(&shctx->active.key.node.branches, \
#define shsess_tree_insert(s) (struct shared_session *)ebmb_insert(&shctx->active.data.session.key.node.branches, \
&(s)->key, SSL_MAX_SSL_SESSION_ID_LENGTH);
#define shsess_tree_lookup(k) (struct shared_session *)ebmb_lookup(&shctx->active.key.node.branches, \
#define shsess_tree_lookup(k) (struct shared_session *)ebmb_lookup(&shctx->active.data.session.key.node.branches, \
(k), SSL_MAX_SSL_SESSION_ID_LENGTH);
/* Other Macros */
/* shared session functions */
#define shsess_set_key(s,k,l) { memcpy((s)->key_data, (k), (l)); \
if ((l) < SSL_MAX_SSL_SESSION_ID_LENGTH) \
memset((s)->key_data+(l), 0, SSL_MAX_SSL_SESSION_ID_LENGTH-(l)); };
/* Free session blocks, returns number of freed blocks */
static int shsess_free(struct shared_session *shsess)
{
struct shared_block *block;
int ret = 1;
if (((struct shared_block *)shsess)->data_len <= sizeof(shsess->data)) {
shblock_set_free((struct shared_block *)shsess);
return ret;
}
block = ((struct shared_block *)shsess)->n;
shblock_set_free((struct shared_block *)shsess);
while (1) {
struct shared_block *next;
if (block->data_len <= sizeof(block->data)) {
/* last block */
shblock_set_free(block);
ret++;
break;
}
next = block->n;
shblock_set_free(block);
ret++;
block = next;
}
return ret;
}
/* This function frees enough blocks to store a new session of data_len.
* Returns a ptr on a free block if it succeeds, or NULL if there are not
* enough blocks to store that session.
*/
static struct shared_session *shsess_get_next(int data_len)
{
int head = 0;
struct shared_block *b;
b = shctx->free.n;
while (b != &shctx->free) {
if (!head) {
data_len -= sizeof(b->data.session.data);
head = 1;
}
else
data_len -= sizeof(b->data.data);
if (data_len <= 0)
return &shctx->free.n->data.session;
b = b->n;
}
b = shctx->active.n;
while (b != &shctx->active) {
int freed;
shsess_tree_delete(&b->data.session);
freed = shsess_free(&b->data.session);
if (!head)
data_len -= sizeof(b->data.session.data) + (freed-1)*sizeof(b->data.data);
else
data_len -= freed*sizeof(b->data.data);
if (data_len <= 0)
return &shctx->free.n->data.session;
b = shctx->active.n;
}
return NULL;
}
/* store a session into the cache
* s_id : session id padded with zero to SSL_MAX_SSL_SESSION_ID_LENGTH
* data: asn1 encoded session
* data_len: asn1 encoded session length
* Returns 1 id session was stored (else 0)
*/
static int shsess_store(unsigned char *s_id, unsigned char *data, int data_len)
{
struct shared_session *shsess, *oldshsess;
shsess = shsess_get_next(data_len);
if (!shsess) {
/* Could not retrieve enough free blocks to store that session */
return 0;
}
/* prepare key */
memcpy(shsess->key_data, s_id, SSL_MAX_SSL_SESSION_ID_LENGTH);
/* it returns the already existing node
or current node if none, never returns null */
oldshsess = shsess_tree_insert(shsess);
if (oldshsess != shsess) {
/* free all blocks used by old node */
shsess_free(oldshsess);
shsess = oldshsess;
}
((struct shared_block *)shsess)->data_len = data_len;
if (data_len <= sizeof(shsess->data)) {
/* Store on a single block */
memcpy(shsess->data, data, data_len);
shblock_set_active((struct shared_block *)shsess);
}
else {
unsigned char *p;
/* Store on multiple blocks */
int cur_len;
memcpy(shsess->data, data, sizeof(shsess->data));
p = data + sizeof(shsess->data);
cur_len = data_len - sizeof(shsess->data);
shblock_set_active((struct shared_block *)shsess);
while (1) {
/* Store next data on free block.
* shsess_get_next guarantees that there are enough
* free blocks in queue.
*/
struct shared_block *block;
block = shctx->free.n;
if (cur_len <= sizeof(block->data)) {
/* This is the last block */
block->data_len = cur_len;
memcpy(block->data.data, p, cur_len);
shblock_set_active(block);
break;
}
/* Intermediate block */
block->data_len = cur_len;
memcpy(block->data.data, p, sizeof(block->data));
p += sizeof(block->data.data);
cur_len -= sizeof(block->data.data);
shblock_set_active(block);
}
}
return 1;
}
/* SSL context callbacks */
@ -198,51 +347,43 @@ static inline void _shared_context_unlock(void)
/* SSL callback used on new session creation */
int shctx_new_cb(SSL *ssl, SSL_SESSION *sess)
{
struct shared_session *shsess;
unsigned char *data,*p;
unsigned int data_len;
unsigned char encsess[SHSESS_MAX_ENCODED_LEN];
(void)ssl;
unsigned char encsess[sizeof(struct shsess_packet)+SHSESS_MAX_DATA_LEN];
struct shsess_packet *packet = (struct shsess_packet *)encsess;
unsigned char *p;
int data_len, sid_length;
/* check if session reserved size in aligned buffer is large enougth for the ASN1 encode session */
/* Session id is already stored in to key and session id is known
* so we dont store it to keep size.
*/
sid_length = sess->session_id_length;
sess->session_id_length = 0;
sess->sid_ctx_length = 0;
/* check if buffer is large enough for the ASN1 encoded session */
data_len = i2d_SSL_SESSION(sess, NULL);
if (data_len > SHSESS_MAX_DATA_LEN)
return 0;
goto err;
/* process ASN1 session encoding before the lock: lower cost */
p = data = encsess+SSL_MAX_SSL_SESSION_ID_LENGTH;
/* process ASN1 session encoding before the lock */
p = packet->data;
i2d_SSL_SESSION(sess, &p);
memcpy(packet->hdr.id, sess->session_id, sid_length);
if (sid_length < SSL_MAX_SSL_SESSION_ID_LENGTH)
memset(&packet->hdr.id[sid_length], 0, SSL_MAX_SSL_SESSION_ID_LENGTH-sid_length);
shared_context_lock();
shsess = shsess_get_next();
shsess_tree_delete(shsess);
shsess_set_key(shsess, sess->session_id, sess->session_id_length);
/* it returns the already existing node or current node if none, never returns null */
shsess = shsess_tree_insert(shsess);
/* store ASN1 encoded session into cache */
shsess->data_len = data_len;
memcpy(shsess->data, data, data_len);
/* store creation date */
shsess->c_date = SSL_SESSION_get_time(sess);
shsess_set_active(shsess);
/* store to cache */
shsess_store(packet->hdr.id, packet->data, data_len);
shared_context_unlock();
if (shared_session_new_cbk) { /* if user level callback is set */
/* copy sessionid padded with 0 into the sessionid + data aligned buffer */
memcpy(encsess, sess->session_id, sess->session_id_length);
if (sess->session_id_length < SSL_MAX_SSL_SESSION_ID_LENGTH)
memset(encsess+sess->session_id_length, 0, SSL_MAX_SSL_SESSION_ID_LENGTH-sess->session_id_length);
shared_session_new_cbk(encsess, SSL_MAX_SSL_SESSION_ID_LENGTH+data_len, SSL_SESSION_get_time(sess));
}
err:
/* reset original length values */
sess->sid_ctx_length = ssl->sid_ctx_length;
sess->session_id_length = sid_length;
return 0; /* do not increment session reference count */
}
@ -253,10 +394,8 @@ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_cop
struct shared_session *shsess;
unsigned char data[SHSESS_MAX_DATA_LEN], *p;
unsigned char tmpkey[SSL_MAX_SSL_SESSION_ID_LENGTH];
unsigned int data_len;
long cdate;
int data_len;
SSL_SESSION *sess;
(void)ssl;
/* allow the session to be freed automatically by openssl */
*do_copy = 0;
@ -279,24 +418,52 @@ SSL_SESSION *shctx_get_cb(SSL *ssl, unsigned char *key, int key_len, int *do_cop
return NULL;
}
/* backup creation date to reset in session after ASN1 decode */
cdate = shsess->c_date;
data_len = ((struct shared_block *)shsess)->data_len;
if (data_len <= sizeof(shsess->data)) {
/* Session stored on single block */
memcpy(data, shsess->data, data_len);
shblock_set_active((struct shared_block *)shsess);
}
else {
/* Session stored on multiple blocks */
struct shared_block *block;
/* copy ASN1 session data to decode outside the lock */
data_len = shsess->data_len;
memcpy(data, shsess->data, shsess->data_len);
memcpy(data, shsess->data, sizeof(shsess->data));
p = data + sizeof(shsess->data);
block = ((struct shared_block *)shsess)->n;
shblock_set_active((struct shared_block *)shsess);
while (1) {
/* Retrieve data from next block */
struct shared_block *next;
shsess_set_active(shsess);
if (block->data_len <= sizeof(block->data.data)) {
/* This is the last block */
memcpy(p, block->data.data, block->data_len);
p += block->data_len;
shblock_set_active(block);
break;
}
/* Intermediate block */
memcpy(p, block->data.data, sizeof(block->data.data));
p += sizeof(block->data.data);
next = block->n;
shblock_set_active(block);
block = next;
}
}
shared_context_unlock();
/* decode ASN1 session */
p = data;
sess = d2i_SSL_SESSION(NULL, (const unsigned char **)&p, data_len);
/* reset creation date */
if (sess)
SSL_SESSION_set_time(sess, cdate);
/* Reset session id and session id contenxt */
if (sess) {
memcpy(sess->session_id, key, key_len);
sess->session_id_length = key_len;
memcpy(sess->sid_ctx, ssl->sid_ctx, ssl->sid_ctx_length);
sess->sid_ctx_length = ssl->sid_ctx_length;
}
return sess;
}
@ -321,59 +488,21 @@ void shctx_remove_cb(SSL_CTX *ctx, SSL_SESSION *sess)
/* lookup for session */
shsess = shsess_tree_lookup(key);
if (shsess) {
shsess_set_free(shsess);
/* free session */
shsess_tree_delete(shsess);
shsess_free(shsess);
}
/* unlock cache */
shared_context_unlock();
}
/* User level function called to add a session to the cache (remote updates) */
void shctx_sess_add(const unsigned char *encsess, unsigned int len, long cdate)
{
struct shared_session *shsess;
/* check buffer is at least padded key long + 1 byte
and data_len not too long */
if ((len <= SSL_MAX_SSL_SESSION_ID_LENGTH)
|| (len > SHSESS_MAX_DATA_LEN+SSL_MAX_SSL_SESSION_ID_LENGTH))
return;
shared_context_lock();
shsess = shsess_get_next();
shsess_tree_delete(shsess);
shsess_set_key(shsess, encsess, SSL_MAX_SSL_SESSION_ID_LENGTH);
/* it returns the already existing node or current node if none, never returns null */
shsess = shsess_tree_insert(shsess);
/* store into cache and update earlier on session get events */
if (cdate)
shsess->c_date = (long)cdate;
/* copy ASN1 session data into cache */
shsess->data_len = len-SSL_MAX_SSL_SESSION_ID_LENGTH;
memcpy(shsess->data, encsess+SSL_MAX_SSL_SESSION_ID_LENGTH, shsess->data_len);
shsess_set_active(shsess);
shared_context_unlock();
}
/* Function used to set a callback on new session creation */
void shsess_set_new_cbk(void (*func)(unsigned char *, unsigned int, long))
{
shared_session_new_cbk = func;
}
/* Allocate shared memory context.
* size is maximum cached sessions.
* if set less or equal to 0, SHCTX_DEFAULT_SIZE is used.
* Returns: -1 on alloc failure, size if it performs context alloc,
* and 0 if cache is already allocated */
* <size> is maximum cached sessions.
* If <size> is set to less or equal to 0, SHCTX_DEFAULT_SIZE is used.
* Returns: -1 on alloc failure, <size> if it performs context alloc,
* and 0 if cache is already allocated.
*/
int shared_context_init(int size, int shared)
{
int i;
@ -382,7 +511,7 @@ int shared_context_init(int size, int shared)
pthread_mutexattr_t attr;
#endif /* USE_SYSCALL_FUTEX */
#endif
struct shared_session *prev,*cur;
struct shared_block *prev,*cur;
int maptype = MAP_PRIVATE;
if (shctx)
@ -391,12 +520,14 @@ int shared_context_init(int size, int shared)
if (size<=0)
size = SHCTX_DEFAULT_SIZE;
/* Increate size by one to reserve one node for lookup */
size++;
#ifndef USE_PRIVATE_CACHE
if (shared)
maptype = MAP_SHARED;
#endif
shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_session)),
shctx = (struct shared_context *)mmap(NULL, sizeof(struct shared_context)+(size*sizeof(struct shared_block)),
PROT_READ | PROT_WRITE, maptype | MAP_ANON, -1, 0);
if (!shctx || shctx == MAP_FAILED) {
shctx = NULL;
@ -415,12 +546,16 @@ int shared_context_init(int size, int shared)
use_shared_mem = 1;
#endif
memset(&shctx->active.key, 0, sizeof(struct ebmb_node));
memset(&shctx->free.key, 0, sizeof(struct ebmb_node));
memset(&shctx->active.data.session.key, 0, sizeof(struct ebmb_node));
memset(&shctx->free.data.session.key, 0, sizeof(struct ebmb_node));
/* No duplicate authorized in tree: */
//shctx->active.key.node.branches.b[1] = (void *)1;
shctx->active.key.node.branches = EB_ROOT_UNIQUE;
shctx->active.data.session.key.node.branches = EB_ROOT_UNIQUE;
/* Init remote update cache */
shctx->upd.eol = 0;
shctx->upd.seq = 0;
shctx->data_len = 0;
cur = &shctx->active;
cur->n = cur->p = cur;
@ -428,7 +563,7 @@ int shared_context_init(int size, int shared)
cur = &shctx->free;
for (i = 0 ; i < size ; i++) {
prev = cur;
cur = (struct shared_session *)((char *)prev + sizeof(struct shared_session));
cur = (struct shared_block *)((char *)prev + sizeof(struct shared_block));
prev->n = cur;
cur->p = prev;
}