From f487bbbfe2222986ccf9674e6714c7dd506a5a16 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 9 Jan 2011 15:56:50 +0100 Subject: [PATCH 1/6] allocation stats in INFO --- src/redis.c | 13 +++++++++++++ src/zmalloc.c | 35 +++++++++++++++++++++++------------ src/zmalloc.h | 3 +++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/redis.c b/src/redis.c index b9033d3f0..fa01d7c3b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1313,6 +1313,19 @@ sds genRedisInfoString(void) { eta ); } + + info = sdscat(info,"allocation_stats:"); + for (j = 0; j <= ZMALLOC_MAX_ALLOC_STAT; j++) { + size_t count = zmalloc_allocations_for_size(j); + if (count) { + if (info[sdslen(info)-1] != ':') info = sdscatlen(info,",",1); + info = sdscatprintf(info,"%s%d=%zu", + (j == ZMALLOC_MAX_ALLOC_STAT) ? ">=" : "", + j,count); + } + } + info = sdscat(info,"\r\n"); + for (j = 0; j < server.dbnum; j++) { long long keys, vkeys; diff --git a/src/zmalloc.c b/src/zmalloc.c index 1917a549a..ed19616b8 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -33,6 +33,7 @@ #include #include #include "config.h" +#include "zmalloc.h" #ifdef HAVE_MALLOC_SIZE #define PREFIX_SIZE (0) @@ -52,19 +53,22 @@ #define free(ptr) tc_free(ptr) #endif -#define increment_used_memory(__n) do { \ +#define update_zmalloc_stat_alloc(__n,__size) do { \ size_t _n = (__n); \ + size_t _stat_slot = (__size < ZMALLOC_MAX_ALLOC_STAT) ? __size : ZMALLOC_MAX_ALLOC_STAT; \ if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ if (zmalloc_thread_safe) { \ pthread_mutex_lock(&used_memory_mutex); \ used_memory += _n; \ + zmalloc_allocations[_stat_slot]++; \ pthread_mutex_unlock(&used_memory_mutex); \ } else { \ used_memory += _n; \ + zmalloc_allocations[_stat_slot]++; \ } \ } while(0) -#define decrement_used_memory(__n) do { \ +#define update_zmalloc_stat_free(__n) do { \ size_t _n = (__n); \ if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \ if (zmalloc_thread_safe) { \ @@ -79,6 +83,8 @@ static size_t used_memory = 0; static int zmalloc_thread_safe = 0; pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER; +/* Note that malloc_allocations elements are initialized to zero by C */ +size_t zmalloc_allocations[ZMALLOC_MAX_ALLOC_STAT+1]; static void zmalloc_oom(size_t size) { fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", @@ -92,11 +98,11 @@ void *zmalloc(size_t size) { if (!ptr) zmalloc_oom(size); #ifdef HAVE_MALLOC_SIZE - increment_used_memory(redis_malloc_size(ptr)); + update_zmalloc_stat_alloc(redis_malloc_size(ptr),size); return ptr; #else *((size_t*)ptr) = size; - increment_used_memory(size+PREFIX_SIZE); + update_zmalloc_stat_alloc(size+PREFIX_SIZE,size); return (char*)ptr+PREFIX_SIZE; #endif } @@ -106,11 +112,11 @@ void *zcalloc(size_t size) { if (!ptr) zmalloc_oom(size); #ifdef HAVE_MALLOC_SIZE - increment_used_memory(redis_malloc_size(ptr)); + update_zmalloc_stat_alloc(redis_malloc_size(ptr),size); return ptr; #else *((size_t*)ptr) = size; - increment_used_memory(size+PREFIX_SIZE); + update_zmalloc_stat_alloc(size+PREFIX_SIZE,size); return (char*)ptr+PREFIX_SIZE; #endif } @@ -128,8 +134,8 @@ void *zrealloc(void *ptr, size_t size) { newptr = realloc(ptr,size); if (!newptr) zmalloc_oom(size); - decrement_used_memory(oldsize); - increment_used_memory(redis_malloc_size(newptr)); + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(redis_malloc_size(newptr),size); return newptr; #else realptr = (char*)ptr-PREFIX_SIZE; @@ -138,8 +144,8 @@ void *zrealloc(void *ptr, size_t size) { if (!newptr) zmalloc_oom(size); *((size_t*)newptr) = size; - decrement_used_memory(oldsize); - increment_used_memory(size); + update_zmalloc_stat_free(oldsize); + update_zmalloc_stat_alloc(size,size); return (char*)newptr+PREFIX_SIZE; #endif } @@ -152,12 +158,12 @@ void zfree(void *ptr) { if (ptr == NULL) return; #ifdef HAVE_MALLOC_SIZE - decrement_used_memory(redis_malloc_size(ptr)); + update_zmalloc_stat_free(redis_malloc_size(ptr)); free(ptr); #else realptr = (char*)ptr-PREFIX_SIZE; oldsize = *((size_t*)realptr); - decrement_used_memory(oldsize+PREFIX_SIZE); + update_zmalloc_stat_free(oldsize+PREFIX_SIZE); free(realptr); #endif } @@ -179,6 +185,11 @@ size_t zmalloc_used_memory(void) { return um; } +size_t zmalloc_allocations_for_size(size_t size) { + if (size > ZMALLOC_MAX_ALLOC_STAT || size < 0) return 0; + return zmalloc_allocations[size]; +} + void zmalloc_enable_thread_safeness(void) { zmalloc_thread_safe = 1; } diff --git a/src/zmalloc.h b/src/zmalloc.h index bb6f629af..d19979a30 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -40,5 +40,8 @@ size_t zmalloc_used_memory(void); void zmalloc_enable_thread_safeness(void); float zmalloc_get_fragmentation_ratio(void); size_t zmalloc_get_rss(void); +size_t zmalloc_allocations_for_size(size_t size); + +#define ZMALLOC_MAX_ALLOC_STAT 256 #endif /* _ZMALLOC_H */ From 360664c9c8da2cbacc5000d7a058c754a36f8ec7 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Jan 2011 09:53:57 +0100 Subject: [PATCH 2/6] master-slave replication fixed, it was not listing any key using KEYS command in the slave. --- src/db.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index aa1c14ad4..ede242dbe 100644 --- a/src/db.c +++ b/src/db.c @@ -453,6 +453,8 @@ void propagateExpire(redisDb *db, robj *key) { int expireIfNeeded(redisDb *db, robj *key) { time_t when = getExpire(db,key); + if (when < 0) return 0; /* No expire for this key */ + /* If we are running in the context of a slave, return ASAP: * the slave key expiration is controlled by the master that will * send us synthesized DEL operations for expired keys. @@ -464,8 +466,6 @@ int expireIfNeeded(redisDb *db, robj *key) { return time(NULL) > when; } - if (when < 0) return 0; - /* Return when this key has not expired */ if (time(NULL) <= when) return 0; From c13c4080e513cda2a1f1fce593ec45fb3a3c9666 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Jan 2011 09:57:07 +0100 Subject: [PATCH 3/6] useless expression removed in zmalloc.c --- src/zmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index ed19616b8..b438017dc 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -186,7 +186,7 @@ size_t zmalloc_used_memory(void) { } size_t zmalloc_allocations_for_size(size_t size) { - if (size > ZMALLOC_MAX_ALLOC_STAT || size < 0) return 0; + if (size > ZMALLOC_MAX_ALLOC_STAT) return 0; return zmalloc_allocations[size]; } From ebb07fb4bbe971460df206f5c2fded037207d7b0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Jan 2011 10:20:02 +0100 Subject: [PATCH 4/6] implemented two new INFO fields showing the size of clients max input and output buffers. --- src/networking.c | 19 +++++++++++++++++++ src/redis.c | 5 +++++ src/redis.h | 2 ++ 3 files changed, 26 insertions(+) diff --git a/src/networking.c b/src/networking.c index 79596bc24..9716ca319 100644 --- a/src/networking.c +++ b/src/networking.c @@ -821,3 +821,22 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { } processInputBuffer(c); } + +void getClientsMaxBuffers(unsigned long *longest_output_list, + unsigned long *biggest_input_buffer) { + redisClient *c; + listNode *ln; + listIter li; + unsigned long lol = 0, bib = 0; + + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + c = listNodeValue(ln); + + if (listLength(c->reply) > lol) lol = listLength(c->reply); + if (sdslen(c->querybuf) > bib) bib = sdslen(c->querybuf); + } + *longest_output_list = lol; + *biggest_input_buffer = bib; +} + diff --git a/src/redis.c b/src/redis.c index fa01d7c3b..2b7935e3a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1141,9 +1141,11 @@ sds genRedisInfoString(void) { int j; char hmem[64]; struct rusage self_ru, c_ru; + unsigned long lol, bib; getrusage(RUSAGE_SELF, &self_ru); getrusage(RUSAGE_CHILDREN, &c_ru); + getClientsMaxBuffers(&lol,&bib); bytesToHuman(hmem,zmalloc_used_memory()); info = sdscatprintf(sdsempty(), @@ -1162,6 +1164,8 @@ sds genRedisInfoString(void) { "used_cpu_user_childrens:%.2f\r\n" "connected_clients:%d\r\n" "connected_slaves:%d\r\n" + "client_longest_output_list:%lu\r\n" + "client_biggest_input_buf:%lu\r\n" "blocked_clients:%d\r\n" "used_memory:%zu\r\n" "used_memory_human:%s\r\n" @@ -1201,6 +1205,7 @@ sds genRedisInfoString(void) { (float)c_ru.ru_stime.tv_sec+(float)c_ru.ru_stime.tv_usec/1000000, listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), + lol, bib, server.bpop_blocked_clients, zmalloc_used_memory(), hmem, diff --git a/src/redis.h b/src/redis.h index 6d212edad..4fea3a44f 100644 --- a/src/redis.h +++ b/src/redis.h @@ -656,6 +656,8 @@ void addReplyDouble(redisClient *c, double d); void addReplyLongLong(redisClient *c, long long ll); void addReplyMultiBulkLen(redisClient *c, long length); void *dupClientReplyValue(void *o); +void getClientsMaxBuffers(unsigned long *longest_output_list, + unsigned long *biggest_input_buffer); #ifdef __GNUC__ void addReplyErrorFormat(redisClient *c, const char *fmt, ...) From f43502fb0e9b0bc84116357ba85b489c3fceb095 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 15 Jan 2011 22:24:22 +0100 Subject: [PATCH 5/6] lame typo causing bad hashes memory characteristic --- src/redis.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.h b/src/redis.h index 4fea3a44f..aa5ed24f9 100644 --- a/src/redis.h +++ b/src/redis.h @@ -189,8 +189,8 @@ #define APPENDFSYNC_EVERYSEC 2 /* Zip structure related defaults */ -#define REDIS_HASH_MAX_ZIPMAP_ENTRIES 64 -#define REDIS_HASH_MAX_ZIPMAP_VALUE 512 +#define REDIS_HASH_MAX_ZIPMAP_ENTRIES 512 +#define REDIS_HASH_MAX_ZIPMAP_VALUE 64 #define REDIS_LIST_MAX_ZIPLIST_ENTRIES 512 #define REDIS_LIST_MAX_ZIPLIST_VALUE 64 #define REDIS_SET_MAX_INTSET_ENTRIES 512 From 99677e89d0194fd5017210c250f97fd09492eaf2 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 15 Jan 2011 22:31:07 +0100 Subject: [PATCH 6/6] default hash encoding settings also fixed in redis.conf --- redis.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 7d34ae3a7..ff694e43a 100644 --- a/redis.conf +++ b/redis.conf @@ -369,8 +369,8 @@ vm-max-threads 4 # have at max a given numer of elements, and the biggest element does not # exceed a given threshold. You can configure this limits with the following # configuration directives. -hash-max-zipmap-entries 64 -hash-max-zipmap-value 512 +hash-max-zipmap-entries 512 +hash-max-zipmap-value 64 # Similarly to hashes, small lists are also encoded in a special way in order # to save a lot of space. The special representation is only used when