mirror of
http://github.com/valkey-io/valkey
synced 2024-11-21 16:46:15 +00:00
Fix ziplist and listpack overflows and truncations (CVE-2021-32627, CVE-2021-32628)
- fix possible heap corruption in ziplist and listpack resulting by trying to allocate more than the maximum size of 4GB. - prevent ziplist (hash and zset) from reaching size of above 1GB, will be converted to HT encoding, that's not a useful size. - prevent listpack (stream) from reaching size of above 1GB. - XADD will start a new listpack if the new record may cause the previous listpack to grow over 1GB. - XADD will respond with an error if a single stream record is over 1GB - List type (ziplist in quicklist) was truncating strings that were over 4GB, now it'll respond with an error.
This commit is contained in:
Oran Agra
parent
666ed7facf
commit
f6a40570fa
@ -635,7 +635,7 @@ void georadiusGeneric(client *c, int flags) {
|
||||
robj *zobj;
|
||||
zset *zs;
|
||||
int i;
|
||||
size_t maxelelen = 0;
|
||||
size_t maxelelen = 0, totelelen = 0;
|
||||
|
||||
if (returned_items) {
|
||||
zobj = createZsetObject();
|
||||
@ -650,13 +650,14 @@ void georadiusGeneric(client *c, int flags) {
|
||||
size_t elelen = sdslen(gp->member);
|
||||
|
||||
if (maxelelen < elelen) maxelelen = elelen;
|
||||
totelelen += elelen;
|
||||
znode = zslInsert(zs->zsl,score,gp->member);
|
||||
serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);
|
||||
gp->member = NULL;
|
||||
}
|
||||
|
||||
if (returned_items) {
|
||||
zsetConvertToZiplistIfNeeded(zobj,maxelelen);
|
||||
zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen);
|
||||
setKey(c,c->db,storekey,zobj);
|
||||
decrRefCount(zobj);
|
||||
notifyKeyspaceEvent(NOTIFY_ZSET,"georadiusstore",storekey,
|
||||
|
||||
@ -283,7 +283,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui
|
||||
} else {
|
||||
if (size < 64) *enclen = 1+size;
|
||||
else if (size < 4096) *enclen = 2+size;
|
||||
else *enclen = 5+size;
|
||||
else *enclen = 5+(uint64_t)size;
|
||||
return LP_ENCODING_STRING;
|
||||
}
|
||||
}
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
*/
|
||||
|
||||
#include <string.h> /* for memcpy */
|
||||
#include "redisassert.h"
|
||||
#include "quicklist.h"
|
||||
#include "zmalloc.h"
|
||||
#include "ziplist.h"
|
||||
@ -43,11 +44,16 @@
|
||||
#define REDIS_STATIC static
|
||||
#endif
|
||||
|
||||
/* Optimization levels for size-based filling */
|
||||
/* Optimization levels for size-based filling.
|
||||
* Note that the largest possible limit is 16k, so even if each record takes
|
||||
* just one byte, it still won't overflow the 16 bit count field. */
|
||||
static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
|
||||
|
||||
/* Maximum size in bytes of any multi-element ziplist.
|
||||
* Larger values will live in their own isolated ziplists. */
|
||||
* Larger values will live in their own isolated ziplists.
|
||||
* This is used only if we're limited by record count. when we're limited by
|
||||
* size, the maximum limit is bigger, but still safe.
|
||||
* 8k is a recommended / default size limit */
|
||||
#define SIZE_SAFETY_LIMIT 8192
|
||||
|
||||
/* Minimum ziplist size in bytes for attempting compression. */
|
||||
@ -449,6 +455,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
|
||||
unsigned int new_sz = node->sz + sz + ziplist_overhead;
|
||||
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))
|
||||
return 1;
|
||||
/* when we return 1 above we know that the limit is a size limit (which is
|
||||
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
|
||||
else if (!sizeMeetsSafetyLimit(new_sz))
|
||||
return 0;
|
||||
else if ((int)node->count < fill)
|
||||
@ -468,6 +476,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
|
||||
unsigned int merge_sz = a->sz + b->sz - 11;
|
||||
if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)))
|
||||
return 1;
|
||||
/* when we return 1 above we know that the limit is a size limit (which is
|
||||
* safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
|
||||
else if (!sizeMeetsSafetyLimit(merge_sz))
|
||||
return 0;
|
||||
else if ((int)(a->count + b->count) <= fill)
|
||||
@ -487,6 +497,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
|
||||
* Returns 1 if new head created. */
|
||||
int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
|
||||
quicklistNode *orig_head = quicklist->head;
|
||||
assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
|
||||
if (likely(
|
||||
_quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) {
|
||||
quicklist->head->zl =
|
||||
@ -510,6 +521,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
|
||||
* Returns 1 if new tail created. */
|
||||
int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) {
|
||||
quicklistNode *orig_tail = quicklist->tail;
|
||||
assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
|
||||
if (likely(
|
||||
_quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) {
|
||||
quicklist->tail->zl =
|
||||
@ -852,6 +864,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry,
|
||||
int fill = quicklist->fill;
|
||||
quicklistNode *node = entry->node;
|
||||
quicklistNode *new_node = NULL;
|
||||
assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
|
||||
|
||||
if (!node) {
|
||||
/* we have no reference node, so let's create only node in the list */
|
||||
|
||||
36
src/rdb.c
36
src/rdb.c
@ -1561,7 +1561,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
|
||||
} else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) {
|
||||
/* Read list/set value. */
|
||||
uint64_t zsetlen;
|
||||
size_t maxelelen = 0;
|
||||
size_t maxelelen = 0, totelelen = 0;
|
||||
zset *zs;
|
||||
|
||||
if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
|
||||
@ -1598,6 +1598,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
|
||||
|
||||
/* Don't care about integer-encoded strings. */
|
||||
if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele);
|
||||
totelelen += sdslen(sdsele);
|
||||
|
||||
znode = zslInsert(zs->zsl,score,sdsele);
|
||||
dictAdd(zs->dict,sdsele,&znode->score);
|
||||
@ -1605,8 +1606,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
|
||||
|
||||
/* Convert *after* loading, since sorted sets are not stored ordered. */
|
||||
if (zsetLength(o) <= server.zset_max_ziplist_entries &&
|
||||
maxelelen <= server.zset_max_ziplist_value)
|
||||
zsetConvert(o,OBJ_ENCODING_ZIPLIST);
|
||||
maxelelen <= server.zset_max_ziplist_value &&
|
||||
ziplistSafeToAdd(NULL, totelelen))
|
||||
{
|
||||
zsetConvert(o,OBJ_ENCODING_ZIPLIST);
|
||||
}
|
||||
} else if (rdbtype == RDB_TYPE_HASH) {
|
||||
uint64_t len;
|
||||
int ret;
|
||||
@ -1635,21 +1639,25 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Convert to hash table if size threshold is exceeded */
|
||||
if (sdslen(field) > server.hash_max_ziplist_value ||
|
||||
sdslen(value) > server.hash_max_ziplist_value ||
|
||||
!ziplistSafeToAdd(o->ptr, sdslen(field)+sdslen(value)))
|
||||
{
|
||||
hashTypeConvert(o, OBJ_ENCODING_HT);
|
||||
ret = dictAdd((dict*)o->ptr, field, value);
|
||||
if (ret == DICT_ERR) {
|
||||
rdbExitReportCorruptRDB("Duplicate hash fields detected");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
/* Add pair to ziplist */
|
||||
o->ptr = ziplistPush(o->ptr, (unsigned char*)field,
|
||||
sdslen(field), ZIPLIST_TAIL);
|
||||
o->ptr = ziplistPush(o->ptr, (unsigned char*)value,
|
||||
sdslen(value), ZIPLIST_TAIL);
|
||||
|
||||
/* Convert to hash table if size threshold is exceeded */
|
||||
if (sdslen(field) > server.hash_max_ziplist_value ||
|
||||
sdslen(value) > server.hash_max_ziplist_value)
|
||||
{
|
||||
sdsfree(field);
|
||||
sdsfree(value);
|
||||
hashTypeConvert(o, OBJ_ENCODING_HT);
|
||||
break;
|
||||
}
|
||||
sdsfree(field);
|
||||
sdsfree(value);
|
||||
}
|
||||
@ -1726,6 +1734,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) {
|
||||
while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {
|
||||
if (flen > maxlen) maxlen = flen;
|
||||
if (vlen > maxlen) maxlen = vlen;
|
||||
if (!ziplistSafeToAdd(zl, (size_t)flen + vlen)) {
|
||||
rdbExitReportCorruptRDB("Hash zipmap too big (%u)", flen);
|
||||
}
|
||||
|
||||
zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);
|
||||
zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
|
||||
}
|
||||
|
||||
@ -1999,7 +1999,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range);
|
||||
unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range);
|
||||
unsigned long zsetLength(const robj *zobj);
|
||||
void zsetConvert(robj *zobj, int encoding);
|
||||
void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen);
|
||||
void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen);
|
||||
int zsetScore(robj *zobj, sds member, double *score);
|
||||
unsigned long zslGetRank(zskiplist *zsl, double score, sds o);
|
||||
int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore);
|
||||
|
||||
13
src/t_hash.c
13
src/t_hash.c
@ -39,17 +39,22 @@
|
||||
* as their string length can be queried in constant time. */
|
||||
void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
|
||||
int i;
|
||||
size_t sum = 0;
|
||||
|
||||
if (o->encoding != OBJ_ENCODING_ZIPLIST) return;
|
||||
|
||||
for (i = start; i <= end; i++) {
|
||||
if (sdsEncodedObject(argv[i]) &&
|
||||
sdslen(argv[i]->ptr) > server.hash_max_ziplist_value)
|
||||
{
|
||||
if (!sdsEncodedObject(argv[i]))
|
||||
continue;
|
||||
size_t len = sdslen(argv[i]->ptr);
|
||||
if (len > server.hash_max_ziplist_value) {
|
||||
hashTypeConvert(o, OBJ_ENCODING_HT);
|
||||
break;
|
||||
return;
|
||||
}
|
||||
sum += len;
|
||||
}
|
||||
if (!ziplistSafeToAdd(o->ptr, sum))
|
||||
hashTypeConvert(o, OBJ_ENCODING_HT);
|
||||
}
|
||||
|
||||
/* Get the value from a ziplist encoded hash, identified by field.
|
||||
|
||||
30
src/t_list.c
30
src/t_list.c
@ -29,6 +29,8 @@
|
||||
|
||||
#include "server.h"
|
||||
|
||||
#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024)
|
||||
|
||||
/*-----------------------------------------------------------------------------
|
||||
* List API
|
||||
*----------------------------------------------------------------------------*/
|
||||
@ -196,6 +198,14 @@ void listTypeConvert(robj *subject, int enc) {
|
||||
|
||||
void pushGenericCommand(client *c, int where) {
|
||||
int j, pushed = 0;
|
||||
|
||||
for (j = 2; j < c->argc; j++) {
|
||||
if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) {
|
||||
addReplyError(c, "Element too large");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
robj *lobj = lookupKeyWrite(c->db,c->argv[1]);
|
||||
|
||||
if (lobj && lobj->type != OBJ_LIST) {
|
||||
@ -277,6 +287,11 @@ void linsertCommand(client *c) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) {
|
||||
addReplyError(c, "Element too large");
|
||||
return;
|
||||
}
|
||||
|
||||
if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
|
||||
checkType(c,subject,OBJ_LIST)) return;
|
||||
|
||||
@ -344,6 +359,11 @@ void lsetCommand(client *c) {
|
||||
long index;
|
||||
robj *value = c->argv[3];
|
||||
|
||||
if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) {
|
||||
addReplyError(c, "Element too large");
|
||||
return;
|
||||
}
|
||||
|
||||
if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))
|
||||
return;
|
||||
|
||||
@ -510,6 +530,11 @@ void lposCommand(client *c) {
|
||||
int direction = LIST_TAIL;
|
||||
long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */
|
||||
|
||||
if (sdslen(ele->ptr) > LIST_MAX_ITEM_SIZE) {
|
||||
addReplyError(c, "Element too large");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Parse the optional arguments. */
|
||||
for (int j = 3; j < c->argc; j++) {
|
||||
char *opt = c->argv[j]->ptr;
|
||||
@ -610,6 +635,11 @@ void lremCommand(client *c) {
|
||||
long toremove;
|
||||
long removed = 0;
|
||||
|
||||
if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) {
|
||||
addReplyError(c, "Element too large");
|
||||
return;
|
||||
}
|
||||
|
||||
if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK))
|
||||
return;
|
||||
|
||||
|
||||
@ -40,6 +40,12 @@
|
||||
#define STREAM_ITEM_FLAG_DELETED (1<<0) /* Entry is deleted. Skip it. */
|
||||
#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1) /* Same fields as master entry. */
|
||||
|
||||
/* Don't let listpacks grow too big, even if the user config allows it.
|
||||
* doing so can lead to an overflow (trying to store more than 32bit length
|
||||
* into the listpack header), or actually an assertion since lpInsert
|
||||
* will return NULL. */
|
||||
#define STREAM_LISTPACK_MAX_SIZE (1<<30)
|
||||
|
||||
void streamFreeCG(streamCG *cg);
|
||||
void streamFreeNACK(streamNACK *na);
|
||||
size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
|
||||
@ -191,8 +197,11 @@ int streamCompareID(streamID *a, streamID *b) {
|
||||
*
|
||||
* The function returns C_OK if the item was added, this is always true
|
||||
* if the ID was generated by the function. However the function may return
|
||||
* C_ERR if an ID was given via 'use_id', but adding it failed since the
|
||||
* current top ID is greater or equal. */
|
||||
* C_ERR in several cases:
|
||||
* 1. If an ID was given via 'use_id', but adding it failed since the
|
||||
* current top ID is greater or equal. errno will be set to EDOM.
|
||||
* 2. If a size of a single element or the sum of the elements is too big to
|
||||
* be stored into the stream. errno will be set to ERANGE. */
|
||||
int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) {
|
||||
|
||||
/* Generate the new entry ID. */
|
||||
@ -206,7 +215,23 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
|
||||
* or return an error. Automatically generated IDs might
|
||||
* overflow (and wrap-around) when incrementing the sequence
|
||||
part. */
|
||||
if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR;
|
||||
if (streamCompareID(&id,&s->last_id) <= 0) {
|
||||
errno = EDOM;
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
/* Avoid overflow when trying to add an element to the stream (listpack
|
||||
* can only host up to 32bit length sttrings, and also a total listpack size
|
||||
* can't be bigger than 32bit length. */
|
||||
size_t totelelen = 0;
|
||||
for (int64_t i = 0; i < numfields*2; i++) {
|
||||
sds ele = argv[i]->ptr;
|
||||
totelelen += sdslen(ele);
|
||||
}
|
||||
if (totelelen > STREAM_LISTPACK_MAX_SIZE) {
|
||||
errno = ERANGE;
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
/* Add the new entry. */
|
||||
raxIterator ri;
|
||||
@ -265,9 +290,10 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
|
||||
* if we need to switch to the next one. 'lp' will be set to NULL if
|
||||
* the current node is full. */
|
||||
if (lp != NULL) {
|
||||
if (server.stream_node_max_bytes &&
|
||||
lp_bytes >= server.stream_node_max_bytes)
|
||||
{
|
||||
size_t node_max_bytes = server.stream_node_max_bytes;
|
||||
if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE)
|
||||
node_max_bytes = STREAM_LISTPACK_MAX_SIZE;
|
||||
if (lp_bytes + totelelen >= node_max_bytes) {
|
||||
lp = NULL;
|
||||
} else if (server.stream_node_max_entries) {
|
||||
int64_t count = lpGetInteger(lpFirst(lp));
|
||||
@ -1267,11 +1293,13 @@ void xaddCommand(client *c) {
|
||||
|
||||
/* Append using the low level function and return the ID. */
|
||||
if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,
|
||||
&id, id_given ? &id : NULL)
|
||||
== C_ERR)
|
||||
&id, id_given ? &id : NULL) == C_ERR)
|
||||
{
|
||||
addReplyError(c,"The ID specified in XADD is equal or smaller than the "
|
||||
"target stream top item");
|
||||
if (errno == EDOM)
|
||||
addReplyError(c,"The ID specified in XADD is equal or smaller than "
|
||||
"the target stream top item");
|
||||
else
|
||||
addReplyError(c,"Elements are too large to be stored");
|
||||
return;
|
||||
}
|
||||
addReplyStreamID(c,&id);
|
||||
|
||||
43
src/t_zset.c
43
src/t_zset.c
@ -1238,15 +1238,18 @@ void zsetConvert(robj *zobj, int encoding) {
|
||||
}
|
||||
|
||||
/* Convert the sorted set object into a ziplist if it is not already a ziplist
|
||||
* and if the number of elements and the maximum element size is within the
|
||||
* expected ranges. */
|
||||
void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) {
|
||||
* and if the number of elements and the maximum element size and total elements size
|
||||
* are within the expected ranges. */
|
||||
void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) {
|
||||
if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return;
|
||||
zset *zset = zobj->ptr;
|
||||
|
||||
if (zset->zsl->length <= server.zset_max_ziplist_entries &&
|
||||
maxelelen <= server.zset_max_ziplist_value)
|
||||
zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);
|
||||
maxelelen <= server.zset_max_ziplist_value &&
|
||||
ziplistSafeToAdd(NULL, totelelen))
|
||||
{
|
||||
zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);
|
||||
}
|
||||
}
|
||||
|
||||
/* Return (by reference) the score of the specified member of the sorted set
|
||||
@ -1355,20 +1358,28 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) {
|
||||
}
|
||||
return 1;
|
||||
} else if (!xx) {
|
||||
/* Optimize: check if the element is too large or the list
|
||||
/* check if the element is too large or the list
|
||||
* becomes too long *before* executing zzlInsert. */
|
||||
zobj->ptr = zzlInsert(zobj->ptr,ele,score);
|
||||
if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries ||
|
||||
sdslen(ele) > server.zset_max_ziplist_value)
|
||||
if (zzlLength(zobj->ptr)+1 > server.zset_max_ziplist_entries ||
|
||||
sdslen(ele) > server.zset_max_ziplist_value ||
|
||||
!ziplistSafeToAdd(zobj->ptr, sdslen(ele)))
|
||||
{
|
||||
zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
|
||||
if (newscore) *newscore = score;
|
||||
*flags |= ZADD_ADDED;
|
||||
return 1;
|
||||
} else {
|
||||
zobj->ptr = zzlInsert(zobj->ptr,ele,score);
|
||||
if (newscore) *newscore = score;
|
||||
*flags |= ZADD_ADDED;
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
*flags |= ZADD_NOP;
|
||||
return 1;
|
||||
}
|
||||
} else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
|
||||
}
|
||||
|
||||
/* Note that the above block handling ziplist would have either returned or
|
||||
* converted the key to skiplist. */
|
||||
if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
|
||||
zset *zs = zobj->ptr;
|
||||
zskiplistNode *znode;
|
||||
dictEntry *de;
|
||||
@ -2180,7 +2191,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
|
||||
zsetopsrc *src;
|
||||
zsetopval zval;
|
||||
sds tmp;
|
||||
size_t maxelelen = 0;
|
||||
size_t maxelelen = 0, totelelen = 0;
|
||||
robj *dstobj;
|
||||
zset *dstzset;
|
||||
zskiplistNode *znode;
|
||||
@ -2304,6 +2315,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
|
||||
tmp = zuiNewSdsFromValue(&zval);
|
||||
znode = zslInsert(dstzset->zsl,score,tmp);
|
||||
dictAdd(dstzset->dict,tmp,&znode->score);
|
||||
totelelen += sdslen(tmp);
|
||||
if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
|
||||
}
|
||||
}
|
||||
@ -2340,6 +2352,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
|
||||
/* Remember the longest single element encountered,
|
||||
* to understand if it's possible to convert to ziplist
|
||||
* at the end. */
|
||||
totelelen += sdslen(tmp);
|
||||
if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
|
||||
/* Update the element with its initial score. */
|
||||
dictSetKey(accumulator, de, tmp);
|
||||
@ -2380,7 +2393,7 @@ void zunionInterGenericCommand(client *c, robj *dstkey, int op) {
|
||||
if (dbDelete(c->db,dstkey))
|
||||
touched = 1;
|
||||
if (dstzset->zsl->length) {
|
||||
zsetConvertToZiplistIfNeeded(dstobj,maxelelen);
|
||||
zsetConvertToZiplistIfNeeded(dstobj,maxelelen,totelelen);
|
||||
dbAdd(c->db,dstkey,dstobj);
|
||||
addReplyLongLong(c,zsetLength(dstobj));
|
||||
signalModifiedKey(c,c->db,dstkey);
|
||||
|
||||
@ -265,6 +265,17 @@
|
||||
ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
|
||||
}
|
||||
|
||||
/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in
|
||||
* zlbytes*/
|
||||
#define ZIPLIST_MAX_SAFETY_SIZE (1<<30)
|
||||
int ziplistSafeToAdd(unsigned char* zl, size_t add) {
|
||||
size_t len = zl? ziplistBlobLen(zl): 0;
|
||||
if (len + add > ZIPLIST_MAX_SAFETY_SIZE)
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
/* We use this function to receive information about a ziplist entry.
|
||||
* Note that this is not how the data is actually encoded, is just what we
|
||||
* get filled by a function in order to operate more easily. */
|
||||
@ -586,7 +597,8 @@ unsigned char *ziplistNew(void) {
|
||||
}
|
||||
|
||||
/* Resize the ziplist. */
|
||||
unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
|
||||
unsigned char *ziplistResize(unsigned char *zl, size_t len) {
|
||||
assert(len < UINT32_MAX);
|
||||
zl = zrealloc(zl,len);
|
||||
ZIPLIST_BYTES(zl) = intrev32ifbe(len);
|
||||
zl[len-1] = ZIP_END;
|
||||
@ -898,6 +910,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
|
||||
/* Combined zl length should be limited within UINT16_MAX */
|
||||
zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
|
||||
|
||||
/* larger values can't be stored into ZIPLIST_BYTES */
|
||||
assert(zlbytes < UINT32_MAX);
|
||||
|
||||
/* Save offset positions before we start ripping memory apart. */
|
||||
size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
|
||||
size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
|
||||
|
||||
@ -49,6 +49,7 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v
|
||||
unsigned int ziplistLen(unsigned char *zl);
|
||||
size_t ziplistBlobLen(unsigned char *zl);
|
||||
void ziplistRepr(unsigned char *zl);
|
||||
int ziplistSafeToAdd(unsigned char* zl, size_t add);
|
||||
|
||||
#ifdef REDIS_TEST
|
||||
int ziplistTest(int argc, char *argv[]);
|
||||
|
||||
@ -109,7 +109,23 @@ proc wait_done_loading r {
|
||||
|
||||
# count current log lines in server's stdout
|
||||
proc count_log_lines {srv_idx} {
|
||||
set _ [exec wc -l < [srv $srv_idx stdout]]
|
||||
set _ [string trim [exec wc -l < [srv $srv_idx stdout]]]
|
||||
}
|
||||
|
||||
# returns the number of times a line with that pattern appears in a file
|
||||
proc count_message_lines {file pattern} {
|
||||
set res 0
|
||||
# exec fails when grep exists with status other than 0 (when the patter wasn't found)
|
||||
catch {
|
||||
set res [string trim [exec grep $pattern $file 2> /dev/null | wc -l]]
|
||||
}
|
||||
return $res
|
||||
}
|
||||
|
||||
# returns the number of times a line with that pattern appears in the log
|
||||
proc count_log_message {srv_idx pattern} {
|
||||
set stdout [srv $srv_idx stdout]
|
||||
return [count_message_lines $stdout $pattern]
|
||||
}
|
||||
|
||||
# verify pattern exists in server's sdtout after a certain line number
|
||||
|
||||
156
tests/unit/violations.tcl
Normal file
156
tests/unit/violations.tcl
Normal file
@ -0,0 +1,156 @@
|
||||
# These tests consume massive amounts of memory, and are not
|
||||
# suitable to be executed as part of the normal test suite
|
||||
set ::str500 [string repeat x 500000000] ;# 500mb
|
||||
|
||||
# Utility function to write big argument into redis client connection
|
||||
proc write_big_bulk {size} {
|
||||
r write "\$$size\r\n"
|
||||
while {$size >= 500000000} {
|
||||
r write $::str500
|
||||
incr size -500000000
|
||||
}
|
||||
if {$size > 0} {
|
||||
r write [string repeat x $size]
|
||||
}
|
||||
r write "\r\n"
|
||||
}
|
||||
|
||||
# One XADD with one huge 5GB field
|
||||
# Expected to fail resulting in an empty stream
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {XADD one huge field} {
|
||||
r config set proto-max-bulk-len 10000000000 ;#10gb
|
||||
r config set client-query-buffer-limit 10000000000 ;#10gb
|
||||
r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"
|
||||
r write "\$1\r\nA\r\n"
|
||||
write_big_bulk 5000000000 ;#5gb
|
||||
r flush
|
||||
catch {r read} err
|
||||
assert_match {*too large*} $err
|
||||
r xlen S1
|
||||
} {0}
|
||||
}
|
||||
|
||||
# One XADD with one huge (exactly nearly) 4GB field
|
||||
# This uncovers the overflow in lpEncodeGetType
|
||||
# Expected to fail resulting in an empty stream
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {XADD one huge field - 1} {
|
||||
r config set proto-max-bulk-len 10000000000 ;#10gb
|
||||
r config set client-query-buffer-limit 10000000000 ;#10gb
|
||||
r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n"
|
||||
r write "\$1\r\nA\r\n"
|
||||
write_big_bulk 4294967295 ;#4gb-1
|
||||
r flush
|
||||
catch {r read} err
|
||||
assert_match {*too large*} $err
|
||||
r xlen S1
|
||||
} {0}
|
||||
}
|
||||
|
||||
# Gradually add big stream fields using repeated XADD calls
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {several XADD big fields} {
|
||||
r config set stream-node-max-bytes 0
|
||||
for {set j 0} {$j<10} {incr j} {
|
||||
r xadd stream * 1 $::str500 2 $::str500
|
||||
}
|
||||
r ping
|
||||
r xlen stream
|
||||
} {10}
|
||||
}
|
||||
|
||||
# Add over 4GB to a single stream listpack (one XADD command)
|
||||
# Expected to fail resulting in an empty stream
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {single XADD big fields} {
|
||||
r write "*23\r\n\$4\r\nXADD\r\n\$1\r\nS\r\n\$1\r\n*\r\n"
|
||||
for {set j 0} {$j<10} {incr j} {
|
||||
r write "\$1\r\n$j\r\n"
|
||||
write_big_bulk 500000000 ;#500mb
|
||||
}
|
||||
r flush
|
||||
catch {r read} err
|
||||
assert_match {*too large*} $err
|
||||
r xlen S
|
||||
} {0}
|
||||
}
|
||||
|
||||
# Gradually add big hash fields using repeated HSET calls
|
||||
# This reproduces the overflow in the call to ziplistResize
|
||||
# Object will be converted to hashtable encoding
|
||||
start_server [list overrides [list save ""] ] {
|
||||
r config set hash-max-ziplist-value 1000000000 ;#1gb
|
||||
test {hash with many big fields} {
|
||||
for {set j 0} {$j<10} {incr j} {
|
||||
r hset h $j $::str500
|
||||
}
|
||||
r object encoding h
|
||||
} {hashtable}
|
||||
}
|
||||
|
||||
# Add over 4GB to a single hash field (one HSET command)
|
||||
# Object will be converted to hashtable encoding
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {hash with one huge field} {
|
||||
catch {r config set hash-max-ziplist-value 10000000000} ;#10gb
|
||||
r config set proto-max-bulk-len 10000000000 ;#10gb
|
||||
r config set client-query-buffer-limit 10000000000 ;#10gb
|
||||
r write "*4\r\n\$4\r\nHSET\r\n\$2\r\nH1\r\n"
|
||||
r write "\$1\r\nA\r\n"
|
||||
write_big_bulk 5000000000 ;#5gb
|
||||
r flush
|
||||
r read
|
||||
r object encoding H1
|
||||
} {hashtable}
|
||||
}
|
||||
|
||||
# Add over 4GB to a single list member (one LPUSH command)
|
||||
# Currently unsupported, and expected to fail rather than being truncated
|
||||
# Expected to fail resulting in a non-existing list
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {list with one huge field} {
|
||||
r config set proto-max-bulk-len 10000000000 ;#10gb
|
||||
r config set client-query-buffer-limit 10000000000 ;#10gb
|
||||
r write "*3\r\n\$5\r\nLPUSH\r\n\$2\r\nL1\r\n"
|
||||
write_big_bulk 5000000000 ;#5gb
|
||||
r flush
|
||||
catch {r read} err
|
||||
assert_match {*too large*} $err
|
||||
r exists L1
|
||||
} {0}
|
||||
}
|
||||
|
||||
# SORT which attempts to store an element larger than 4GB into a list.
|
||||
# Currently unsupported and results in an assertion instead of truncation
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {SORT adds huge field to list} {
|
||||
r config set proto-max-bulk-len 10000000000 ;#10gb
|
||||
r config set client-query-buffer-limit 10000000000 ;#10gb
|
||||
r write "*3\r\n\$3\r\nSET\r\n\$2\r\nS1\r\n"
|
||||
write_big_bulk 5000000000 ;#5gb
|
||||
r flush
|
||||
r read
|
||||
assert_equal [r strlen S1] 5000000000
|
||||
r set S2 asdf
|
||||
r sadd myset 1 2
|
||||
r mset D1 1 D2 2
|
||||
catch {r sort myset by D* get S* store mylist}
|
||||
# assert_equal [count_log_message 0 "crashed by signal"] 0 - not suitable for 6.0
|
||||
assert_equal [count_log_message 0 "ASSERTION FAILED"] 1
|
||||
}
|
||||
}
|
||||
|
||||
# SORT which stores an integer encoded element into a list.
|
||||
# Just for coverage, no news here.
|
||||
start_server [list overrides [list save ""] ] {
|
||||
test {SORT adds integer field to list} {
|
||||
r set S1 asdf
|
||||
r set S2 123 ;# integer encoded
|
||||
assert_encoding "int" S2
|
||||
r sadd myset 1 2
|
||||
r mset D1 1 D2 2
|
||||
r sort myset by D* get S* store mylist
|
||||
r llen mylist
|
||||
} {2}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user