mirror of
http://github.com/valkey-io/valkey
synced 2024-11-22 00:52:38 +00:00
Make sure that fork child doesn't do incremental rehashing (#11692)
Turns out that a fork child calling getExpire while persisting keys (and
possibly also a result of some module fork tasks) could cause dictFind
to do incremental rehashing in the child process, which is both a waste
of time, and also causes COW harm.
(cherry picked from commit 2bec254d89
)
This commit is contained in:
parent
574a49b96c
commit
3e82bdf738
33
src/dict.c
33
src/dict.c
@ -47,15 +47,15 @@
|
|||||||
#include "zmalloc.h"
|
#include "zmalloc.h"
|
||||||
#include "redisassert.h"
|
#include "redisassert.h"
|
||||||
|
|
||||||
/* Using dictEnableResize() / dictDisableResize() we make possible to
|
/* Using dictEnableResize() / dictDisableResize() we make possible to disable
|
||||||
* enable/disable resizing of the hash table as needed. This is very important
|
* resizing and rehashing of the hash table as needed. This is very important
|
||||||
* for Redis, as we use copy-on-write and don't want to move too much memory
|
* for Redis, as we use copy-on-write and don't want to move too much memory
|
||||||
* around when there is a child performing saving operations.
|
* around when there is a child performing saving operations.
|
||||||
*
|
*
|
||||||
* Note that even when dict_can_resize is set to 0, not all resizes are
|
* Note that even when dict_can_resize is set to 0, not all resizes are
|
||||||
* prevented: a hash table is still allowed to grow if the ratio between
|
* prevented: a hash table is still allowed to grow if the ratio between
|
||||||
* the number of elements and the buckets > dict_force_resize_ratio. */
|
* the number of elements and the buckets > dict_force_resize_ratio. */
|
||||||
static int dict_can_resize = 1;
|
static dictResizeEnable dict_can_resize = DICT_RESIZE_ENABLE;
|
||||||
static unsigned int dict_force_resize_ratio = 5;
|
static unsigned int dict_force_resize_ratio = 5;
|
||||||
|
|
||||||
/* -------------------------- private prototypes ---------------------------- */
|
/* -------------------------- private prototypes ---------------------------- */
|
||||||
@ -127,7 +127,7 @@ int dictResize(dict *d)
|
|||||||
{
|
{
|
||||||
unsigned long minimal;
|
unsigned long minimal;
|
||||||
|
|
||||||
if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
|
if (dict_can_resize != DICT_RESIZE_ENABLE || dictIsRehashing(d)) return DICT_ERR;
|
||||||
minimal = d->ht_used[0];
|
minimal = d->ht_used[0];
|
||||||
if (minimal < DICT_HT_INITIAL_SIZE)
|
if (minimal < DICT_HT_INITIAL_SIZE)
|
||||||
minimal = DICT_HT_INITIAL_SIZE;
|
minimal = DICT_HT_INITIAL_SIZE;
|
||||||
@ -210,7 +210,12 @@ int dictTryExpand(dict *d, unsigned long size) {
|
|||||||
* work it does would be unbound and the function may block for a long time. */
|
* work it does would be unbound and the function may block for a long time. */
|
||||||
int dictRehash(dict *d, int n) {
|
int dictRehash(dict *d, int n) {
|
||||||
int empty_visits = n*10; /* Max number of empty buckets to visit. */
|
int empty_visits = n*10; /* Max number of empty buckets to visit. */
|
||||||
if (!dictIsRehashing(d)) return 0;
|
if (dict_can_resize == DICT_RESIZE_FORBID || !dictIsRehashing(d)) return 0;
|
||||||
|
if (dict_can_resize == DICT_RESIZE_AVOID &&
|
||||||
|
(DICTHT_SIZE(d->ht_size_exp[1]) / DICTHT_SIZE(d->ht_size_exp[0]) < dict_force_resize_ratio))
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
while(n-- && d->ht_used[0] != 0) {
|
while(n-- && d->ht_used[0] != 0) {
|
||||||
dictEntry *de, *nextde;
|
dictEntry *de, *nextde;
|
||||||
@ -1000,10 +1005,12 @@ static int _dictExpandIfNeeded(dict *d)
|
|||||||
* table (global setting) or we should avoid it but the ratio between
|
* table (global setting) or we should avoid it but the ratio between
|
||||||
* elements/buckets is over the "safe" threshold, we resize doubling
|
* elements/buckets is over the "safe" threshold, we resize doubling
|
||||||
* the number of buckets. */
|
* the number of buckets. */
|
||||||
if (d->ht_used[0] >= DICTHT_SIZE(d->ht_size_exp[0]) &&
|
if (!dictTypeExpandAllowed(d))
|
||||||
(dict_can_resize ||
|
return DICT_OK;
|
||||||
d->ht_used[0]/ DICTHT_SIZE(d->ht_size_exp[0]) > dict_force_resize_ratio) &&
|
if ((dict_can_resize == DICT_RESIZE_ENABLE &&
|
||||||
dictTypeExpandAllowed(d))
|
d->ht_used[0] >= DICTHT_SIZE(d->ht_size_exp[0])) ||
|
||||||
|
(dict_can_resize != DICT_RESIZE_FORBID &&
|
||||||
|
d->ht_used[0] / DICTHT_SIZE(d->ht_size_exp[0]) > dict_force_resize_ratio))
|
||||||
{
|
{
|
||||||
return dictExpand(d, d->ht_used[0] + 1);
|
return dictExpand(d, d->ht_used[0] + 1);
|
||||||
}
|
}
|
||||||
@ -1063,12 +1070,8 @@ void dictEmpty(dict *d, void(callback)(dict*)) {
|
|||||||
d->pauserehash = 0;
|
d->pauserehash = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void dictEnableResize(void) {
|
void dictSetResizeEnabled(dictResizeEnable enable) {
|
||||||
dict_can_resize = 1;
|
dict_can_resize = enable;
|
||||||
}
|
|
||||||
|
|
||||||
void dictDisableResize(void) {
|
|
||||||
dict_can_resize = 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t dictGetHash(dict *d, const void *key) {
|
uint64_t dictGetHash(dict *d, const void *key) {
|
||||||
|
@ -169,6 +169,12 @@ typedef void (dictScanBucketFunction)(dict *d, dictEntry **bucketref);
|
|||||||
#define randomULong() random()
|
#define randomULong() random()
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
DICT_RESIZE_ENABLE,
|
||||||
|
DICT_RESIZE_AVOID,
|
||||||
|
DICT_RESIZE_FORBID,
|
||||||
|
} dictResizeEnable;
|
||||||
|
|
||||||
/* API */
|
/* API */
|
||||||
dict *dictCreate(dictType *type);
|
dict *dictCreate(dictType *type);
|
||||||
int dictExpand(dict *d, unsigned long size);
|
int dictExpand(dict *d, unsigned long size);
|
||||||
@ -195,8 +201,7 @@ void dictGetStats(char *buf, size_t bufsize, dict *d);
|
|||||||
uint64_t dictGenHashFunction(const void *key, size_t len);
|
uint64_t dictGenHashFunction(const void *key, size_t len);
|
||||||
uint64_t dictGenCaseHashFunction(const unsigned char *buf, size_t len);
|
uint64_t dictGenCaseHashFunction(const unsigned char *buf, size_t len);
|
||||||
void dictEmpty(dict *d, void(callback)(dict*));
|
void dictEmpty(dict *d, void(callback)(dict*));
|
||||||
void dictEnableResize(void);
|
void dictSetResizeEnabled(dictResizeEnable enable);
|
||||||
void dictDisableResize(void);
|
|
||||||
int dictRehash(dict *d, int n);
|
int dictRehash(dict *d, int n);
|
||||||
int dictRehashMilliseconds(dict *d, int ms);
|
int dictRehashMilliseconds(dict *d, int ms);
|
||||||
void dictSetHashFunctionSeed(uint8_t *seed);
|
void dictSetHashFunctionSeed(uint8_t *seed);
|
||||||
|
11
src/server.c
11
src/server.c
@ -594,13 +594,15 @@ int incrementallyRehash(int dbid) {
|
|||||||
* as we want to avoid resizing the hash tables when there is a child in order
|
* as we want to avoid resizing the hash tables when there is a child in order
|
||||||
* to play well with copy-on-write (otherwise when a resize happens lots of
|
* to play well with copy-on-write (otherwise when a resize happens lots of
|
||||||
* memory pages are copied). The goal of this function is to update the ability
|
* memory pages are copied). The goal of this function is to update the ability
|
||||||
* for dict.c to resize the hash tables accordingly to the fact we have an
|
* for dict.c to resize or rehash the tables accordingly to the fact we have an
|
||||||
* active fork child running. */
|
* active fork child running. */
|
||||||
void updateDictResizePolicy(void) {
|
void updateDictResizePolicy(void) {
|
||||||
if (!hasActiveChildProcess())
|
if (server.in_fork_child != CHILD_TYPE_NONE)
|
||||||
dictEnableResize();
|
dictSetResizeEnabled(DICT_RESIZE_FORBID);
|
||||||
|
else if (hasActiveChildProcess())
|
||||||
|
dictSetResizeEnabled(DICT_RESIZE_AVOID);
|
||||||
else
|
else
|
||||||
dictDisableResize();
|
dictSetResizeEnabled(DICT_RESIZE_ENABLE);
|
||||||
}
|
}
|
||||||
|
|
||||||
const char *strChildType(int type) {
|
const char *strChildType(int type) {
|
||||||
@ -6417,6 +6419,7 @@ int redisFork(int purpose) {
|
|||||||
server.in_fork_child = purpose;
|
server.in_fork_child = purpose;
|
||||||
setupChildSignalHandlers();
|
setupChildSignalHandlers();
|
||||||
setOOMScoreAdj(CONFIG_OOM_BGCHILD);
|
setOOMScoreAdj(CONFIG_OOM_BGCHILD);
|
||||||
|
updateDictResizePolicy();
|
||||||
dismissMemoryInChild();
|
dismissMemoryInChild();
|
||||||
closeChildUnusedResourceAfterFork();
|
closeChildUnusedResourceAfterFork();
|
||||||
/* Close the reading part, so that if the parent crashes, the child will
|
/* Close the reading part, so that if the parent crashes, the child will
|
||||||
|
Loading…
Reference in New Issue
Block a user