mirrors/valkey
1
0
Fork 0
mirror of http://github.com/valkey-io/valkey synced 2024-11-22 09:17:20 +00:00
Code Issues Packages Projects Releases Wiki Activity
e71be72745
valkey/tests/modules/testrdb.c
Meir Shpilraien (Spielrein) b43f254813
Avoid saving module aux on RDB if no aux data was saved by the module. (#11374) 
### Background

The issue is that when saving an RDB with module AUX data, the module AUX metadata
(moduleid, when, ...) is saved to the RDB even though the module did not saved any actual data.
This prevent loading the RDB in the absence of the module (although there is no actual data in
the RDB that requires the module to be loaded).

### Solution

The solution suggested in this PR is that module AUX will be saved on the RDB only if the module
actually saved something during `aux_save` function.

To support backward compatibility, we introduce `aux_save2` callback that acts the same as
`aux_save` with the tiny change of avoid saving the aux field if no data was actually saved by
the module. Modules can use the new API to make sure that if they have no data to save,
then it will be possible to load the created RDB even without the module.

### Concerns

A module may register for the aux load and save hooks just in order to be notified when
saving or loading starts or completed (there are better ways to do that, but it still possible
that someone used it).

However, if a module didn't save a single field in the save callback, it means it's not allowed
to read in the read callback, since it has no way to distinguish between empty and non-empty
payloads. furthermore, it means that if the module did that, it must never change it, since it'll
break compatibility with it's old RDB files, so this is really not a valid use case.

Since some modules (ones who currently save one field indicating an empty payload), need
to know if saving an empty payload is valid, and if Redis is gonna ignore an empty payload
or store it, we opted to add a new API (rather than change behavior of an existing API and
expect modules to check the redis version)

### Technical Details

To avoid saving AUX data on RDB, we change the code to first save the AUX metadata
(moduleid, when, ...) into a temporary buffer. The buffer is then flushed to the rio at the first
time the module makes a write operation inside the `aux_save` function. If the module saves
nothing (and `aux_save2` was used), the entire temporary buffer is simply dropped and no
data about this AUX field is saved to the RDB. This make it possible to load the RDB even in
the absence of the module.

Test was added to verify the fix.
2022-10-18 19:45:46 +03:00

406 lines
14 KiB
C
Raw Blame History

#include "redismodule.h"
#include <string.h>
#include <assert.h>
/* Module configuration, save aux or not? */
#define CONF_AUX_OPTION_NO_AUX 0
#define CONF_AUX_OPTION_SAVE2 1 << 0
#define CONF_AUX_OPTION_BEFORE_KEYSPACE 1 << 1
#define CONF_AUX_OPTION_AFTER_KEYSPACE 1 << 2
#define CONF_AUX_OPTION_NO_DATA 1 << 3
long long conf_aux_count = 0;
/* Registered type */
RedisModuleType *testrdb_type = NULL;
/* Global values to store and persist to aux */
RedisModuleString *before_str = NULL;
RedisModuleString *after_str = NULL;
/* Global values used to keep aux from db being loaded (in case of async_loading) */
RedisModuleString *before_str_temp = NULL;
RedisModuleString *after_str_temp = NULL;
/* Indicates whether there is an async replication in progress.
* We control this value from RedisModuleEvent_ReplAsyncLoad events. */
int async_loading = 0;
int n_aux_load_called = 0;
void replAsyncLoadCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data)
{
REDISMODULE_NOT_USED(e);
REDISMODULE_NOT_USED(data);
switch (sub) {
case REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_STARTED:
assert(async_loading == 0);
async_loading = 1;
break;
case REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_ABORTED:
/* Discard temp aux */
if (before_str_temp)
RedisModule_FreeString(ctx, before_str_temp);
if (after_str_temp)
RedisModule_FreeString(ctx, after_str_temp);
before_str_temp = NULL;
after_str_temp = NULL;
async_loading = 0;
break;
case REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_COMPLETED:
if (before_str)
RedisModule_FreeString(ctx, before_str);
if (after_str)
RedisModule_FreeString(ctx, after_str);
before_str = before_str_temp;
after_str = after_str_temp;
before_str_temp = NULL;
after_str_temp = NULL;
async_loading = 0;
break;
default:
assert(0);
}
}
void *testrdb_type_load(RedisModuleIO *rdb, int encver) {
int count = RedisModule_LoadSigned(rdb);
RedisModuleString *str = RedisModule_LoadString(rdb);
float f = RedisModule_LoadFloat(rdb);
long double ld = RedisModule_LoadLongDouble(rdb);
if (RedisModule_IsIOError(rdb)) {
RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb);
if (str)
RedisModule_FreeString(ctx, str);
return NULL;
}
/* Using the values only after checking for io errors. */
assert(count==1);
assert(encver==1);
assert(f==1.5f);
assert(ld==0.333333333333333333L);
return str;
}
void testrdb_type_save(RedisModuleIO *rdb, void *value) {
RedisModuleString *str = (RedisModuleString*)value;
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, str);
RedisModule_SaveFloat(rdb, 1.5);
RedisModule_SaveLongDouble(rdb, 0.333333333333333333L);
}
void testrdb_aux_save(RedisModuleIO *rdb, int when) {
if (!(conf_aux_count & CONF_AUX_OPTION_BEFORE_KEYSPACE)) assert(when == REDISMODULE_AUX_AFTER_RDB);
if (!(conf_aux_count & CONF_AUX_OPTION_AFTER_KEYSPACE)) assert(when == REDISMODULE_AUX_BEFORE_RDB);
assert(conf_aux_count!=CONF_AUX_OPTION_NO_AUX);
if (when == REDISMODULE_AUX_BEFORE_RDB) {
if (before_str) {
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, before_str);
} else {
RedisModule_SaveSigned(rdb, 0);
}
} else {
if (after_str) {
RedisModule_SaveSigned(rdb, 1);
RedisModule_SaveString(rdb, after_str);
} else {
RedisModule_SaveSigned(rdb, 0);
}
}
}
int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) {
assert(encver == 1);
if (!(conf_aux_count & CONF_AUX_OPTION_BEFORE_KEYSPACE)) assert(when == REDISMODULE_AUX_AFTER_RDB);
if (!(conf_aux_count & CONF_AUX_OPTION_AFTER_KEYSPACE)) assert(when == REDISMODULE_AUX_BEFORE_RDB);
assert(conf_aux_count!=CONF_AUX_OPTION_NO_AUX);
RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb);
if (when == REDISMODULE_AUX_BEFORE_RDB) {
if (async_loading == 0) {
if (before_str)
RedisModule_FreeString(ctx, before_str);
before_str = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
before_str = RedisModule_LoadString(rdb);
} else {
if (before_str_temp)
RedisModule_FreeString(ctx, before_str_temp);
before_str_temp = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
before_str_temp = RedisModule_LoadString(rdb);
}
} else {
if (async_loading == 0) {
if (after_str)
RedisModule_FreeString(ctx, after_str);
after_str = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
after_str = RedisModule_LoadString(rdb);
} else {
if (after_str_temp)
RedisModule_FreeString(ctx, after_str_temp);
after_str_temp = NULL;
int count = RedisModule_LoadSigned(rdb);
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
if (count)
after_str_temp = RedisModule_LoadString(rdb);
}
}
if (RedisModule_IsIOError(rdb))
return REDISMODULE_ERR;
return REDISMODULE_OK;
}
void testrdb_type_free(void *value) {
if (value)
RedisModule_FreeString(NULL, (RedisModuleString*)value);
}
int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2) {
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (before_str)
RedisModule_FreeString(ctx, before_str);
before_str = argv[1];
RedisModule_RetainString(ctx, argv[1]);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(argv);
if (argc != 1){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (before_str)
RedisModule_ReplyWithString(ctx, before_str);
else
RedisModule_ReplyWithStringBuffer(ctx, "", 0);
return REDISMODULE_OK;
}
/* For purpose of testing module events, expose variable state during async_loading. */
int testrdb_async_loading_get_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(argv);
if (argc != 1){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (before_str_temp)
RedisModule_ReplyWithString(ctx, before_str_temp);
else
RedisModule_ReplyWithStringBuffer(ctx, "", 0);
return REDISMODULE_OK;
}
int testrdb_set_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (after_str)
RedisModule_FreeString(ctx, after_str);
after_str = argv[1];
RedisModule_RetainString(ctx, argv[1]);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(argv);
if (argc != 1){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
if (after_str)
RedisModule_ReplyWithString(ctx, after_str);
else
RedisModule_ReplyWithStringBuffer(ctx, "", 0);
return REDISMODULE_OK;
}
int testrdb_set_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 3){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE);
RedisModuleString *str = RedisModule_ModuleTypeGetValue(key);
if (str)
RedisModule_FreeString(ctx, str);
RedisModule_ModuleTypeSetValue(key, testrdb_type, argv[2]);
RedisModule_RetainString(ctx, argv[2]);
RedisModule_CloseKey(key);
RedisModule_ReplyWithLongLong(ctx, 1);
return REDISMODULE_OK;
}
int testrdb_get_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
if (argc != 2){
RedisModule_WrongArity(ctx);
return REDISMODULE_OK;
}
RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ);
RedisModuleString *str = RedisModule_ModuleTypeGetValue(key);
RedisModule_CloseKey(key);
RedisModule_ReplyWithString(ctx, str);
return REDISMODULE_OK;
}
int testrdb_get_n_aux_load_called(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
{
REDISMODULE_NOT_USED(ctx);
REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc);
RedisModule_ReplyWithLongLong(ctx, n_aux_load_called);
return REDISMODULE_OK;
}
int test2rdb_aux_load(RedisModuleIO *rdb, int encver, int when) {
REDISMODULE_NOT_USED(rdb);
REDISMODULE_NOT_USED(encver);
REDISMODULE_NOT_USED(when);
n_aux_load_called++;
return REDISMODULE_OK;
}
void test2rdb_aux_save(RedisModuleIO *rdb, int when) {
REDISMODULE_NOT_USED(rdb);
REDISMODULE_NOT_USED(when);
}
int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) {
REDISMODULE_NOT_USED(argv);
REDISMODULE_NOT_USED(argc);
if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS | REDISMODULE_OPTIONS_HANDLE_REPL_ASYNC_LOAD);
if (argc > 0)
RedisModule_StringToLongLong(argv[0], &conf_aux_count);
if (conf_aux_count==CONF_AUX_OPTION_NO_AUX) {
RedisModuleTypeMethods datatype_methods = {
.version = 1,
.rdb_load = testrdb_type_load,
.rdb_save = testrdb_type_save,
.aof_rewrite = NULL,
.digest = NULL,
.free = testrdb_type_free,
};
testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods);
if (testrdb_type == NULL)
return REDISMODULE_ERR;
} else if (!(conf_aux_count & CONF_AUX_OPTION_NO_DATA)) {
RedisModuleTypeMethods datatype_methods = {
.version = REDISMODULE_TYPE_METHOD_VERSION,
.rdb_load = testrdb_type_load,
.rdb_save = testrdb_type_save,
.aof_rewrite = NULL,
.digest = NULL,
.free = testrdb_type_free,
.aux_load = testrdb_aux_load,
.aux_save = testrdb_aux_save,
.aux_save_triggers = ((conf_aux_count & CONF_AUX_OPTION_BEFORE_KEYSPACE) ? REDISMODULE_AUX_BEFORE_RDB : 0) |
((conf_aux_count & CONF_AUX_OPTION_AFTER_KEYSPACE) ? REDISMODULE_AUX_AFTER_RDB : 0)
};
if (conf_aux_count & CONF_AUX_OPTION_SAVE2) {
datatype_methods.aux_save2 = testrdb_aux_save;
}
testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods);
if (testrdb_type == NULL)
return REDISMODULE_ERR;
} else {
/* Used to verify that aux_save2 api without any data, saves nothing to the RDB. */
RedisModuleTypeMethods datatype_methods = {
.version = REDISMODULE_TYPE_METHOD_VERSION,
.aux_load = test2rdb_aux_load,
.aux_save = test2rdb_aux_save,
.aux_save_triggers = ((conf_aux_count & CONF_AUX_OPTION_BEFORE_KEYSPACE) ? REDISMODULE_AUX_BEFORE_RDB : 0) |
((conf_aux_count & CONF_AUX_OPTION_AFTER_KEYSPACE) ? REDISMODULE_AUX_AFTER_RDB : 0)
};
if (conf_aux_count & CONF_AUX_OPTION_SAVE2) {
datatype_methods.aux_save2 = test2rdb_aux_save;
}
RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods);
}
if (RedisModule_CreateCommand(ctx,"testrdb.set.before", testrdb_set_before,"deny-oom",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.before", testrdb_get_before,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.async_loading.get.before", testrdb_async_loading_get_before,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.set.after", testrdb_set_after,"deny-oom",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.after", testrdb_get_after,"",0,0,0) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.set.key", testrdb_set_key,"deny-oom",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.key", testrdb_get_key,"",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
if (RedisModule_CreateCommand(ctx,"testrdb.get.n_aux_load_called", testrdb_get_n_aux_load_called,"",1,1,1) == REDISMODULE_ERR)
return REDISMODULE_ERR;
RedisModule_SubscribeToServerEvent(ctx,
RedisModuleEvent_ReplAsyncLoad, replAsyncLoadCallback);
return REDISMODULE_OK;
}
int RedisModule_OnUnload(RedisModuleCtx *ctx) {
if (before_str)
RedisModule_FreeString(ctx, before_str);
if (after_str)
RedisModule_FreeString(ctx, after_str);
if (before_str_temp)
RedisModule_FreeString(ctx, before_str_temp);
if (after_str_temp)
RedisModule_FreeString(ctx, after_str_temp);
return REDISMODULE_OK;
}
Reference in New Issue View Git Blame Copy Permalink