From 2257f38b68826576044814283fe8a6cfb162c2e1 Mon Sep 17 00:00:00 2001 From: "Meir Shpilraien (Spielrein)" Date: Thu, 23 Jul 2020 12:38:51 +0300 Subject: [PATCH] This PR introduces a new loaded keyspace event (#7536) Co-authored-by: Oran Agra Co-authored-by: Itamar Haber (cherry picked from commit 8d826393191399e132bd9e56fb51ed83223cc5ca) --- runtest-moduleapi | 1 + src/module.c | 5 ++ src/rdb.c | 6 +- src/redismodule.h | 2 +- src/server.h | 1 + tests/modules/Makefile | 3 +- tests/modules/keyspace_events.c | 99 ++++++++++++++++++++++++ tests/unit/moduleapi/keyspace_events.tcl | 22 ++++++ 8 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 tests/modules/keyspace_events.c create mode 100644 tests/unit/moduleapi/keyspace_events.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index f6cc0a258..71db27e5e 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -25,4 +25,5 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/scan \ --single unit/moduleapi/datatype \ --single unit/moduleapi/auth \ +--single unit/moduleapi/keyspace_events \ "${@}" diff --git a/src/module.c b/src/module.c index 9755e282a..b15bb7276 100644 --- a/src/module.c +++ b/src/module.c @@ -4864,6 +4864,11 @@ void moduleReleaseGIL(void) { * - REDISMODULE_NOTIFY_STREAM: Stream events * - REDISMODULE_NOTIFY_KEYMISS: Key-miss events * - REDISMODULE_NOTIFY_ALL: All events (Excluding REDISMODULE_NOTIFY_KEYMISS) + * - REDISMODULE_NOTIFY_LOADED: A special notification available only for modules, + * indicates that the key was loaded from persistence. + * Notice, when this event fires, the given key + * can not be retained, use RM_CreateStringFromString + * instead. * * We do not distinguish between key events and keyspace events, and it is up * to the module to filter the actions taken based on the key. diff --git a/src/rdb.c b/src/rdb.c index ac1985d24..54a169cd8 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2307,6 +2307,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { decrRefCount(val); } else { robj keyobj; + initStaticStringObject(keyobj,key); /* Add the new object in the hash table */ int added = dbAddRDBLoad(db,key,val); @@ -2315,7 +2316,6 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { /* This flag is useful for DEBUG RELOAD special modes. * When it's set we allow new keys to replace the current * keys with the same name. */ - initStaticStringObject(keyobj,key); dbSyncDelete(db,&keyobj); dbAddRDBLoad(db,key,val); } else { @@ -2327,12 +2327,14 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { /* Set the expire time if needed */ if (expiretime != -1) { - initStaticStringObject(keyobj,key); setExpire(NULL,db,&keyobj,expiretime); } /* Set usage information (for eviction). */ objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000); + + /* call key space notification on key loaded for modules only */ + moduleNotifyKeyspaceEvent(NOTIFY_LOADED, "loaded", &keyobj, db->id); } /* Loading the database more slowly is useful in order to test diff --git a/src/redismodule.h b/src/redismodule.h index d67b01f68..ffc679ebc 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -128,9 +128,9 @@ #define REDISMODULE_NOTIFY_EVICTED (1<<9) /* e */ #define REDISMODULE_NOTIFY_STREAM (1<<10) /* t */ #define REDISMODULE_NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from REDISMODULE_NOTIFY_ALL on purpose) */ +#define REDISMODULE_NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */ #define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED | REDISMODULE_NOTIFY_STREAM) /* A */ - /* A special pointer that we can use between the core and the module to signal * field deletion, and that is impossible to be a valid pointer. */ #define REDISMODULE_HASH_DELETE ((RedisModuleString*)(long)1) diff --git a/src/server.h b/src/server.h index ba6dffcef..e9b4777ef 100644 --- a/src/server.h +++ b/src/server.h @@ -431,6 +431,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define NOTIFY_EVICTED (1<<9) /* e */ #define NOTIFY_STREAM (1<<10) /* t */ #define NOTIFY_KEY_MISS (1<<11) /* m (Note: This one is excluded from NOTIFY_ALL on purpose) */ +#define NOTIFY_LOADED (1<<12) /* module only key space notification, indicate a key loaded from rdb */ #define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED | NOTIFY_STREAM) /* A flag */ /* Get the first bind addr or NULL */ diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 39b8e6efa..de7407a84 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -22,7 +22,8 @@ TEST_MODULES = \ blockonkeys.so \ scan.so \ datatype.so \ - auth.so + auth.so \ + keyspace_events.so .PHONY: all diff --git a/tests/modules/keyspace_events.c b/tests/modules/keyspace_events.c new file mode 100644 index 000000000..b2296c1cb --- /dev/null +++ b/tests/modules/keyspace_events.c @@ -0,0 +1,99 @@ +/* This module is used to test the server keyspace events API. + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2020, Meir Shpilraien + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API + +#include "redismodule.h" +#include +#include + +/** strores all the keys on which we got 'loaded' keyspace notification **/ +RedisModuleDict *loaded_event_log = NULL; + +static int KeySpace_Notification(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key){ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(type); + + if(strcmp(event, "loaded") == 0){ + const char* keyName = RedisModule_StringPtrLen(key, NULL); + int nokey; + RedisModule_DictGetC(loaded_event_log, (void*)keyName, strlen(keyName), &nokey); + if(nokey){ + RedisModule_DictSetC(loaded_event_log, (void*)keyName, strlen(keyName), NULL); + } + } + + return REDISMODULE_OK; +} + +static int cmdIsKeyLoaded(RedisModuleCtx *ctx, RedisModuleString **argv, int argc){ + if(argc != 2){ + return RedisModule_WrongArity(ctx); + } + + const char* key = RedisModule_StringPtrLen(argv[1], NULL); + + int nokey; + RedisModule_DictGetC(loaded_event_log, (void*)key, strlen(key), &nokey); + + RedisModule_ReplyWithLongLong(ctx, !nokey); + return REDISMODULE_OK; +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"testkeyspace",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR){ + return REDISMODULE_ERR; + } + + loaded_event_log = RedisModule_CreateDict(ctx); + + if(RedisModule_SubscribeToKeyspaceEvents(ctx, REDISMODULE_NOTIFY_LOADED, KeySpace_Notification) != REDISMODULE_OK){ + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx,"keyspace.is_key_loaded", cmdIsKeyLoaded,"",0,0,0) == REDISMODULE_ERR){ + return REDISMODULE_ERR; + } + + return REDISMODULE_OK; +} + +int RedisModule_OnUnload(RedisModuleCtx *ctx) { + RedisModule_FreeDict(ctx, loaded_event_log); + loaded_event_log = NULL; + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/keyspace_events.tcl b/tests/unit/moduleapi/keyspace_events.tcl new file mode 100644 index 000000000..cb959ab52 --- /dev/null +++ b/tests/unit/moduleapi/keyspace_events.tcl @@ -0,0 +1,22 @@ +set testmodule [file normalize tests/modules/keyspace_events.so] + +tags "modules" { + start_server [list overrides [list loadmodule "$testmodule"]] { + + test {Test loaded key space event} { + r set x 1 + r hset y f v + r lpush z 1 2 3 + r sadd p 1 2 3 + r zadd t 1 f1 2 f2 + r xadd s * f v + r debug reload + assert_equal 1 [r keyspace.is_key_loaded x] + assert_equal 1 [r keyspace.is_key_loaded y] + assert_equal 1 [r keyspace.is_key_loaded z] + assert_equal 1 [r keyspace.is_key_loaded p] + assert_equal 1 [r keyspace.is_key_loaded t] + assert_equal 1 [r keyspace.is_key_loaded s] + } + } +} \ No newline at end of file