mirrors/valkey
1
0
Fork 0
mirror of http://github.com/valkey-io/valkey synced 2024-11-22 00:52:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

memory reporting of clients argv (#7874)

Browse Source 
track and report memory used by clients argv.
this is very usaful in case clients started sending a command and didn't
complete it. in which case the first args of the command are already
trimmed from the query buffer.

in an effort to avoid cache misses and overheads while keeping track of
these, i avoid calling sdsZmallocSize and instead use the sdslen /
bulk-len which can at least give some insight into the problem.

This memory is now added to the total clients memory usage, as well as
the client list.

(cherry picked from commit bea40e6a41)
This commit is contained in:
Oran Agra 2020-10-05 11:15:36 +03:00
parent 93b4c6b31e
commit 76f3a63de1
5 changed files with 55 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/aof.c
View File

@ -669,6 +669,7 @@ struct client *createAOFClient(void) {
c->querybuf_peak = 0;
c->argc = 0;
c->argv = NULL;
c->argv_len_sum = 0;
c->bufpos = 0;
c->flags = 0;
c->btype = BLOCKED_NONE;
@ -694,6 +695,7 @@ void freeFakeClientArgv(struct client *c) {
for (j = 0; j < c->argc; j++)
decrRefCount(c->argv[j]);
zfree(c->argv);
c->argv_len_sum = 0;
}
void freeFakeClient(struct client *c) {

51
src/networking.c
View File

@ -48,7 +48,7 @@ size_t sdsZmallocSize(sds s) {
}
/* Return the amount of memory used by the sds string at object->ptr
* for a string object. */
* for a string object. This includes internal fragmentation. */
size_t getStringObjectSdsUsedMemory(robj *o) {
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
switch(o->encoding) {
@ -58,6 +58,17 @@ size_t getStringObjectSdsUsedMemory(robj *o) {
}
}
/* Return the length of a string object.
* This does NOT includes internal fragmentation or sds unused space. */
size_t getStringObjectLen(robj *o) {
serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);
switch(o->encoding) {
case OBJ_ENCODING_RAW: return sdslen(o->ptr);
case OBJ_ENCODING_EMBSTR: return sdslen(o->ptr);
default: return 0; /* Just integer encoding for now. */
}
}
/* Client.reply list dup and free methods. */
void *dupClientReplyValue(void *o) {
clientReplyBlock *old = o;
@ -116,6 +127,7 @@ client *createClient(connection *conn) {
c->reqtype = 0;
c->argc = 0;
c->argv = NULL;
c->argv_len_sum = 0;
c->cmd = c->lastcmd = NULL;
c->user = DefaultUser;
c->multibulklen = 0;
@ -1051,6 +1063,7 @@ static void freeClientArgv(client *c) {
decrRefCount(c->argv[j]);
c->argc = 0;
c->cmd = NULL;
c->argv_len_sum = 0;
}
/* Close all the slaves connections. This is useful in chained replication
@ -1249,6 +1262,7 @@ void freeClient(client *c) {
* and finally release the client structure itself. */
if (c->name) decrRefCount(c->name);
zfree(c->argv);
c->argv_len_sum = 0;
freeClientMultiState(c);
sdsfree(c->peerid);
zfree(c);
@ -1595,12 +1609,14 @@ int processInlineBuffer(client *c) {
if (argc) {
if (c->argv) zfree(c->argv);
c->argv = zmalloc(sizeof(robj*)*argc);
c->argv_len_sum = 0;
}
/* Create redis objects for all arguments. */
for (c->argc = 0, j = 0; j < argc; j++) {
c->argv[c->argc] = createObject(OBJ_STRING,argv[j]);
c->argc++;
c->argv_len_sum += sdslen(argv[j]);
}
zfree(argv);
return C_OK;
@ -1692,6 +1708,7 @@ int processMultibulkBuffer(client *c) {
/* Setup argv array on client structure */
if (c->argv) zfree(c->argv);
c->argv = zmalloc(sizeof(robj*)*c->multibulklen);
c->argv_len_sum = 0;
}
serverAssertWithInfo(c,NULL,c->multibulklen > 0);
@ -1764,6 +1781,7 @@ int processMultibulkBuffer(client *c) {
sdslen(c->querybuf) == (size_t)(c->bulklen+2))
{
c->argv[c->argc++] = createObject(OBJ_STRING,c->querybuf);
c->argv_len_sum += c->bulklen;
sdsIncrLen(c->querybuf,-2); /* remove CRLF */
/* Assume that if we saw a fat argument we'll see another one
* likely... */
@ -1772,6 +1790,7 @@ int processMultibulkBuffer(client *c) {
} else {
c->argv[c->argc++] =
createStringObject(c->querybuf+c->qb_pos,c->bulklen);
c->argv_len_sum += c->bulklen;
c->qb_pos += c->bulklen+2;
}
c->bulklen = -1;
@ -2094,8 +2113,21 @@ sds catClientInfoString(sds s, client *client) {
if (connHasWriteHandler(client->conn)) *p++ = 'w';
}
*p = '\0';
/* Compute the total memory consumed by this client. */
size_t obufmem = getClientOutputBufferMemoryUsage(client);
size_t total_mem = obufmem;
total_mem += zmalloc_size(client); /* includes client->buf */
total_mem += sdsZmallocSize(client->querybuf);
/* For efficiency (less work keeping track of the argv memory), it doesn't include the used memory
* i.e. unused sds space and internal fragmentation, just the string length. but this is enough to
* spot problematic clients. */
total_mem += client->argv_len_sum;
if (client->argv)
total_mem += zmalloc_size(client->argv);
return sdscatfmt(s,
"id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s",
"id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U argv-mem=%U obl=%U oll=%U omem=%U tot-mem=%U events=%s cmd=%s user=%s",
(unsigned long long) client->id,
getClientPeerId(client),
connGetInfo(client->conn, conninfo, sizeof(conninfo)),
@ -2109,9 +2141,11 @@ sds catClientInfoString(sds s, client *client) {
(client->flags & CLIENT_MULTI) ? client->mstate.count : -1,
(unsigned long long) sdslen(client->querybuf),
(unsigned long long) sdsavail(client->querybuf),
(unsigned long long) client->argv_len_sum,
(unsigned long long) client->bufpos,
(unsigned long long) listLength(client->reply),
(unsigned long long) getClientOutputBufferMemoryUsage(client),
(unsigned long long) obufmem, /* should not include client->buf since we want to see 0 for static clients. */
(unsigned long long) total_mem,
events,
client->lastcmd ? client->lastcmd->name : "NULL",
client->user ? client->user->name : "(superuser)");
@ -2649,6 +2683,10 @@ void rewriteClientCommandVector(client *c, int argc, ...) {
/* Replace argv and argc with our new versions. */
c->argv = argv;
c->argc = argc;
c->argv_len_sum = 0;
for (j = 0; j < c->argc; j++)
if (c->argv[j])
c->argv_len_sum += getStringObjectLen(c->argv[j]);
c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr);
serverAssertWithInfo(c,NULL,c->cmd != NULL);
va_end(ap);
@ -2656,10 +2694,15 @@ void rewriteClientCommandVector(client *c, int argc, ...) {
/* Completely replace the client command vector with the provided one. */
void replaceClientCommandVector(client *c, int argc, robj **argv) {
int j;
freeClientArgv(c);
zfree(c->argv);
c->argv = argv;
c->argc = argc;
c->argv_len_sum = 0;
for (j = 0; j < c->argc; j++)
if (c->argv[j])
c->argv_len_sum += getStringObjectLen(c->argv[j]);
c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr);
serverAssertWithInfo(c,NULL,c->cmd != NULL);
}
@ -2684,6 +2727,8 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval) {
c->argv[i] = NULL;
}
oldval = c->argv[i];
if (oldval) c->argv_len_sum -= getStringObjectLen(oldval);
if (newval) c->argv_len_sum += getStringObjectLen(newval);
c->argv[i] = newval;
incrRefCount(newval);
if (oldval) decrRefCount(oldval);

4
src/server.c
View File

@ -1609,7 +1609,9 @@ int clientsCronTrackClientsMemUsage(client *c) {
int type = getClientType(c);
mem += getClientOutputBufferMemoryUsage(c);
mem += sdsZmallocSize(c->querybuf);
mem += sizeof(client);
mem += zmalloc_size(c);
mem += c->argv_len_sum;
if (c->argv) mem += zmalloc_size(c->argv);
/* Now that we have the memory used by the client, remove the old
* value from the old category, and add it back. */
server.stat_clients_type_memory[c->client_cron_last_memory_type] -=

1
src/server.h
View File

@ -799,6 +799,7 @@ typedef struct client {
size_t querybuf_peak; /* Recent (100ms or more) peak of querybuf size. */
int argc; /* Num of arguments of current command. */
robj **argv; /* Arguments of current command. */
size_t argv_len_sum; /* Sum of lengths of objects in argv list. */
struct redisCommand *cmd, *lastcmd; /* Last command executed. */
user *user; /* User associated with this connection. If the
user is set to NULL the connection can do

2
tests/unit/introspection.tcl
View File

@ -1,7 +1,7 @@
start_server {tags {"introspection"}} {
test {CLIENT LIST} {
r client list
} {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*}
} {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* argv-mem=* obl=0 oll=0 omem=0 tot-mem=* events=r cmd=client*}
test {MONITOR can log executed commands} {
set rd [redis_deferring_client]