From 51942be0a6c67f2fb262b5d8eca3adf7a5827415 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Sat, 5 Mar 2022 18:46:18 +0100 Subject: [PATCH] loading redis keys --- .../src/controllers/databaseConnections.js | 10 +++ .../api/src/proc/databaseConnectionProcess.js | 12 ++++ packages/types/engines.d.ts | 1 + packages/web/src/utility/metadataLoaders.ts | 13 ++++ .../web/src/widgets/DatabaseWidget.svelte | 5 +- packages/web/src/widgets/DbKeysSubTree.svelte | 16 +++++ packages/web/src/widgets/DbKeysTree.svelte | 8 +++ .../src/widgets/SqlObjectListWrapper.svelte | 19 ------ plugins/dbgate-plugin-redis/package.json | 1 + .../dbgate-plugin-redis/src/backend/driver.js | 61 ++++++++++++++++++- yarn.lock | 5 ++ 11 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 packages/web/src/widgets/DbKeysSubTree.svelte create mode 100644 packages/web/src/widgets/DbKeysTree.svelte delete mode 100644 packages/web/src/widgets/SqlObjectListWrapper.svelte diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 21baa660..a83cc6d0 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -151,6 +151,16 @@ module.exports = { return res.result; }, + loadKeys_meta: true, + async loadKeys({ conid, database, root }) { + const opened = await this.ensureOpened(conid, database); + const res = await this.sendRequest(opened, { msgtype: 'loadKeys', root }); + if (res.errorMessage) { + console.error(res.errorMessage); + } + return res.result || null; + }, + updateCollection_meta: true, async updateCollection({ conid, database, changeSet }) { const opened = await this.ensureOpened(conid, database); diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index c3ae8a3e..9b5721d9 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -183,6 +183,17 @@ async function handleCollectionData({ msgid, options }) { } } +async function handleLoadKeys({ msgid, root }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + try { + const result = await driver.loadKeys(systemConnection, root); + process.send({ msgtype: 'response', msgid, result }); + } catch (err) { + process.send({ msgtype: 'response', msgid, errorMessage: err.message }); + } +} + async function handleUpdateCollection({ msgid, changeSet }) { await waitConnected(); const driver = requireEngineDriver(storedConnection); @@ -248,6 +259,7 @@ const messageHandlers = { runScript: handleRunScript, updateCollection: handleUpdateCollection, collectionData: handleCollectionData, + loadKeys: handleLoadKeys, sqlPreview: handleSqlPreview, ping: handlePing, syncModel: handleSyncModel, diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 31504062..8d226816 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -75,6 +75,7 @@ export interface EngineDriver { name: string; }[] >; + loadKeys(pool, root: string): Promise; analyseFull(pool: any, serverVersion): Promise; analyseIncremental(pool: any, structure: DatabaseInfo, serverVersion): Promise; dialect: SqlDialect; diff --git a/packages/web/src/utility/metadataLoaders.ts b/packages/web/src/utility/metadataLoaders.ts index 4f6aa1bd..1fa2fbf8 100644 --- a/packages/web/src/utility/metadataLoaders.ts +++ b/packages/web/src/utility/metadataLoaders.ts @@ -78,6 +78,12 @@ const databaseListLoader = ({ conid }) => ({ }, }); +const databaseKeysLoader = ({ conid, database, root }) => ({ + url: 'database-connections/load-keys', + params: { conid, database, root }, + reloadTrigger: `database-keys-changed-${conid}-${database}`, +}); + const serverVersionLoader = ({ conid }) => ({ url: 'server-connections/version', params: { conid }, @@ -429,3 +435,10 @@ export function getAuthTypes(args) { export function useAuthTypes(args) { return useCore(authTypesLoader, args); } + +export function getDatabaseKeys(args) { + return getCore(databaseKeysLoader, args); +} +export function useDatabaseKeys(args) { + return useCore(databaseKeysLoader, args); +} diff --git a/packages/web/src/widgets/DatabaseWidget.svelte b/packages/web/src/widgets/DatabaseWidget.svelte index 6e3a28d2..807fe4ca 100644 --- a/packages/web/src/widgets/DatabaseWidget.svelte +++ b/packages/web/src/widgets/DatabaseWidget.svelte @@ -11,6 +11,7 @@ import WidgetColumnBar from './WidgetColumnBar.svelte'; import WidgetColumnBarItem from './WidgetColumnBarItem.svelte'; import SqlObjectList from './SqlObjectList.svelte'; + import DbKeysTree from './DbKeysTree.svelte'; export let hidden = false; @@ -49,7 +50,9 @@ {:else if driver?.databaseEngineTypes?.includes('keyvalue')} - + + + {/if} {:else} diff --git a/packages/web/src/widgets/DbKeysSubTree.svelte b/packages/web/src/widgets/DbKeysSubTree.svelte new file mode 100644 index 00000000..61721306 --- /dev/null +++ b/packages/web/src/widgets/DbKeysSubTree.svelte @@ -0,0 +1,16 @@ + + +{#each $keys || [] as key} +
+ {key.text} +
+{/each} diff --git a/packages/web/src/widgets/DbKeysTree.svelte b/packages/web/src/widgets/DbKeysTree.svelte new file mode 100644 index 00000000..b3303b17 --- /dev/null +++ b/packages/web/src/widgets/DbKeysTree.svelte @@ -0,0 +1,8 @@ + + + diff --git a/packages/web/src/widgets/SqlObjectListWrapper.svelte b/packages/web/src/widgets/SqlObjectListWrapper.svelte deleted file mode 100644 index a7f7852e..00000000 --- a/packages/web/src/widgets/SqlObjectListWrapper.svelte +++ /dev/null @@ -1,19 +0,0 @@ - - -{#if conid && (database || singleDatabase)} - -{:else} - - - -{/if} diff --git a/plugins/dbgate-plugin-redis/package.json b/plugins/dbgate-plugin-redis/package.json index 777e938a..75f1f19a 100644 --- a/plugins/dbgate-plugin-redis/package.json +++ b/plugins/dbgate-plugin-redis/package.json @@ -37,6 +37,7 @@ "webpack-cli": "^3.3.11" }, "dependencies": { + "async": "^3.2.3", "ioredis": "^4.28.5" } } diff --git a/plugins/dbgate-plugin-redis/src/backend/driver.js b/plugins/dbgate-plugin-redis/src/backend/driver.js index e48ab07f..c2870085 100644 --- a/plugins/dbgate-plugin-redis/src/backend/driver.js +++ b/plugins/dbgate-plugin-redis/src/backend/driver.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const async = require('async'); const stream = require('stream'); const driverBase = require('../frontend/driver'); const Analyser = require('./Analyser'); @@ -9,11 +10,14 @@ const driver = { ...driverBase, analyserClass: Analyser, async connect({ server, port, password, database }) { + let db = 0; + if (_.isString(database) && database.startsWith('db')) db = parseInt(database.substring(2)); + if (_.isNumber(database)) db = database; const pool = new Redis({ host: server, port, password, - db: 0, + db, }); return pool; }, @@ -62,6 +66,61 @@ const driver = { return _.range(16).map((index) => ({ name: `db${index}`, extInfo: info[`db${index}`], sortOrder: index })); }, + + async loadKeys(pool, root = '') { + const keys = await this.getKeys(pool, root); + const res = this.extractKeysFromLevel(root, keys); + await this.enrichKeyInfo(pool, res); + return res; + }, + + async getKeys(pool, root = '') { + const res = []; + let cursor = 0; + do { + const [strCursor, keys] = await pool.scan(cursor, 'MATCH', root ? `${root}:*` : '*', 'COUNT', 100); + res.push(...keys); + cursor = parseInt(strCursor); + } while (cursor > 0); + return res; + }, + + extractKeysFromLevel(root, keys) { + const prefix = root ? `${root}:` : ''; + const rootSplit = _.compact(root.split(':')); + const res = {}; + for (const key of keys) { + if (!key.startsWith(prefix)) continue; + const keySplit = key.split(':'); + if (keySplit.length > rootSplit.length) { + if (keySplit.length == rootSplit.length + 1) { + res[keySplit[rootSplit.length]] = { + text: keySplit[rootSplit.length], + key, + }; + } else { + res[keySplit[rootSplit.length]] = { + text: keySplit[rootSplit.length], + type: 'dir', + }; + } + } + } + return Object.values(res); + }, + + async enrichOneKeyInfo(pool, item) { + const type = await pool.type(item.key); + item.type = type; + }, + + async enrichKeyInfo(pool, levelInfo) { + await async.eachLimit( + levelInfo.filter((x) => x.key), + 10, + async (item) => await this.enrichOneKeyInfo(pool, item) + ); + }, }; module.exports = driver; diff --git a/yarn.lock b/yarn.lock index 61955250..9d96cc45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1856,6 +1856,11 @@ async@^2.6.2: dependencies: lodash "^4.17.14" +async@^3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9" + integrity sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g== + asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"