From 4ea6595ca7540e7747286aeb8166dbe1dd2ba9f2 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Thu, 19 Sep 2024 14:15:22 +0200 Subject: [PATCH] postgre analyser supports compisite db names --- .../__tests__/schema-tests.spec.js | 21 ++++++++++++++++++- packages/tools/src/schemaInfoTools.ts | 13 ++++++++++++ .../src/backend/Analyser.js | 2 +- .../src/backend/driver.js | 3 ++- .../dbgate-plugin-mongo/src/backend/driver.js | 1 - .../dbgate-plugin-mssql/src/backend/driver.js | 11 +++++++--- .../src/backend/Analyser.js | 10 +++++++-- .../src/backend/drivers.js | 1 + .../src/backend/sql/columns.js | 1 + .../src/backend/sql/fk_key_column_usage.js | 2 +- .../backend/sql/fk_referential_constraints.js | 1 + .../src/backend/sql/fk_table_constraints.js | 2 +- .../src/backend/sql/geographyColumns.js | 2 +- .../src/backend/sql/geometryColumns.js | 2 +- .../src/backend/sql/indexcols.js | 1 + .../src/backend/sql/indexes.js | 1 + .../src/backend/sql/matviewColumns.js | 1 + .../src/backend/sql/matviewModifications.js | 2 +- .../src/backend/sql/matviews.js | 1 + .../src/backend/sql/primaryKeys.js | 1 + .../src/backend/sql/routineModifications.js | 2 +- .../src/backend/sql/routines.js | 1 + .../src/backend/sql/tableList.js | 1 + .../src/backend/sql/tableModifications.js | 1 + .../src/backend/sql/uniqueNames.js | 2 +- .../src/backend/sql/viewModifications.js | 2 +- .../src/backend/sql/views.js | 2 +- 27 files changed, 72 insertions(+), 18 deletions(-) diff --git a/integration-tests/__tests__/schema-tests.spec.js b/integration-tests/__tests__/schema-tests.spec.js index cb823d09..8aa4b962 100644 --- a/integration-tests/__tests__/schema-tests.spec.js +++ b/integration-tests/__tests__/schema-tests.spec.js @@ -1,7 +1,7 @@ const stableStringify = require('json-stable-stringify'); const _ = require('lodash'); const fp = require('lodash/fp'); -const { testWrapper } = require('../tools'); +const { testWrapper, extractConnection } = require('../tools'); const engines = require('../engines'); const { runCommandOnDriver } = require('dbgate-tools'); @@ -54,6 +54,25 @@ describe('Schema tests', () => { expect(structure2).toBeNull(); }) ); + + test.each(engines.filter(x => x.supportSchemas).map(engine => [engine.label, engine]))( + 'Table inside schema - %s', + testWrapper(async (conn, driver, engine) => { + await baseStructure(conn, driver); + await runCommandOnDriver(conn, driver, dmp => dmp.createSchema('myschema')); + + const schemaConnDef = { + ...extractConnection(engine), + database: `${conn._database_name}::myschema`, + }; + + const schemaConn = await driver.connect(schemaConnDef); + await driver.query(schemaConn, `create table myschema.myt1 (id int not null primary key)`); + const structure1 = await driver.analyseFull(schemaConn); + expect(structure1.tables.length).toEqual(1); + expect(structure1.tables[0].tableName).toEqual('myt1'); + }) + ); }); describe('Base analyser test', () => { diff --git a/packages/tools/src/schemaInfoTools.ts b/packages/tools/src/schemaInfoTools.ts index 298b4cac..ad82ab04 100644 --- a/packages/tools/src/schemaInfoTools.ts +++ b/packages/tools/src/schemaInfoTools.ts @@ -13,3 +13,16 @@ export function findDefaultSchema(schemaList: SchemaInfo[], dialect: SqlDialect) } return schemaList[0]?.schemaName; } + +export function isCompositeDbName(name: string) { + return name?.includes('::'); +} + +export function splitCompositeDbName(name: string) { + const [database, schema] = name.split('::'); + return { database, schema }; +} + +export function extractDbNameFromComposite(name: string) { + return isCompositeDbName(name) ? splitCompositeDbName(name).database : name; +} diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js index ca2e5553..798e0309 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/Analyser.js @@ -1,4 +1,4 @@ -const { DatabaseAnalyser } = require('dbgate-tools'); +const { DatabaseAnalyser } = global.DBGATE_PACKAGES['dbgate-tools']; const sql = require('./sql'); function extractDataType(dataType) { diff --git a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js index 82e4a736..266f33b4 100644 --- a/plugins/dbgate-plugin-clickhouse/src/backend/driver.js +++ b/plugins/dbgate-plugin-clickhouse/src/backend/driver.js @@ -5,6 +5,7 @@ const Analyser = require('./Analyser'); const { createClient } = require('@clickhouse/client'); const createBulkInsertStream = require('./createBulkInsertStream'); + /** @type {import('dbgate-types').EngineDriver} */ const driver = { ...driverBase, @@ -15,7 +16,7 @@ const driver = { url: databaseUrl, username: user, password: password, - database: database, + database, }); client._database_name = database; diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index db2f637e..5c8385b2 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -81,7 +81,6 @@ const driver = { await pool.connect(); // const pool = await MongoClient.connect(mongoUrl); pool.__getDatabase = database ? () => pool.db(database) : () => pool.db(); - pool.__databaseName = database; return pool; }, // @ts-ignore diff --git a/plugins/dbgate-plugin-mssql/src/backend/driver.js b/plugins/dbgate-plugin-mssql/src/backend/driver.js index eddc2a97..3f54ea66 100644 --- a/plugins/dbgate-plugin-mssql/src/backend/driver.js +++ b/plugins/dbgate-plugin-mssql/src/backend/driver.js @@ -79,11 +79,16 @@ const driver = { async connect(conn) { const { authType } = conn; - if (requireMsnodesqlv8 && (authType == 'sspi' || authType == 'sql')) { - return nativeConnect(conn); + const result = + requireMsnodesqlv8 && (authType == 'sspi' || authType == 'sql') + ? await nativeConnect(conn) + : await tediousConnect(conn); + + if (result) { + result._database_name = conn.database; } - return tediousConnect(conn); + return result; }, async close(pool) { return pool.close(); diff --git a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js index f5153af8..0c9acaff 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/Analyser.js +++ b/plugins/dbgate-plugin-postgres/src/backend/Analyser.js @@ -2,7 +2,8 @@ const fp = require('lodash/fp'); const _ = require('lodash'); const sql = require('./sql'); -const { DatabaseAnalyser, isTypeString, isTypeNumeric } = global.DBGATE_PACKAGES['dbgate-tools']; +const { DatabaseAnalyser, isTypeString, isTypeNumeric, isCompositeDbName, splitCompositeDbName } = + global.DBGATE_PACKAGES['dbgate-tools']; function normalizeTypeName(dataType) { if (dataType == 'character varying') return 'varchar'; @@ -56,7 +57,12 @@ class Analyser extends DatabaseAnalyser { createQuery(resFileName, typeFields, replacements = {}) { const query = super.createQuery(sql[resFileName], typeFields, replacements); - return query; + const dbname = this.pool._database_name; + const schemaCondition = isCompositeDbName(dbname) + ? `= '${splitCompositeDbName(dbname).schema}' ` + : ' IS NOT NULL '; + + return query.replace(/=SCHEMA_NAME_CONDITION/g, schemaCondition); } async _computeSingleObjectId() { diff --git a/plugins/dbgate-plugin-postgres/src/backend/drivers.js b/plugins/dbgate-plugin-postgres/src/backend/drivers.js index d93e7ab6..6fdc6db1 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/drivers.js +++ b/plugins/dbgate-plugin-postgres/src/backend/drivers.js @@ -89,6 +89,7 @@ const drivers = driverBases.map(driverBase => ({ await this.query(client, 'SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY'); } + client._database_name = database; return client; }, async close(pool) { diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/columns.js b/plugins/dbgate-plugin-postgres/src/backend/sql/columns.js index 32b0b16f..6053e1ed 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/columns.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/columns.js @@ -20,5 +20,6 @@ where or ('views:' || table_schema || '.' || table_name) =OBJECT_ID_CONDITION ) + and table_schema =SCHEMA_NAME_CONDITION order by ordinal_position `; \ No newline at end of file diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js index f7f6e300..3667cad7 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_key_column_usage.js @@ -7,5 +7,5 @@ select basecol.table_name, basecol.ordinal_position from information_schema.key_column_usage basecol -where ('tables:' || basecol.table_schema || '.' || basecol.table_name) =OBJECT_ID_CONDITION +where ('tables:' || basecol.table_schema || '.' || basecol.table_name) =OBJECT_ID_CONDITION and basecol.table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_referential_constraints.js b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_referential_constraints.js index a93584d6..243272ef 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_referential_constraints.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_referential_constraints.js @@ -7,4 +7,5 @@ select fk.unique_constraint_name as "unique_constraint_name", fk.unique_constraint_schema as "unique_constraint_schema" from information_schema.referential_constraints fk +where fk.constraint_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js index faca2c5f..f381ff0e 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/fk_table_constraints.js @@ -5,5 +5,5 @@ select base.constraint_name as "constraint_name", base.constraint_schema as "constraint_schema" from information_schema.table_constraints base -where ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION +where ('tables:' || base.table_schema || '.' || base.table_name) =OBJECT_ID_CONDITION and base.table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/geographyColumns.js b/plugins/dbgate-plugin-postgres/src/backend/sql/geographyColumns.js index 343d9c5d..0dac1a0f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/geographyColumns.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/geographyColumns.js @@ -4,5 +4,5 @@ select f_table_name as "pure_name", f_geography_column as "column_name" from public.geography_columns -where ('tables:' || f_table_schema || '.' || f_table_name) =OBJECT_ID_CONDITION +where ('tables:' || f_table_schema || '.' || f_table_name) =OBJECT_ID_CONDITION and f_table_schema =SCHEMA_NAME_CONDITION `; \ No newline at end of file diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/geometryColumns.js b/plugins/dbgate-plugin-postgres/src/backend/sql/geometryColumns.js index 94b8d551..386ea112 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/geometryColumns.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/geometryColumns.js @@ -4,5 +4,5 @@ select f_table_name as "pure_name", f_geometry_column as "column_name" from public.geometry_columns -where ('tables:' || f_table_schema || '.' || f_table_name) =OBJECT_ID_CONDITION +where ('tables:' || f_table_schema || '.' || f_table_name) =OBJECT_ID_CONDITION and f_table_schema =SCHEMA_NAME_CONDITION `; \ No newline at end of file diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/indexcols.js b/plugins/dbgate-plugin-postgres/src/backend/sql/indexcols.js index 4acf4fbd..e14c8a56 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/indexcols.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/indexcols.js @@ -19,6 +19,7 @@ module.exports = ` and t.relnamespace = c.oid and c.nspname != 'pg_catalog' and ('tables:' || c.nspname || '.' || t.relname) =OBJECT_ID_CONDITION + and c.nspname =SCHEMA_NAME_CONDITION order by t.relname `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/indexes.js b/plugins/dbgate-plugin-postgres/src/backend/sql/indexes.js index 8cb8de24..53bd6505 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/indexes.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/indexes.js @@ -21,6 +21,7 @@ module.exports = ` and t.relnamespace = c.oid and c.nspname != 'pg_catalog' and ('tables:' || c.nspname || '.' || t.relname) =OBJECT_ID_CONDITION + and c.nspname =SCHEMA_NAME_CONDITION order by t.relname `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/matviewColumns.js b/plugins/dbgate-plugin-postgres/src/backend/sql/matviewColumns.js index 292d90c4..6330bba9 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/matviewColumns.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/matviewColumns.js @@ -12,6 +12,7 @@ FROM pg_catalog.pg_class WHERE pg_class.relkind = 'm' AND pg_attribute.attnum >= 1 AND ('matviews:' || pg_namespace.nspname || '.' || pg_class.relname) =OBJECT_ID_CONDITION + AND pg_namespace.nspname =SCHEMA_NAME_CONDITION ORDER BY pg_attribute.attnum `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/matviewModifications.js b/plugins/dbgate-plugin-postgres/src/backend/sql/matviewModifications.js index f8ad85ad..abade80f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/matviewModifications.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/matviewModifications.js @@ -4,5 +4,5 @@ select schemaname as "schema_name", md5(definition) as "hash_code" from - pg_catalog.pg_matviews WHERE schemaname NOT LIKE 'pg_%' + pg_catalog.pg_matviews WHERE schemaname NOT LIKE 'pg_%' AND schemaname =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/matviews.js b/plugins/dbgate-plugin-postgres/src/backend/sql/matviews.js index de1105d8..9210e432 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/matviews.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/matviews.js @@ -7,4 +7,5 @@ select from pg_catalog.pg_matviews WHERE schemaname NOT LIKE 'pg_%' and ('matviews:' || schemaname || '.' || matviewname) =OBJECT_ID_CONDITION + and schemaname =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/primaryKeys.js b/plugins/dbgate-plugin-postgres/src/backend/sql/primaryKeys.js index 6df4ca5c..05792ab3 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/primaryKeys.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/primaryKeys.js @@ -14,5 +14,6 @@ where and table_constraints.table_schema !~ '^_timescaledb_' and table_constraints.constraint_type = 'PRIMARY KEY' and ('tables:' || table_constraints.table_schema || '.' || table_constraints.table_name) =OBJECT_ID_CONDITION + and table_constraints.table_schema =SCHEMA_NAME_CONDITION order by key_column_usage.ordinal_position `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/routineModifications.js b/plugins/dbgate-plugin-postgres/src/backend/sql/routineModifications.js index 1eae4ca8..663d948b 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/routineModifications.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/routineModifications.js @@ -6,5 +6,5 @@ select routine_type as "object_type" from information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog' and routine_schema !~ '^_timescaledb_' - and routine_type in ('PROCEDURE', 'FUNCTION') + and routine_type in ('PROCEDURE', 'FUNCTION') and routine_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/routines.js b/plugins/dbgate-plugin-postgres/src/backend/sql/routines.js index d864780c..aed4889f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/routines.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/routines.js @@ -9,6 +9,7 @@ select max(external_language) as "language" from information_schema.routines where routine_schema != 'information_schema' and routine_schema != 'pg_catalog' and routine_schema !~ '^_timescaledb_' + and routine_schema =SCHEMA_NAME_CONDITION and ( (routine_type = 'PROCEDURE' and ('procedures:' || routine_schema || '.' || routine_name) =OBJECT_ID_CONDITION) or diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js b/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js index 1e9f44db..e08ff01a 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/tableList.js @@ -8,4 +8,5 @@ and infoTables.table_schema <> 'information_schema' and infoTables.table_schema <> 'pg_internal' and infoTables.table_schema !~ '^pg_toast' and infoTables.table_schema !~ '^_timescaledb_' +and infoTables.table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js b/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js index 1e074769..8a70848f 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/tableModifications.js @@ -26,4 +26,5 @@ and infoTables.table_schema <> 'information_schema' and infoTables.table_schema <> 'pg_internal' and infoTables.table_schema !~ '^pg_toast' and infoTables.table_schema !~ '^_timescaledb_' +and infoTables.table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/uniqueNames.js b/plugins/dbgate-plugin-postgres/src/backend/sql/uniqueNames.js index a6eec71b..7d413db2 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/uniqueNames.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/uniqueNames.js @@ -1,3 +1,3 @@ module.exports = ` - select conname as "constraint_name" from pg_constraint where contype = 'u' + select conname as "constraint_name" from pg_constraint where contype = 'u' and connamespace = SCHEMA_ID_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/viewModifications.js b/plugins/dbgate-plugin-postgres/src/backend/sql/viewModifications.js index e344bbdd..c2363392 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/viewModifications.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/viewModifications.js @@ -4,5 +4,5 @@ select table_schema as "schema_name", md5(view_definition) as "hash_code" from - information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog' and table_schema !~ '^_timescaledb_' + information_schema.views where table_schema != 'information_schema' and table_schema != 'pg_catalog' and table_schema !~ '^_timescaledb_' and table_schema =SCHEMA_NAME_CONDITION `; diff --git a/plugins/dbgate-plugin-postgres/src/backend/sql/views.js b/plugins/dbgate-plugin-postgres/src/backend/sql/views.js index a009f3ac..2ee21ae9 100644 --- a/plugins/dbgate-plugin-postgres/src/backend/sql/views.js +++ b/plugins/dbgate-plugin-postgres/src/backend/sql/views.js @@ -6,6 +6,6 @@ select md5(view_definition) as "hash_code" from information_schema.views -where table_schema != 'information_schema' and table_schema != 'pg_catalog' and table_schema !~ '^_timescaledb_' +where table_schema != 'information_schema' and table_schema != 'pg_catalog' and table_schema !~ '^_timescaledb_' and table_schema =SCHEMA_NAME_CONDITION and ('views:' || table_schema || '.' || table_name) =OBJECT_ID_CONDITION `;