oracle: import data works

2024-11-07 20:26:23 +00:00 · 2024-06-01 14:35:40 +02:00 · 2024-06-01 14:35:40 +02:00 · a812ff510d
commit a812ff510d
parent cafe0e68c3
16 changed files with 68 additions and 45 deletions
--- a/packages/tools/src/createBulkInsertStreamBase.ts
+++ b/packages/tools/src/createBulkInsertStreamBase.ts
@ -56,24 +56,42 @@ export function createBulkInsertStreamBase(driver: EngineDriver, stream, pool, n
    const rows = writable.buffer;
    writable.buffer = [];

-    const dmp = driver.createDumper();
+    if (driver.dialect.allowMultipleValuesInsert) {
+      const dmp = driver.createDumper();
+      dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
+      dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string)));
+      dmp.putRaw(')\n VALUES\n');

-    dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
-    dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string)));
-    dmp.putRaw(')\n VALUES\n');
+      let wasRow = false;
+      for (const row of rows) {
+        if (wasRow) dmp.putRaw(',\n');
+        dmp.putRaw('(');
+        dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string]));
+        dmp.putRaw(')');
+        wasRow = true;
+      }
+      dmp.putRaw(';');
+      // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
+      // console.log(dmp.s);
+      await driver.query(pool, dmp.s, { discardResult: true });
+    } else {
+      for (const row of rows) {
+        const dmp = driver.createDumper();
+        dmp.putRaw(`INSERT INTO ${fullNameQuoted} (`);
+        dmp.putCollection(',', writable.columnNames, col => dmp.putRaw(driver.dialect.quoteIdentifier(col as string)));
+        dmp.putRaw(')\n VALUES\n');

-    let wasRow = false;
-    for (const row of rows) {
-      if (wasRow) dmp.putRaw(',\n');
-      dmp.putRaw('(');
-      dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string]));
-      dmp.putRaw(')');
-      wasRow = true;
+        dmp.putRaw('(');
+        dmp.putCollection(',', writable.columnNames, col => dmp.putValue(row[col as string]));
+        dmp.putRaw(')');
+        await driver.query(pool, dmp.s, { discardResult: true });
+      }
+    }
+    if (options.commitAfterInsert) {
+      const dmp = driver.createDumper();
+      dmp.commitTransaction();
+      await driver.query(pool, dmp.s, { discardResult: true });
    }
-    dmp.putRaw(';');
-    // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
-    // console.log(dmp.s);
-    await driver.query(pool, dmp.s, { discardResult: true });
  };

  writable.sendIfFull = async () => {
--- a/packages/types/dialect.d.ts
+++ b/packages/types/dialect.d.ts
@ -12,6 +12,7 @@ export interface SqlDialect {
  defaultSchemaName?: string;
  enableConstraintsPerTable?: boolean;
  requireStandaloneSelectForScopeIdentity?: boolean;
+  allowMultipleValuesInsert?: boolean;

  dropColumnDependencies?: string[];
  changeColumnDependencies?: string[];
--- a/packages/types/engines.d.ts
+++ b/packages/types/engines.d.ts
@ -24,6 +24,7 @@ export interface WriteTableOptions {
  dropIfExists?: boolean;
  truncate?: boolean;
  createIfNotExists?: boolean;
+  commitAfterInsert?: boolean;
 }

 export interface EngineAuthType {
--- a/plugins/dbgate-plugin-mysql/src/frontend/drivers.js
+++ b/plugins/dbgate-plugin-mysql/src/frontend/drivers.js
@ -22,6 +22,7 @@ const dialect = {
  enableConstraintsPerTable: false,
  anonymousPrimaryKey: true,
  explicitDropConstraint: true,
+  allowMultipleValuesInsert: true,
  quoteIdentifier(s) {
    return '`' + s + '`';
  },
--- a/plugins/dbgate-plugin-oracle/src/backend/Analyser.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/Analyser.js
@ -42,8 +42,8 @@ class Analyser extends DatabaseAnalyser {
  }

  async _computeSingleObjectId() {
-    const { typeField, schemaName, pureName } = this.singleObjectFilter;
-    this.singleObjectId = `${typeField}:${schemaName || 'public'}.${pureName}`;
+    const { typeField,  pureName } = this.singleObjectFilter;
+    this.singleObjectId = `${typeField}:${pureName}`;
  }

  async _runAnalysis() {
@ -78,7 +78,7 @@ class Analyser extends DatabaseAnalyser {

    const fkColumnsMapped = fkColumns.rows.map(x => ({
      pureName: x.pure_name,
-      schemaName: x.schema_name,
+      // schemaName: x.schema_name,
      constraintSchema: x.constraint_schema,
      constraintName: x.constraint_name,
      columnName: x.column_name,
@ -86,11 +86,11 @@ class Analyser extends DatabaseAnalyser {
      updateAction: x.update_action,
      deleteAction: x.delete_action,
      refTableName: x.ref_table_name,
-      refSchemaName: x.ref_schema_name,
+      // refSchemaName: x.ref_schema_name,
    }));
    const pkColumnsMapped = pkColumns.rows.map(x => ({
      pureName: x.pure_name,
-      schemaName: x.schema_name,
+      // schemaName: x.schema_name,
      constraintSchema: x.constraint_schema,
      constraintName: x.constraint_name,
      columnName: x.column_name,
@ -103,8 +103,8 @@ class Analyser extends DatabaseAnalyser {
      tables: tables.rows.map(table => {
        const newTable = {
          pureName: table.pure_name,
-          schemaName: table.schema_name,
-          objectId: `tables:${table.schema_name}.${table.pure_name}`,
+          // schemaName: table.schema_name,
+          objectId: `tables:${table.pure_name}`,
          contentHash: table.hash_code_columns ? `${table.hash_code_columns}-${table.hash_code_constraints}` : null,
        };
        return {
@ -146,39 +146,39 @@ class Analyser extends DatabaseAnalyser {
        };
      }),
      views: views.rows.map(view => ({
-        objectId: `views:${view.schema_name}.${view.pure_name}`,
+        objectId: `views:${view.pure_name}`,
        pureName: view.pure_name,
-        schemaName: view.schema_name,
+        // schemaName: view.schema_name,
        contentHash: view.hash_code,
-        createSql: `CREATE VIEW "${view.schema_name}"."${view.pure_name}"\nAS\n${view.create_sql}`,
+        createSql: `CREATE VIEW "${view.pure_name}"\nAS\n${view.create_sql}`,
        columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)),
      })),
      matviews: matviews
        ? matviews.rows.map(matview => ({
-            objectId: `matviews:${matview.schema_name}.${matview.pure_name}`,
+            objectId: `matviews:${matview.pure_name}`,
            pureName: matview.pure_name,
-            schemaName: matview.schema_name,
+            // schemaName: matview.schema_name,
            contentHash: matview.hash_code,
-            createSql: `CREATE MATERIALIZED VIEW "${matview.schema_name}"."${matview.pure_name}"\nAS\n${matview.definition}`,
+            createSql: `CREATE MATERIALIZED VIEW "${matview.pure_name}"\nAS\n${matview.definition}`,
            columns: (columnsGrouped[columnGroup(view)] || []).map(col => getColumnInfo(col)),
          }))
        : undefined,
      procedures: routines.rows
        .filter(x => x.object_type == 'PROCEDURE')
        .map(proc => ({
-          objectId: `procedures:${proc.schema_name}.${proc.pure_name}`,
+          objectId: `procedures:${proc.pure_name}`,
          pureName: proc.pure_name,
-          schemaName: proc.schema_name,
-          createSql: `CREATE PROCEDURE "${proc.schema_name}"."${proc.pure_name}"() LANGUAGE ${proc.language}\nAS\n$$\n${proc.definition}\n$$`,
+          // schemaName: proc.schema_name,
+          createSql: `CREATE PROCEDURE "${proc.pure_name}"() LANGUAGE ${proc.language}\nAS\n$$\n${proc.definition}\n$$`,
          contentHash: proc.hash_code,
        })),
      functions: routines.rows
        .filter(x => x.object_type == 'FUNCTION')
        .map(func => ({
-          objectId: `functions:${func.schema_name}.${func.pure_name}`,
-          createSql: `CREATE FUNCTION "${func.schema_name}"."${func.pure_name}"() RETURNS ${func.data_type} LANGUAGE ${func.language}\nAS\n$$\n${func.definition}\n$$`,
+          objectId: `functions:${func.pure_name}`,
+          createSql: `CREATE FUNCTION "${func.pure_name}"() RETURNS ${func.data_type} LANGUAGE ${func.language}\nAS\n$$\n${func.definition}\n$$`,
          pureName: func.pure_name,
-          schemaName: func.schema_name,
+          // schemaName: func.schema_name,
          contentHash: func.hash_code,
        })),
    };
--- a/plugins/dbgate-plugin-oracle/src/backend/drivers.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/drivers.js
@ -289,7 +289,7 @@ const drivers = driverBases.map(driverBase => ({
  },
  async writeTable(pool, name, options) {
    // @ts-ignore
-    return createBulkInsertStreamBase(this, stream, pool, name, options);
+    return createBulkInsertStreamBase(this, stream, pool, name, { ...options, commitAfterInsert: true });
  },
  async listDatabases(client) {
    const { rows } = await this.query(client, 'SELECT username as "name" from all_users order by username');
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/columns.js
@ -10,6 +10,6 @@ select
  data_scale as "numeric_scale",
  data_default as "default_value"
  FROM all_tab_columns av
-  where OWNER='$owner' AND TABLE_NAME =OBJECT_ID_CONDITION
+  where OWNER='$owner' AND 'tables:' || TABLE_NAME =OBJECT_ID_CONDITION
 order by column_id
 `;
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/foreignKeys.js
@ -19,6 +19,6 @@ and basecol.table_name = fk.table_name
 and refcol.owner = ref.owner
 and refcol.constraint_name = ref.constraint_name
 and refcol.table_name = ref.table_name
-AND fk.constraint_name =OBJECT_ID_CONDITION
+AND 'tables:' || fk.table_name =OBJECT_ID_CONDITION
 order by basecol.position
 `;
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/indexes.js
@ -9,7 +9,7 @@ select  i.table_name as "tableName",
 from all_ind_columns ic, all_indexes i
 where INDEX_OWNER = '$owner' AND ic.index_owner = i.owner
 and ic.index_name = i.index_name
-and i.index_name =OBJECT_ID_CONDITION
+and 'tables:' || i.table_name =OBJECT_ID_CONDITION
 order by i.table_owner,
         i.table_name,
         i.index_name,
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/matviews.js
@ -14,6 +14,6 @@ SELECT -- owner as schema_name,
                  '//text()'
                )) definition
 FROM all_mviews
-where OWNER = '$owner' AND mview_name=OBJECT_ID_CONDITION
+where OWNER = '$owner' AND 'matviews:' || mview_name=OBJECT_ID_CONDITION
 order by owner, mview_name
 `;
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/primaryKeys.js
@ -11,7 +11,7 @@ where constraint_type = 'P'
 and basecol.owner = pk.owner
 and basecol.constraint_name = pk.constraint_name
 and basecol.table_name = pk.table_name
-and pk.constraint_name =OBJECT_ID_CONDITION
+and 'tables:' || basecol.table_name =OBJECT_ID_CONDITION
 and pk.owner = '$owner'
 order by basecol.position
 `;
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/tableList.js
@ -1,9 +1,9 @@
 module.exports = `
 select
-    -- owner                                   "schema_name",
-    table_name                              "pure_name"
+    -- owner "schema_name",
+    table_name "pure_name"
  from
    all_tables
-  where OWNER='$owner' AND TABLE_NAME =OBJECT_ID_CONDITION
+  where OWNER='$owner' AND 'tables:' || TABLE_NAME =OBJECT_ID_CONDITION
 `;

--- a/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/uniqueNames.js
@ -2,5 +2,5 @@ module.exports = `
 select constraint_name as "constraintName"
 from all_constraints
 where owner='$owner' and constraint_type = 'U'
-  and constraint_name =OBJECT_ID_CONDITION
+  and 'tables:' || table_name =OBJECT_ID_CONDITION
 `;
--- a/plugins/dbgate-plugin-oracle/src/backend/sql/views.js
+++ b/plugins/dbgate-plugin-oracle/src/backend/sql/views.js
@ -8,5 +8,5 @@ from (select
  from all_views av
  where owner = '$owner' and text_vc is not null
  ) avv
-  where "pure_name" =OBJECT_ID_CONDITION
+  where 'views:' || "pure_name" =OBJECT_ID_CONDITION
 `;
--- a/plugins/dbgate-plugin-postgres/src/frontend/drivers.js
+++ b/plugins/dbgate-plugin-postgres/src/frontend/drivers.js
@ -33,6 +33,7 @@ const dialect = {
  dropUnique: true,
  createCheck: true,
  dropCheck: true,
+  allowMultipleValuesInsert: true,

  dropReferencesWhenDropTable: true,
  requireStandaloneSelectForScopeIdentity: true,
--- a/plugins/dbgate-plugin-sqlite/src/frontend/driver.js
+++ b/plugins/dbgate-plugin-sqlite/src/frontend/driver.js
@ -17,6 +17,7 @@ const dialect = {
  explicitDropConstraint: true,
  stringEscapeChar: "'",
  fallbackDataType: 'nvarchar',
+  allowMultipleValuesInsert: true,
  dropColumnDependencies: ['indexes', 'primaryKey', 'uniques'],
  quoteIdentifier(s) {
    return `[${s}]`;