support app queries

packages/api/src/controllers/jsldata.js

@@ -111,18 +111,22 @@ module.exports = {
   getInfo_meta: true,
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
-    const firstLine = await readFirstLine(file);
-    if (firstLine) {
-      const parsed = JSON.parse(firstLine);
-      if (parsed.__isStreamHeader) {
-        return parsed;
+    try {
+      const firstLine = await readFirstLine(file);
+      if (firstLine) {
+        const parsed = JSON.parse(firstLine);
+        if (parsed.__isStreamHeader) {
+          return parsed;
+        }
+        return {
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+        };
       }
-      return {
-        __isStreamHeader: true,
-        __isDynamicStructure: true,
-      };
+      return null;
+    } catch (err) {
+      return null;
     }
-    return null;
   },
 
   getRows_meta: true,

packages/api/src/controllers/sessions.js

@@ -4,8 +4,10 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const path = require('path');
 const { handleProcessCommunication } = require('../utility/processComm');
 const processArgs = require('../utility/processArgs');
+const { appdir } = require('../utility/directories');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -46,9 +48,15 @@ module.exports = {
     this.dispatchMessage(sesid, info);
   },
 
-  handle_done(sesid) {
+  handle_done(sesid, props) {
     socket.emit(`session-done-${sesid}`);
-    this.dispatchMessage(sesid, 'Query execution finished');
+    if (!props.skipFinishedMessage) {
+      this.dispatchMessage(sesid, 'Query execution finished');
+    }
+    const session = this.opened.find(x => x.sesid == sesid);
+    if (session.killOnDone) {
+      this.kill({ sesid });
+    }
   },
 
   handle_recordset(sesid, props) {
@@ -60,6 +68,11 @@ module.exports = {
     jsldata.notifyChangedStats(stats);
   },
 
+  handle_initializeFile(sesid, props) {
+    const { jslid } = props;
+    socket.emit(`session-initialize-file-${jslid}`);
+  },
+
   handle_ping() {},
 
   create_meta: true,
@@ -105,6 +118,19 @@ module.exports = {
     return { state: 'ok' };
   },
 
+  executeReader_meta: true,
+  async executeReader({ conid, database, sql, queryName, appFolder }) {
+    const { sesid } = await this.create({ conid, database });
+    const session = this.opened.find(x => x.sesid == sesid);
+    session.killOnDone = true;
+    const jslid = uuidv1();
+    const fileName = queryName && appFolder ? path.join(appdir(), appFolder, `${queryName}.query.sql`) : null;
+
+    session.subprocess.send({ msgtype: 'executeReader', sql, fileName, jslid });
+
+    return { jslid };
+  },
+
   // cancel_meta: true,
   // async cancel({ sesid }) {
   //   const session = this.opened.find((x) => x.sesid == sesid);

packages/api/src/proc/sessionProcess.js

@@ -17,11 +17,15 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(structure, resultIndex) {
-    this.jslid = uuidv1();
-    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+  constructor() {
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
+    this.initializedFile = false;
+  }
+
+  initializeFromQuery(structure, resultIndex) {
+    this.jslid = uuidv1();
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     fs.writeFileSync(
       this.currentFile,
       JSON.stringify({
@@ -32,13 +36,21 @@ class TableWriter {
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
+    this.initializedFile = true;
     process.send({ msgtype: 'recordset', jslid: this.jslid, resultIndex });
   }
 
+  initializeFromReader(jslid) {
+    this.jslid = jslid;
+    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
+    this.writeCurrentStats(false, false);
+  }
+
   row(row) {
     // console.log('ACCEPT ROW', row);
     this.currentStream.write(JSON.stringify(row) + '\n');
     this.currentRowCount += 1;
+
     if (!this.plannedStats) {
       this.plannedStats = true;
       process.nextTick(() => {
@@ -49,6 +61,21 @@ class TableWriter {
     }
   }
 
+  rowFromReader(row) {
+    if (!this.initializedFile) {
+      process.send({ msgtype: 'initializeFile', jslid: this.jslid });
+      this.initializedFile = true;
+
+      fs.writeFileSync(this.currentFile, JSON.stringify(row) + '\n');
+      this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
+      this.writeCurrentStats(false, false);
+      this.initializedFile = true;
+      return;
+    }
+
+    this.row(row);
+  }
+
   writeCurrentStats(isFinished = false, emitEvent = false) {
     const stats = {
       rowCount: this.currentRowCount,
@@ -63,10 +90,11 @@ class TableWriter {
     }
   }
 
-  close() {
+  close(afterClose) {
     if (this.currentStream) {
       this.currentStream.end(() => {
         this.writeCurrentStats(true, true);
+        if (afterClose) afterClose();
       });
     }
   }
@@ -98,7 +126,11 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter();
+    this.currentWriter.initializeFromQuery(
+      Array.isArray(columns) ? { columns } : columns,
+      this.resultIndexHolder.value
+    );
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -110,7 +142,6 @@ class StreamHandler {
     // }, 500);
   }
   row(row) {
-    // console.log('ACCEPT ROW', row);
     if (this.currentWriter) this.currentWriter.row(row);
     else if (row.message) process.send({ msgtype: 'info', info: { message: row.message } });
     // this.onRow(this.jslid);
@@ -135,20 +166,21 @@ function handleStream(driver, resultIndexHolder, sql) {
   });
 }
 
-function ensureExecuteCustomScript(driver) {
+function allowExecuteCustomScript(driver) {
   if (driver.readOnlySessions) {
-    return;
+    return true;
   }
   if (storedConnection.isReadOnly) {
-    throw new Error('Connection is read only');
+    return false;
+    // throw new Error('Connection is read only');
   }
+  return true;
 }
 
 async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  ensureExecuteCustomScript(driver);
   systemConnection = await connectUtility(driver, storedConnection);
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
@@ -173,6 +205,19 @@ async function handleExecuteQuery({ sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
 
+  if (!allowExecuteCustomScript(driver)) {
+    process.send({
+      msgtype: 'info',
+      info: {
+        message: 'Connection without read-only sessions is read only',
+        severity: 'error',
+      },
+    });
+    process.send({ msgtype: 'done', skipFinishedMessage: true });
+    return;
+    //process.send({ msgtype: 'error', error: e.message });
+  }
+
   const resultIndexHolder = {
     value: 0,
   };
@@ -186,9 +231,39 @@ async function handleExecuteQuery({ sql }) {
   process.send({ msgtype: 'done' });
 }
 
+async function handleExecuteReader({ jslid, sql, fileName }) {
+  await waitConnected();
+
+  const driver = requireEngineDriver(storedConnection);
+
+  if (fileName) {
+    sql = fs.readFileSync(fileName, 'utf-8');
+  } else {
+    if (!allowExecuteCustomScript(driver)) {
+      process.send({ msgtype: 'done' });
+      return;
+    }
+  }
+
+  const writer = new TableWriter();
+  writer.initializeFromReader(jslid);
+
+  const reader = await driver.readQuery(systemConnection, sql);
+
+  reader.on('data', data => {
+    writer.rowFromReader(data);
+  });
+  reader.on('end', () => {
+    writer.close(() => {
+      process.send({ msgtype: 'done' });
+    });
+  });
+}
+
 const messageHandlers = {
   connect: handleConnect,
   executeQuery: handleExecuteQuery,
+  executeReader: handleExecuteReader,
   // cancel: handleCancel,
 };
 

packages/types/index.d.ts

@@ -19,6 +19,7 @@ export interface OpenedSession {
   sesid: string;
   conid: string;
   database: string;
+  killOnDone?: boolean;
   subprocess: ChildProcess;
 }
 

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -10,6 +10,7 @@
     matviews: 'img view',
     procedures: 'img procedure',
     functions: 'img function',
+    queries: 'img query-data',
   };
 
   const defaultTabs = {
@@ -17,6 +18,7 @@
     collections: 'CollectionDataTab',
     views: 'ViewDataTab',
     matviews: 'ViewDataTab',
+    queries: 'QueryDataTab',
   };
 
   const menus = {
@@ -231,6 +233,13 @@
         },
       },
     ],
+    queries: [
+      {
+        label: 'Open data',
+        tab: 'QueryDataTab',
+        forceNewTab: true,
+      },
+    ],
     procedures: [
       {
         label: 'Drop procedure',

packages/web/src/datagrid/JslDataGrid.svelte

@@ -1,22 +1,42 @@
 <script lang="ts">
   import { createGridCache, createGridConfig, JslGridDisplay } from 'dbgate-datalib';
   import { writable } from 'svelte/store';
-  import { useApiCall } from '../utility/api';
+  import { apiOff, apiOn, useApiCall } from '../utility/api';
+  import useEffect from '../utility/useEffect';
 
   import DataGrid from './DataGrid.svelte';
   import JslDataGridCore from './JslDataGridCore.svelte';
 
   export let jslid;
-  export let supportsReload;
+  export let supportsReload = false;
+  export let listenInitializeFile = false;
 
   let loadedRows;
+  let infoCounter = 0;
 
-  $: info = useApiCall('jsldata/get-info', { jslid }, {});
+  $: info = useApiCall('jsldata/get-info', { jslid, infoCounter }, {});
 
   // $: columns = ($info && $info.columns) || [];
   const config = writable(createGridConfig());
   const cache = writable(createGridCache());
 
+  function handleInitializeFile() {
+    infoCounter += 1;
+  }
+
+  $: effect = useEffect(() => onJslId(jslid));
+  function onJslId(jslidVal) {
+    if (jslidVal && listenInitializeFile) {
+      apiOn(`session-initialize-file-${jslidVal}`, handleInitializeFile);
+      return () => {
+        apiOff(`session-initialize-file-${jslidVal}`, handleInitializeFile);
+      };
+    } else {
+      return () => {};
+    }
+  }
+  $: $effect;
+
   $: display = new JslGridDisplay(
     jslid,
     $info,

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -81,6 +81,8 @@
       return () => {
         apiOff(`jsldata-stats-${jslidVal}`, handleJslDataStats);
       };
+    } else {
+      return () => {};
     }
   }
   $: $effect;

packages/web/src/icons/FontIcon.svelte

@@ -155,6 +155,7 @@
     'img sqlite-database': 'mdi mdi-database color-icon-blue',
     'img table': 'mdi mdi-table color-icon-blue',
     'img collection': 'mdi mdi-table color-icon-red',
+    'img query-data': 'mdi mdi-table color-icon-yellow',
     'img view': 'mdi mdi-table color-icon-magenta',
     'img procedure': 'mdi mdi-cog color-icon-blue',
     'img function': 'mdi mdi-function-variant',

packages/web/src/tabs/QueryDataTab.svelte

@@ -0,0 +1,43 @@
+<script lang="ts" context="module">
+  export const matchingProps = ['conid', 'database', 'pureName', 'sql'];
+</script>
+
+<script lang="ts">
+  import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
+
+  import ToolStripContainer from '../buttons/ToolStripContainer.svelte';
+  import ToolStripExportButton, { createQuickExportHandlerRef } from '../buttons/ToolStripExportButton.svelte';
+
+  import JslDataGrid from '../datagrid/JslDataGrid.svelte';
+  import LoadingInfo from '../elements/LoadingInfo.svelte';
+  import { apiCall } from '../utility/api';
+
+  export let sql = undefined;
+  export let pureName = undefined;
+  export let schemaName = undefined;
+  export let conid;
+  export let database;
+
+  let jslid;
+
+  async function loadData(conid, database, sql) {
+    const resp = await apiCall('sessions/execute-reader', { conid, database, sql, appFolder: schemaName, queryName: pureName });
+    jslid = resp.jslid;
+  }
+
+  const quickExportHandlerRef = createQuickExportHandlerRef();
+
+  $: loadData(conid, database, sql);
+</script>
+
+<ToolStripContainer>
+  {#if jslid}
+    <JslDataGrid {jslid} listenInitializeFile />
+  {:else}
+    <LoadingInfo message="Loading data..." />
+  {/if}
+  <svelte:fragment slot="toolstrip">
+    <ToolStripCommandButton command="dataGrid.refresh" />
+    <ToolStripExportButton command="jslTableGrid.export" {quickExportHandlerRef} />
+  </svelte:fragment>
+</ToolStripContainer>

packages/web/src/tabs/QueryTab.svelte

@@ -140,7 +140,7 @@
   }
 
   export function hasConnection() {
-    return !!conid && (!$connection.isReadOnly || driver.readOnlySessions);
+    return !!conid && (!$connection?.isReadOnly || driver?.readOnlySessions);
   }
 
   async function executeCore(sql) {

packages/web/src/tabs/index.js

@@ -22,6 +22,7 @@ import * as JsonTab from './JsonTab.svelte';
 import * as ChangelogTab from './ChangelogTab.svelte';
 import * as DiagramTab from './DiagramTab.svelte';
 import * as DbKeyDetailTab from './DbKeyDetailTab.svelte';
+import * as QueryDataTab from './QueryDataTab.svelte';
 
 export default {
   TableDataTab,
@@ -48,4 +49,5 @@ export default {
   ChangelogTab,
   DiagramTab,
   DbKeyDetailTab,
+  QueryDataTab,
 };

packages/web/src/widgets/AppFilesList.svelte

@@ -79,12 +79,16 @@
         text: 'New SQL command',
         onClick: () => handleNewSqlFile('command.sql', 'Create new SQL command', COMMAND_TEMPLATE),
       },
+      {
+        text: 'New SQL query',
+        onClick: () => handleNewSqlFile('query.sql', 'Create new SQL query', QUERY_TEMPLATE),
+      },
       {
         text: 'New virtual references file',
         onClick: () => handleNewConfigFile('virtual-references.config.json', []),
       },
       {
-        text: 'New disctionary descriptions file',
+        text: 'New dictionary descriptions file',
         onClick: () => handleNewConfigFile('dictionary-descriptions.config.json', []),
       },
 

packages/web/src/widgets/SqlObjectList.svelte

@@ -16,7 +16,7 @@
   import InlineButton from '../buttons/InlineButton.svelte';
   import SearchInput from '../elements/SearchInput.svelte';
   import WidgetsInnerContainer from './WidgetsInnerContainer.svelte';
-  import { useConnectionInfo, useDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
+  import { useConnectionInfo, useDatabaseInfo, useDatabaseStatus, useUsedApps } from '../utility/metadataLoaders';
   import SearchBoxWrapper from '../elements/SearchBoxWrapper.svelte';
   import AppObjectList from '../appobj/AppObjectList.svelte';
   import _ from 'lodash';
@@ -30,10 +30,11 @@
   import FontIcon from '../icons/FontIcon.svelte';
   import CloseSearchButton from '../buttons/CloseSearchButton.svelte';
   import { findEngineDriver } from 'dbgate-tools';
-  import { extensions } from '../stores';
+  import { currentDatabase, extensions } from '../stores';
   import newQuery from '../query/newQuery';
   import runCommand from '../commands/runCommand';
   import { apiCall } from '../utility/api';
+  import { filterAppsForDatabase } from '../utility/appTools';
 
   export let conid;
   export let database;
@@ -46,16 +47,28 @@
   $: connection = useConnectionInfo({ conid });
   $: driver = findEngineDriver($connection, $extensions);
 
+  $: apps = useUsedApps();
+
+  $: dbApps = filterAppsForDatabase($currentDatabase.connection, $currentDatabase.name, $apps || []);
+
   // $: console.log('OBJECTS', $objects);
 
-  $: objectList = _.flatten(
-    ['tables', 'collections', 'views', 'matviews', 'procedures', 'functions'].map(objectTypeField =>
+  $: objectList = _.flatten([
+    ...['tables', 'collections', 'views', 'matviews', 'procedures', 'functions'].map(objectTypeField =>
       _.sortBy(
         (($objects || {})[objectTypeField] || []).map(obj => ({ ...obj, objectTypeField })),
         ['schemaName', 'pureName']
       )
-    )
-  );
+    ),
+    ...dbApps.map(app =>
+      app.queries.map(query => ({
+        objectTypeField: 'queries',
+        pureName: query.name,
+        schemaName: app.name,
+        sql: query.sql
+      }))
+    ),
+  ]);
 
   // let generateIndex = 0;
   // setInterval(() => (generateIndex += 1), 2000);
@@ -69,7 +82,7 @@
     const res = [];
     if (driver?.databaseEngineTypes?.includes('document')) {
       res.push({ command: 'new.collection' });
-    } 
+    }
     if (driver?.databaseEngineTypes?.includes('sql')) {
       res.push({ command: 'new.table' });
     }