removed nedb dependency, replaced with own impl

packages/api/package.json

@@ -44,7 +44,6 @@
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",

packages/api/src/controllers/connections.js

@@ -1,7 +1,6 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
 const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
@@ -9,6 +8,7 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
 
 const processArgs = require('../utility/processArgs');
 
@@ -136,7 +136,7 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
@@ -173,7 +173,7 @@ module.exports = {
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
@@ -188,7 +188,7 @@ module.exports = {
   update_meta: true,
   async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.update({ _id }, { $set: values });
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -196,14 +196,14 @@ module.exports = {
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }) {
     if (portalConnections) return;
-    const conn = await this.datastore.find({ _id: conid });
+    const conn = await this.datastore.get(conid);
-    let databases = conn[0].databases || [];
+    let databases = (conn && conn.databases) || [];
     if (databases.find(x => x.name == database)) {
       databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
     } else {
       databases = [...databases, { name: database, ...values }];
     }
-    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    const res = await this.datastore.patch(conid, { databases });
     socket.emitChanged('connection-list-changed');
     socket.emitChanged('used-apps-changed');
     // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
@@ -213,7 +213,7 @@ module.exports = {
   delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.remove(connection._id);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -221,8 +221,8 @@ module.exports = {
   get_meta: true,
   async get({ conid }) {
     if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
-    const res = await this.datastore.find({ _id: conid });
+    const res = await this.datastore.get(conid);
-    return res[0] || null;
+    return res || null;
   },
 
   newSqliteDatabase_meta: true,

packages/api/src/utility/JsonLinesDatabase.js

@@ -4,7 +4,7 @@ const uuidv1 = require('uuid/v1');
 // const lineReader = require('line-reader');
 // const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
 
-export default class JsonLinesDatabase {
+class JsonLinesDatabase {
   constructor(filename) {
     this.filename = filename;
     this.data = [];
@@ -37,6 +37,7 @@ export default class JsonLinesDatabase {
   }
 
   async insert(obj) {
+    await this._ensureLoaded();
     if (obj._id && (await this.get(obj._id))) {
       throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
     }
@@ -52,10 +53,12 @@ export default class JsonLinesDatabase {
   }
 
   async get(id) {
+    await this._ensureLoaded();
     return this.data.find(x => x._id == id);
   }
 
   async find(cond) {
+    await this._ensureLoaded();
     if (cond) {
       return this.data.filter(x => {
         for (const key of Object.keys(cond)) {
@@ -69,17 +72,20 @@ export default class JsonLinesDatabase {
   }
 
   async update(obj) {
+    await this._ensureLoaded();
     this.data = this.data.map(x => (x._id == obj._id ? obj : x));
     await this._write();
   }
 
   async patch(id, values) {
+    await this._ensureLoaded();
     this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
     await this._write();
   }
 
   async remove(id) {
-    this.data = this.data.filter(x => x._id!=id);
+    await this._ensureLoaded();
+    this.data = this.data.filter(x => x._id != id);
     await this._write();
   }
 
@@ -116,3 +122,5 @@ export default class JsonLinesDatabase {
   //     await fw.end();
   //   }
 }
+
+module.exports = JsonLinesDatabase;

packages/api/src/utility/JsonLinesDatastore.js

@@ -4,7 +4,7 @@ const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 
-export async function fetchNextLineFromReader(reader) {
+function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
     if (!reader.hasNextLine()) {
       resolve(null);

yarn.lock

@@ -1844,11 +1844,6 @@ async-lock@^1.2.6:
   resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.2.8.tgz#7b02bdfa2de603c0713acecd11184cf97bbc7c4c"
   integrity sha512-G+26B2jc0Gw0EG/WN2M6IczuGepBsfR1+DtqLnyFSH4p2C668qkOCtEkGNVEaaNAVlYwEMazy1+/jnLxltBkIQ==
 
-async@0.2.10:
-  version "0.2.10"
-  resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1"
-  integrity sha1-trvgsGdLnXGXCMo43owjfLUmw9E=
-
 async@>=0.6.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
@@ -2080,13 +2075,6 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
   integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
 
-binary-search-tree@0.2.5:
-  version "0.2.5"
-  resolved "https://registry.yarnpkg.com/binary-search-tree/-/binary-search-tree-0.2.5.tgz#7dbb3b210fdca082450dad2334c304af39bdc784"
-  integrity sha1-fbs7IQ/coIJFDa0jNMMErzm9x4Q=
-  dependencies:
-    underscore "~1.4.4"
-
 bindings@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
@@ -6811,13 +6799,6 @@ local-access@^1.0.1:
   resolved "https://registry.yarnpkg.com/local-access/-/local-access-1.1.0.tgz#e007c76ba2ca83d5877ba1a125fc8dfe23ba4798"
   integrity sha512-XfegD5pyTAfb+GY6chk283Ox5z8WexG56OvM06RWLpAc/UHozO8X6xAxEkIitZOtsSMM1Yr3DkHgW5W+onLhCw==
 
-localforage@^1.3.0:
-  version "1.7.3"
-  resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.7.3.tgz#0082b3ca9734679e1bd534995bdd3b24cf10f204"
-  integrity sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ==
-  dependencies:
-    lie "3.1.1"
-
 localforage@^1.9.0:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1"
@@ -7315,7 +7296,7 @@ mkdirp@0.3.0:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e"
   integrity sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=
 
-mkdirp@0.x, mkdirp@^0.5.1, mkdirp@~0.5.1:
+mkdirp@0.x, mkdirp@^0.5.1:
   version "0.5.1"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
   integrity sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=
@@ -7485,24 +7466,6 @@ ncp@^2.0.0:
   resolved "https://registry.yarnpkg.com/ncp/-/ncp-2.0.0.tgz#195a21d6c46e361d2fb1281ba38b91e9df7bdbb3"
   integrity sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=
 
-nedb-promises@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/nedb-promises/-/nedb-promises-4.0.1.tgz#4d0bd1553d045acca5d6713ad76eb97aa830b390"
-  integrity sha512-I6nVZ0zjjYGfja2UU8lDSEzjfQTS8bo+8jvn7apILpynYDKzLpl6YRfdPa+uRSUYDN9bH45wJ+gvRWcOjO2g5g==
-  dependencies:
-    nedb "^1.8.0"
-
-nedb@^1.8.0:
-  version "1.8.0"
-  resolved "https://registry.yarnpkg.com/nedb/-/nedb-1.8.0.tgz#0e3502cd82c004d5355a43c9e55577bd7bd91d88"
-  integrity sha1-DjUCzYLABNU1WkPJ5VV3vXvZHYg=
-  dependencies:
-    async "0.2.10"
-    binary-search-tree "0.2.5"
-    localforage "^1.3.0"
-    mkdirp "~0.5.1"
-    underscore "~1.4.4"
-
 negotiator@0.6.2:
   version "0.6.2"
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -10693,11 +10656,6 @@ undefsafe@^2.0.2:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.0.tgz#3ccdcbb824230fc6bf234ad0ddcd83dff4eafe5f"
   integrity sha512-sCs4H3pCytsb5K7i072FAEC9YlSYFIbosvM0tAKAlpSSUgD7yC1iXSEGdl5XrDKQ1YUB+p/HDzYrSG2H2Vl36g==
 
-underscore@~1.4.4:
-  version "1.4.4"
-  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604"
-  integrity sha1-YaajIBBiKvoHljvzJSA88SI51gQ=
-
 union-value@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847"