mirrors/dbgate
1
0
Fork 0
mirror of https://github.com/dbgate/dbgate synced 2024-11-08 04:35:58 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

jsl data refactor

Browse Source
This commit is contained in:
Jan Prochazka 2020-10-22 08:23:53 +02:00
parent c4914429ce
commit da1617729b
5 changed files with 318 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

174
packages/api/src/controllers/jsldata.js
View File

@ -1,6 +1,9 @@
const fs = require('fs'); const fs = require('fs');
const lineReader = require('line-reader'); const lineReader = require('line-reader');
const { off } = require('process');
const DatastoreProxy = require('../utility/DatastoreProxy');
const getJslFileName = require('../utility/getJslFileName'); const getJslFileName = require('../utility/getJslFileName');
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
const socket = require('../utility/socket'); const socket = require('../utility/socket');
function readFirstLine(file) { function readFirstLine(file) {
@ -19,76 +22,85 @@ function readFirstLine(file) {
}); });
} }
module.exports = { module.exports = {
openedReaders: {}, datastores: {},
closeReader(jslid) { // closeReader(jslid) {
// console.log('CLOSING READER'); // // console.log('CLOSING READER');
if (!this.openedReaders[jslid]) return Promise.resolve(); // if (!this.openedReaders[jslid]) return Promise.resolve();
return new Promise((resolve, reject) => { // return new Promise((resolve, reject) => {
this.openedReaders[jslid].reader.close((err) => { // this.openedReaders[jslid].reader.close((err) => {
if (err) reject(err); // if (err) reject(err);
delete this.openedReaders[jslid]; // delete this.openedReaders[jslid];
resolve(); // resolve();
}); // });
}); // });
}, // },
readLine(readerInfo) { // readLine(readerInfo) {
return new Promise((resolve, reject) => { // return new Promise((resolve, reject) => {
const { reader } = readerInfo; // const { reader } = readerInfo;
if (!reader.hasNextLine()) { // if (!reader.hasNextLine()) {
resolve(null); // resolve(null);
return; // return;
} // }
reader.nextLine((err, line) => { // reader.nextLine((err, line) => {
if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1; // if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1;
else readerInfo.readedSchemaRow = true; // else readerInfo.readedSchemaRow = true;
if (err) reject(err); // if (err) reject(err);
resolve(line); // resolve(line);
}); // });
}); // });
}, // },
openReader(jslid) { // openReader(jslid) {
// console.log('OPENING READER'); // // console.log('OPENING READER');
// console.log( // // console.log(
// 'OPENING READER, LINES=', // // 'OPENING READER, LINES=',
// fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length // // fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length
// ); // // );
const file = getJslFileName(jslid); // const file = getJslFileName(jslid);
return new Promise((resolve, reject) => // return new Promise((resolve, reject) =>
lineReader.open(file, (err, reader) => { // lineReader.open(file, (err, reader) => {
if (err) reject(err); // if (err) reject(err);
const readerInfo = { // const readerInfo = {
reader, // reader,
readedDataRowCount: 0, // readedDataRowCount: 0,
readedSchemaRow: false, // readedSchemaRow: false,
isReading: true, // isReading: true,
}; // };
this.openedReaders[jslid] = readerInfo; // this.openedReaders[jslid] = readerInfo;
resolve(readerInfo); // resolve(readerInfo);
}) // })
); // );
}, // },
async ensureReader(jslid, offset) { // async ensureReader(jslid, offset) {
if (this.openedReaders[jslid] && this.openedReaders[jslid].readedDataRowCount > offset) { // if (this.openedReaders[jslid] && this.openedReaders[jslid].readedDataRowCount > offset) {
await this.closeReader(jslid); // await this.closeReader(jslid);
// }
// let readerInfo = this.openedReaders[jslid];
// if (!this.openedReaders[jslid]) {
// readerInfo = await this.openReader(jslid);
// }
// readerInfo.isReading = true;
// if (!readerInfo.readedSchemaRow) {
// await this.readLine(readerInfo); // skip structure
// }
// while (readerInfo.readedDataRowCount < offset) {
// await this.readLine(readerInfo);
// }
// return readerInfo;
// },
async ensureDatastore(jslid) {
let datastore = this.datastores[jslid];
if (!datastore) {
datastore = new JsonLinesDatastore(getJslFileName(jslid));
// datastore = new DatastoreProxy(getJslFileName(jslid));
this.datastores[jslid] = datastore;
} }
let readerInfo = this.openedReaders[jslid]; return datastore;
if (!this.openedReaders[jslid]) {
readerInfo = await this.openReader(jslid);
}
readerInfo.isReading = true;
if (!readerInfo.readedSchemaRow) {
await this.readLine(readerInfo); // skip structure
}
while (readerInfo.readedDataRowCount < offset) {
await this.readLine(readerInfo);
}
return readerInfo;
}, },
getInfo_meta: 'get', getInfo_meta: 'get',
@ -101,20 +113,8 @@ module.exports = {
getRows_meta: 'get', getRows_meta: 'get',
async getRows({ jslid, offset, limit }) { async getRows({ jslid, offset, limit }) {
const readerInfo = await this.ensureReader(jslid, offset); const datastore = await this.ensureDatastore(jslid);
const res = []; return datastore.getRows(offset, limit);
for (let i = 0; i < limit; i += 1) {
const line = await this.readLine(readerInfo);
if (line == null) break;
res.push(JSON.parse(line));
}
readerInfo.isReading = false;
if (readerInfo.closeAfterReadAndSendStats) {
await this.closeReader(jslid);
socket.emit(`jsldata-stats-${jslid}`, readerInfo.closeAfterReadAndSendStats);
readerInfo.closeAfterReadAndSendStats = null;
}
return res;
}, },
getStats_meta: 'get', getStats_meta: 'get',
@ -126,12 +126,16 @@ module.exports = {
async notifyChangedStats(stats) { async notifyChangedStats(stats) {
console.log('SENDING STATS', JSON.stringify(stats)); console.log('SENDING STATS', JSON.stringify(stats));
const readerInfo = this.openedReaders[stats.jslid]; const datastore = this.datastores[stats.jslid];
if (readerInfo && readerInfo.isReading) { if (datastore) await datastore.notifyChanged();
readerInfo.closeAfterReadAndSendStats = stats; socket.emit(`jsldata-stats-${stats.jslid}`, stats);
} else {
await this.closeReader(stats.jslid); // const readerInfo = this.openedReaders[stats.jslid];
socket.emit(`jsldata-stats-${stats.jslid}`, stats); // if (readerInfo && readerInfo.isReading) {
} // readerInfo.closeAfterReadAndSendStats = stats;
// } else {
// await this.closeReader(stats.jslid);
// socket.emit(`jsldata-stats-${stats.jslid}`, stats);
// }
}, },
}; };

2
packages/api/src/proc/index.js
View File

@ -2,10 +2,12 @@ const connectProcess = require('./connectProcess');
const databaseConnectionProcess = require('./databaseConnectionProcess'); const databaseConnectionProcess = require('./databaseConnectionProcess');
const serverConnectionProcess = require('./serverConnectionProcess'); const serverConnectionProcess = require('./serverConnectionProcess');
const sessionProcess = require('./sessionProcess'); const sessionProcess = require('./sessionProcess');
const jslDatastoreProcess = require('./jslDatastoreProcess');
module.exports = { module.exports = {
connectProcess, connectProcess,
databaseConnectionProcess, databaseConnectionProcess,
serverConnectionProcess, serverConnectionProcess,
sessionProcess, sessionProcess,
jslDatastoreProcess,
}; };

57
packages/api/src/proc/jslDatastoreProcess.js Normal file
View File

@ -0,0 +1,57 @@
const childProcessChecker = require('../utility/childProcessChecker');
const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
let lastPing = null;
let datastore = new JsonLinesDatastore();
function handlePing() {
lastPing = new Date().getTime();
}
function handleOpen({file }) {
handlePing();
datastore = new JsonLinesDatastore(file);
}
async function handleRead({ msgid, offset, limit }) {
handlePing();
const rows = await datastore.getRows(offset, limit);
process.send({ msgtype: 'response', msgid, rows });
}
function handleNotifyChanged() {
datastore.notifyChanged();
}
const messageHandlers = {
open: handleOpen,
read: handleRead,
ping: handlePing,
notifyChanged: handleNotifyChanged,
};
async function handleMessage({ msgtype, ...other }) {
const handler = messageHandlers[msgtype];
await handler(other);
}
function start() {
childProcessChecker();
setInterval(() => {
const time = new Date().getTime();
if (time - lastPing > 60 * 1000) {
process.exit(0);
}
}, 60 * 1000);
process.on('message', async (message) => {
try {
await handleMessage(message);
} catch (e) {
process.send({ msgtype: 'error', error: e.message });
}
});
}
module.exports = { start };

55
packages/api/src/utility/DatastoreProxy.js Normal file
View File

@ -0,0 +1,55 @@
const { fork } = require('child_process');
const uuidv1 = require('uuid/v1');
class DatastoreProxy {
constructor(file) {
this.subprocess = null;
this.disconnected = false;
this.file = file;
this.requests = {};
this.handle_response = this.handle_response.bind(this);
this.handle_ping = this.handle_ping.bind(this);
}
// handle_response({ msgid, rows }) {
handle_response({ msgid, rows }) {
const [resolve, reject] = this.requests[msgid];
resolve(rows);
delete this.requests[msgid];
}
handle_ping() {}
async ensureSubprocess() {
if (!this.subprocess) {
this.subprocess = fork(process.argv[1], ['jslDatastoreProcess']);
// @ts-ignore
this.subprocess.on('message', ({ msgtype, ...message }) => {
// if (this.disconnected) return;
this[`handle_${msgtype}`](message);
});
this.subprocess.on('exit', () => {
// if (this.disconnected) return;
this.subprocess = null;
});
this.subprocess.send({ msgtype: 'open', file: this.file });
}
return this.subprocess;
}
async getRows(offset, limit) {
await this.ensureSubprocess();
const msgid = uuidv1();
const promise = new Promise((resolve, reject) => {
this.requests[msgid] = [resolve, reject];
this.subprocess.send({ msgtype: 'read', msgid, offset, limit });
});
return promise;
}
async notifyChanged() {}
}
module.exports = DatastoreProxy;

115
packages/api/src/utility/JsonLinesDatastore.js Normal file
View File

@ -0,0 +1,115 @@
const lineReader = require('line-reader');
class JsonLinesDatastore {
constructor(file) {
this.file = file;
this.readerInfo = {
reader: null,
readedDataRowCount: 0,
readedSchemaRow: false,
isReading: false,
closeAfterRead: null,
closeAfterReadPromise: null,
};
}
async closeReader() {
if (!this.readerInfo) return Promise.resolve();
return new Promise((resolve, reject) => {
this.readerInfo.reader.close((err) => {
if (err) reject(err);
resolve();
});
});
}
async notifyChanged() {
if (this.readerInfo && this.readerInfo.isReading) {
if (this.readerInfo.closeAfterRead) {
return this.readerInfo.closeAfterReadPromise;
}
const promise = new Promise((resolve, reject) => {
this.readerInfo.closeAfterRead = resolve;
});
this.readerInfo.closeAfterReadPromise = promise;
return promise;
} else {
await this.closeReader();
}
}
async openReader() {
return new Promise((resolve, reject) =>
lineReader.open(this.file, (err, reader) => {
if (err) reject(err);
const readerInfo = {
reader,
readedDataRowCount: 0,
readedSchemaRow: false,
isReading: true,
closeAfterRead: null,
closeAfterReadPromise: null,
};
this.readerInfo = readerInfo;
resolve(readerInfo);
})
);
}
readLine(readerInfo) {
return new Promise((resolve, reject) => {
const { reader } = readerInfo;
if (!reader.hasNextLine()) {
resolve(null);
return;
}
reader.nextLine((err, line) => {
if (readerInfo.readedSchemaRow) readerInfo.readedDataRowCount += 1;
else readerInfo.readedSchemaRow = true;
if (err) reject(err);
resolve(line);
});
});
}
async ensureReader(offset) {
if (this.readerInfo && this.readerInfo.readedDataRowCount > offset) {
await this.closeReader();
}
let readerInfo = this.readerInfo;
if (!readerInfo || !readerInfo.reader) {
readerInfo = await this.openReader();
}
readerInfo.isReading = true;
if (!readerInfo.readedSchemaRow) {
await this.readLine(readerInfo); // skip structure
}
while (readerInfo.readedDataRowCount < offset) {
await this.readLine(readerInfo);
}
return readerInfo;
}
async getRows(offset, limit) {
const readerInfo = await this.ensureReader(offset);
const res = [];
for (let i = 0; i < limit; i += 1) {
const line = await this.readLine(readerInfo);
if (line == null) break;
res.push(JSON.parse(line));
}
readerInfo.isReading = false;
if (readerInfo.closeAfterRead) {
await this.closeReader();
// socket.emit(`jsldata-stats-${jslid}`, readerInfo.closeAfterReadAndSendStats);
const resolve = readerInfo.closeAfterRead;
readerInfo.closeAfterRead = null;
readerInfo.closeAfterReadPromise = null;
resolve();
}
return res;
}
}
module.exports = JsonLinesDatastore;