/* eslint-disable prefer-rest-params -- don't want to change ...arguments usage for these sensitive functions without more testing */ import electron from 'electron'; import NeDB from 'nedb'; import fsPath from 'path'; import * as uuid from 'uuid'; import { mustGetModel } from '../models'; import { CookieJar } from '../models/cookie-jar'; import { Environment } from '../models/environment'; import { GitRepository } from '../models/git-repository'; import { getMonkeyPatchedControlledSettings } from '../models/helpers/settings'; import type { BaseModel } from '../models/index'; import * as models from '../models/index'; import { isSettings } from '../models/settings'; import type { Workspace } from '../models/workspace'; import { DB_PERSIST_INTERVAL } from './constants'; import { getDataDirectory } from './electron-helpers'; import { generateId } from './misc'; export interface Query { _id?: string | SpecificQuery; parentId?: string | null; remoteId?: string | null; plugin?: string; key?: string; environmentId?: string | null; protoFileId?: string; } type Sort = Record; interface Operation { upsert?: BaseModel[]; remove?: BaseModel[]; } export interface SpecificQuery { $gt?: number; $in?: string[]; $nin?: string[]; } export type ModelQuery = Partial>; export const database = { all: async function(type: string) { if (db._empty) return _send('all', ...arguments); return database.find(type); }, batchModifyDocs: async function({ upsert = [], remove = [] }: Operation) { if (db._empty) return _send('batchModifyDocs', ...arguments); const flushId = await database.bufferChanges(); // Perform from least to most dangerous await Promise.all(upsert.map(doc => database.upsert(doc, true))); await Promise.all(remove.map(doc => database.unsafeRemove(doc, true))); await database.flushChanges(flushId); }, /** buffers database changes and returns a buffer id */ bufferChanges: async function(millis = 1000) { if (db._empty) return _send('bufferChanges', ...arguments); bufferingChanges = true; setTimeout(database.flushChanges, millis); return ++bufferChangesId; }, /** buffers database changes and returns a buffer id */ bufferChangesIndefinitely: async function() { if (db._empty) return _send('bufferChangesIndefinitely', ...arguments); bufferingChanges = true; return ++bufferChangesId; }, CHANGE_INSERT: 'insert', CHANGE_UPDATE: 'update', CHANGE_REMOVE: 'remove', count: async function(type: string, query: Query = {}) { if (db._empty) return _send('count', ...arguments); return new Promise((resolve, reject) => { (db[type] as NeDB).count(query, (err, count) => { if (err) { return reject(err); } resolve(count); }); }); }, docCreate: async (type: string, ...patches: Patch[]) => { const doc = await models.initModel( type, ...patches, // Fields that the user can't touch { type: type, }, ); return database.insert(doc); }, docUpdate: async (originalDoc: T, ...patches: Patch[]) => { // No need to re-initialize the model during update; originalDoc will be in a valid state by virtue of loading const doc = await models.initModel( originalDoc.type, originalDoc, // NOTE: This is before `patches` because we want `patch.modified` to win if it has it { modified: Date.now(), }, ...patches, ); return database.update(doc); }, duplicate: async function(originalDoc: T, patch: Patch = {}) { if (db._empty) return _send('duplicate', ...arguments); const flushId = await database.bufferChanges(); async function next(docToCopy: T, patch: Patch) { const model = mustGetModel(docToCopy.type); const overrides = { _id: generateId(model.prefix), modified: Date.now(), created: Date.now(), type: docToCopy.type, // Ensure this is not overwritten by the patch }; // 1. Copy the doc const newDoc = Object.assign({}, docToCopy, patch, overrides); // Don't initialize the model during insert, and simply duplicate const createdDoc = await database.insert(newDoc, false, false); // 2. Get all the children for (const type of allTypes()) { // Note: We never want to duplicate a response if (!models.canDuplicate(type)) { continue; } const parentId = docToCopy._id; const children = await database.find(type, { parentId }); for (const doc of children) { await next(doc, { parentId: createdDoc._id }); } } return createdDoc; } const createdDoc = await next(originalDoc, patch); await database.flushChanges(flushId); return createdDoc; }, find: async function( type: string, query: Query | string = {}, sort: Sort = { created: 1 }, ) { if (db._empty) return _send('find', ...arguments); return new Promise((resolve, reject) => { (db[type] as NeDB) .find(query) .sort(sort) .exec(async (err, rawDocs) => { if (err) { reject(err); return; } const docs: T[] = []; for (const rawDoc of rawDocs) { docs.push(await models.initModel(type, rawDoc)); } resolve(docs); }); }); }, findMostRecentlyModified: async function( type: string, query: Query = {}, limit: number | null = null, ) { if (db._empty) return _send('findMostRecentlyModified', ...arguments); return new Promise(resolve => { (db[type] as NeDB) .find(query) .sort({ modified: -1, }) // @ts-expect-error -- TSCONVERSION limit shouldn't be applied if it's null, or default to something that means no-limit .limit(limit) .exec(async (err, rawDocs) => { if (err) { console.warn('[db] Failed to find docs', err); resolve([]); return; } const docs: T[] = []; for (const rawDoc of rawDocs) { docs.push(await models.initModel(type, rawDoc)); } resolve(docs); }); }); }, flushChanges: async function(id = 0, fake = false) { if (db._empty) return _send('flushChanges', ...arguments); // Only flush if ID is 0 or the current flush ID is the same as passed if (id !== 0 && bufferChangesId !== id) { return; } bufferingChanges = false; const changes = [...changeBuffer]; changeBuffer = []; if (changes.length === 0) { // No work to do return; } if (fake) { console.log(`[db] Dropped ${changes.length} changes.`); return; } // Notify local listeners too for (const fn of changeListeners) { await fn(changes); } // Notify remote listeners const windows = electron.BrowserWindow.getAllWindows(); for (const window of windows) { window.webContents.send('db.changes', changes); } }, flushChangesAsync: async (fake = false) => { process.nextTick(async () => { await database.flushChanges(0, fake); }); }, get: async function(type: string, id?: string) { if (db._empty) return _send('get', ...arguments); // Short circuit IDs used to represent nothing if (!id || id === 'n/a') { return null; } else { return database.getWhere(type, { _id: id }); } }, getMostRecentlyModified: async function(type: string, query: Query = {}) { if (db._empty) return _send('getMostRecentlyModified', ...arguments); const docs = await database.findMostRecentlyModified(type, query, 1); return docs.length ? docs[0] : null; }, getWhere: async function(type: string, query: ModelQuery | Query) { if (db._empty) return _send('getWhere', ...arguments); // @ts-expect-error -- TSCONVERSION type narrowing needed const docs = await database.find(type, query); return docs.length ? docs[0] : null; }, init: async ( types: string[], config: NeDB.DataStoreOptions = {}, forceReset = false, consoleLog: typeof console.log = console.log, ) => { if (forceReset) { changeListeners = []; for (const attr of Object.keys(db)) { if (attr === '_empty') { continue; } delete db[attr]; } } // Fill in the defaults for (const modelType of types) { if (db[modelType]) { consoleLog(`[db] Already initialized DB.${modelType}`); continue; } const filePath = getDBFilePath(modelType); const collection = new NeDB( Object.assign( { autoload: true, filename: filePath, corruptAlertThreshold: 0.9, }, config, ), ); collection.persistence.setAutocompactionInterval(DB_PERSIST_INTERVAL); db[modelType] = collection; } delete db._empty; electron.ipcMain.on('db.fn', async (e, fnName, replyChannel, ...args) => { try { const result = await database[fnName](...args); e.sender.send(replyChannel, null, result); } catch (err) { e.sender.send(replyChannel, { message: err.message, stack: err.stack, }); } }); // NOTE: Only repair the DB if we're not running in memory. Repairing here causes tests to hang indefinitely for some reason. // TODO: Figure out why this makes tests hang if (!config.inMemoryOnly) { await _repairDatabase(); consoleLog(`[db] Initialized DB at ${getDBFilePath('$TYPE')}`); } // This isn't the best place for this but w/e // Listen for response deletions and delete corresponding response body files database.onChange(async changes => { for (const [type, doc] of changes) { // TODO(TSCONVERSION) what's returned here is the entire model implementation, not just a model // The type definition will be a little confusing const m: Record | null = models.getModel(doc.type); if (!m) { continue; } if (type === database.CHANGE_REMOVE && typeof m.hookRemove === 'function') { try { await m.hookRemove(doc, consoleLog); } catch (err) { consoleLog(`[db] Delete hook failed for ${type} ${doc._id}: ${err.message}`); } } if (type === database.CHANGE_INSERT && typeof m.hookInsert === 'function') { try { await m.hookInsert(doc, consoleLog); } catch (err) { consoleLog(`[db] Insert hook failed for ${type} ${doc._id}: ${err.message}`); } } if (type === database.CHANGE_UPDATE && typeof m.hookUpdate === 'function') { try { await m.hookUpdate(doc, consoleLog); } catch (err) { consoleLog(`[db] Update hook failed for ${type} ${doc._id}: ${err.message}`); } } } }); for (const model of models.all()) { // @ts-expect-error -- TSCONVERSION optional type on response if (typeof model.hookDatabaseInit === 'function') { // @ts-expect-error -- TSCONVERSION optional type on response await model.hookDatabaseInit?.(consoleLog); } } }, initClient: async () => { electron.ipcRenderer.on('db.changes', async (_e, changes) => { for (const fn of changeListeners) { await fn(changes); } }); console.log('[db] Initialized DB client'); }, insert: async function(doc: T, fromSync = false, initializeModel = true) { if (db._empty) return _send('insert', ...arguments); return new Promise(async (resolve, reject) => { let docWithDefaults: T | null = null; try { if (initializeModel) { docWithDefaults = await models.initModel(doc.type, doc); } else { docWithDefaults = doc; } } catch (err) { return reject(err); } (db[doc.type] as NeDB).insert(docWithDefaults, (err, newDoc: T) => { if (err) { return reject(err); } resolve(newDoc); // NOTE: This needs to be after we resolve notifyOfChange(database.CHANGE_INSERT, newDoc, fromSync); }); }); }, onChange: (callback: ChangeListener) => { changeListeners.push(callback); }, offChange: (callback: ChangeListener) => { changeListeners = changeListeners.filter(l => l !== callback); }, remove: async function(doc: T, fromSync = false) { if (db._empty) return _send('remove', ...arguments); const flushId = await database.bufferChanges(); const docs = await database.withDescendants(doc); const docIds = docs.map(d => d._id); const types = [...new Set(docs.map(d => d.type))]; // Don't really need to wait for this to be over; types.map(t => db[t].remove( { _id: { $in: docIds, }, }, { multi: true, }, ), ); docs.map(d => notifyOfChange(database.CHANGE_REMOVE, d, fromSync)); await database.flushChanges(flushId); }, removeWhere: async function(type: string, query: Query) { if (db._empty) return _send('removeWhere', ...arguments); const flushId = await database.bufferChanges(); for (const doc of await database.find(type, query)) { const docs = await database.withDescendants(doc); const docIds = docs.map(d => d._id); const types = [...new Set(docs.map(d => d.type))]; // Don't really need to wait for this to be over; types.map(t => db[t].remove( { _id: { $in: docIds, }, }, { multi: true, }, ), ); docs.map(d => notifyOfChange(database.CHANGE_REMOVE, d, false)); } await database.flushChanges(flushId); }, /** Removes entries without removing their children */ unsafeRemove: async function(doc: T, fromSync = false) { if (db._empty) return _send('unsafeRemove', ...arguments); (db[doc.type] as NeDB).remove({ _id: doc._id }); notifyOfChange(database.CHANGE_REMOVE, doc, fromSync); }, update: async function(doc: T, fromSync = false) { if (db._empty) return _send('update', ...arguments); return new Promise(async (resolve, reject) => { let docWithDefaults: T; try { docWithDefaults = await models.initModel(doc.type, doc); } catch (err) { return reject(err); } (db[doc.type] as NeDB).update( { _id: docWithDefaults._id }, docWithDefaults, // TODO(TSCONVERSION) see comment below, upsert can happen automatically as part of the update // @ts-expect-error -- TSCONVERSION expects 4 args but only sent 3. Need to validate what UpdateOptions should be. err => { if (err) { return reject(err); } resolve(docWithDefaults); // NOTE: This needs to be after we resolve notifyOfChange(database.CHANGE_UPDATE, docWithDefaults, fromSync); }, ); }); }, // TODO(TSCONVERSION) the update method above can now take an upsert property upsert: async function(doc: T, fromSync = false) { if (db._empty) return _send('upsert', ...arguments); const existingDoc = await database.get(doc.type, doc._id); if (existingDoc) { return database.update(doc, fromSync); } else { return database.insert(doc, fromSync); } }, withAncestors: async function(doc: T | null, types: string[] = allTypes()) { if (db._empty) return _send('withAncestors', ...arguments); if (!doc) { return []; } let docsToReturn: T[] = doc ? [doc] : []; async function next(docs: T[]): Promise { const foundDocs: T[] = []; for (const d of docs) { for (const type of types) { // If the doc is null, we want to search for parentId === null const another = await database.get(type, d.parentId); another && foundDocs.push(another); } } if (foundDocs.length === 0) { // Didn't find anything. We're done return docsToReturn; } // Continue searching for children docsToReturn = [ ...docsToReturn, ...foundDocs, ]; return next(foundDocs); } return next([doc]); }, withDescendants: async function(doc: T | null, stopType: string | null = null): Promise { if (db._empty) return _send('withDescendants', ...arguments); let docsToReturn: BaseModel[] = doc ? [doc] : []; async function next(docs: (BaseModel | null)[]): Promise { let foundDocs: BaseModel[] = []; for (const doc of docs) { if (stopType && doc && doc.type === stopType) { continue; } const promises: Promise[] = []; for (const type of allTypes()) { // If the doc is null, we want to search for parentId === null const parentId = doc ? doc._id : null; const promise = database.find(type, { parentId }); promises.push(promise); } for (const more of await Promise.all(promises)) { foundDocs = [ ...foundDocs, ...more, ]; } } if (foundDocs.length === 0) { // Didn't find anything. We're done return docsToReturn; } // Continue searching for children docsToReturn = [...docsToReturn, ...foundDocs]; return next(foundDocs); } return next([doc]); }, }; interface DB { [index: string]: NeDB; } // @ts-expect-error -- TSCONVERSION _empty doesn't match the index signature, use something other than _empty in future const db: DB = { _empty: true, } as DB; // ~~~~~~~ // // HELPERS // // ~~~~~~~ // const allTypes = () => Object.keys(db); function getDBFilePath(modelType: string) { // NOTE: Do not EVER change this. EVER! return fsPath.join(getDataDirectory(), `insomnia.${modelType}.db`); } // ~~~~~~~~~~~~~~~~ // // Change Listeners // // ~~~~~~~~~~~~~~~~ // let bufferingChanges = false; let bufferChangesId = 1; type ChangeBufferEvent = [ event: string, doc: BaseModel, fromSync: boolean ]; let changeBuffer: ChangeBufferEvent[] = []; type ChangeListener = Function; let changeListeners: ChangeListener[] = []; async function notifyOfChange(event: string, doc: T, fromSync: boolean) { let updatedDoc = doc; // NOTE: this monkeypatching is temporary, and was determined to have the smallest blast radius if it exists here (rather than, say, a reducer or an action creator). // see: INS-1059 if (isSettings(doc)) { updatedDoc = getMonkeyPatchedControlledSettings(doc); } changeBuffer.push([event, updatedDoc, fromSync]); // Flush right away if we're not buffering if (!bufferingChanges) { await database.flushChanges(); } } // ~~~~~~~~~~~~~~~~~~~ // // DEFAULT MODEL STUFF // // ~~~~~~~~~~~~~~~~~~~ // type Patch = Partial; // ~~~~~~~ // // Helpers // // ~~~~~~~ // async function _send(fnName: string, ...args: any[]) { return new Promise((resolve, reject) => { const replyChannel = `db.fn.reply:${uuid.v4()}`; electron.ipcRenderer.send('db.fn', fnName, replyChannel, ...args); electron.ipcRenderer.once(replyChannel, (_e, err, result: T) => { if (err) { reject(err); } else { resolve(result); } }); }); } /** * Run various database repair scripts */ export async function _repairDatabase() { console.log('[fix] Running database repairs'); for (const workspace of await database.find(models.workspace.type)) { await _repairBaseEnvironments(workspace); await _fixMultipleCookieJars(workspace); await _applyApiSpecName(workspace); } for (const gitRepository of await database.find(models.gitRepository.type)) { await _fixOldGitURIs(gitRepository); } } /** * This function ensures that apiSpec exists for each workspace * If the filename on the apiSpec is not set or is the default initialized name * It will apply the workspace name to it */ async function _applyApiSpecName(workspace: Workspace) { const apiSpec = await models.apiSpec.getByParentId(workspace._id); if (apiSpec === null) { return; } if (!apiSpec.fileName || apiSpec.fileName === models.apiSpec.init().fileName) { await models.apiSpec.update(apiSpec, { fileName: workspace.name, }); } } /** * This function repairs workspaces that have multiple base environments. Since a workspace * can only have one, this function walks over all base environments, merges the data, and * moves all children as well. */ async function _repairBaseEnvironments(workspace: Workspace) { const baseEnvironments = await database.find(models.environment.type, { parentId: workspace._id, }); // Nothing to do here if (baseEnvironments.length <= 1) { return; } const chosenBase = baseEnvironments[0]; for (const baseEnvironment of baseEnvironments) { if (baseEnvironment._id === chosenBase._id) { continue; } chosenBase.data = Object.assign(baseEnvironment.data, chosenBase.data); const subEnvironments = await database.find(models.environment.type, { parentId: baseEnvironment._id, }); for (const subEnvironment of subEnvironments) { await database.docUpdate(subEnvironment, { parentId: chosenBase._id, }); } // Remove unnecessary base env await database.remove(baseEnvironment); } // Update remaining base env await database.update(chosenBase); console.log(`[fix] Merged ${baseEnvironments.length} base environments under ${workspace.name}`); } /** * This function repairs workspaces that have multiple cookie jars. Since a workspace * can only have one, this function walks over all jars and merges them and their cookies * together. */ async function _fixMultipleCookieJars(workspace: Workspace) { const cookieJars = await database.find(models.cookieJar.type, { parentId: workspace._id, }); // Nothing to do here if (cookieJars.length <= 1) { return; } const chosenJar = cookieJars[0]; for (const cookieJar of cookieJars) { if (cookieJar._id === chosenJar._id) { continue; } for (const cookie of cookieJar.cookies) { if (chosenJar.cookies.find(c => c.id === cookie.id)) { continue; } chosenJar.cookies.push(cookie); } // Remove unnecessary jar await database.remove(cookieJar); } // Update remaining jar await database.update(chosenJar); console.log(`[fix] Merged ${cookieJars.length} cookie jars under ${workspace.name}`); } // Append .git to old git URIs to mimic previous isomorphic-git behaviour async function _fixOldGitURIs(doc: GitRepository) { if (!doc.uriNeedsMigration) { return; } if (!doc.uri.endsWith('.git')) { doc.uri += '.git'; } doc.uriNeedsMigration = false; await database.update(doc); console.log(`[fix] Fixed git URI for ${doc._id}`); }