diff --git a/package-lock.json b/package-lock.json index 569e5466..b36e5b13 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9798,6 +9798,10 @@ "node": ">=8.6" } }, + "node_modules/migrations-test": { + "resolved": "tools/migrations-test", + "link": true + }, "node_modules/mime": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", @@ -13728,7 +13732,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "license": "MIT", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -14102,7 +14105,8 @@ "uuid": "^9.0.0", "validator": "^13.9.0", "winston": "^3.9.0", - "winston-daily-rotate-file": "^4.7.1" + "winston-daily-rotate-file": "^4.7.1", + "yargs": "^17.7.2" }, "devDependencies": { "@types/node": "^20.5.3", @@ -14399,6 +14403,21 @@ "js-levenshtein": "^1.1.6", "yaml": "^2.4.5" } + }, + "tools/migrations-test": { + "version": "1.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "commander": "^12.1.0" + } + }, + "tools/migrations-test/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "engines": { + "node": ">=18" + } } } } diff --git a/src/backend/package.json b/src/backend/package.json index f3d0d950..cce89a47 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -71,7 +71,8 @@ "uuid": "^9.0.0", "validator": "^13.9.0", "winston": "^3.9.0", - "winston-daily-rotate-file": "^4.7.1" + "winston-daily-rotate-file": "^4.7.1", + "yargs": "^17.7.2" }, "devDependencies": { "@types/node": "^20.5.3", diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js index c320cd15..ef875add 100644 --- a/src/backend/src/CoreModule.js +++ b/src/backend/src/CoreModule.js @@ -311,6 +311,12 @@ const install = async ({ services, app, useapi }) => { const { SUService } = require('./services/SUService'); services.registerService('su', SUService); + + const { ShutdownService } = require('./services/ShutdownService'); + services.registerService('shutdown', ShutdownService); + + const { BootScriptService } = require('./services/BootScriptService'); + services.registerService('boot-script', BootScriptService); } const install_legacy = async ({ services }) => { diff --git a/src/backend/src/Kernel.js b/src/backend/src/Kernel.js index 9f8f2b01..30320029 100644 --- a/src/backend/src/Kernel.js +++ b/src/backend/src/Kernel.js @@ -20,6 +20,9 @@ const { AdvancedBase } = require("@heyputer/puter-js-common"); const { Context } = require('./util/context'); const BaseService = require("./services/BaseService"); const useapi = require('useapi'); +const yargs = require('yargs/yargs') +const { hideBin } = require('yargs/helpers') + class Kernel extends AdvancedBase { constructor ({ entry_path } = {}) { @@ -67,7 +70,9 @@ class Kernel extends AdvancedBase { } boot () { - this._runtime_init(); + const args = yargs(hideBin(process.argv)).argv + + this._runtime_init({ args }); // const express = require('express') // const app = express(); @@ -111,6 +116,7 @@ class Kernel extends AdvancedBase { services, config, logger: this.bootLogger, + args, }, 'app'); globalThis.root_context = root_context; diff --git a/src/backend/src/services/BootScriptService.js b/src/backend/src/services/BootScriptService.js new file mode 100644 index 00000000..4b587cee --- /dev/null +++ b/src/backend/src/services/BootScriptService.js @@ -0,0 +1,42 @@ +const { Context } = require("../util/context"); +const BaseService = require("./BaseService"); + +class BootScriptService extends BaseService { + static MODULES = { + fs: require('fs'), + } + async ['__on_boot.ready'] () { + const args = Context.get('args'); + if ( ! args['boot-script'] ) return; + const script_name = args['boot-script']; + + const require = this.require; + const fs = require('fs'); + const boot_json_raw = fs.readFileSync(script_name, 'utf8'); + const boot_json = JSON.parse(boot_json_raw); + await this.run_script(boot_json); + } + + async run_script (boot_json) { + const scope = { + runner: 'boot-script', + ['end-puter-process']: ({ args }) => { + const svc_shutdown = this.services.get('shutdown'); + svc_shutdown.shutdown(args[0]); + } + }; + + for ( let i=0 ; i < boot_json.length ; i++ ) { + const statement = boot_json[i]; + const [cmd, ...args] = statement; + if ( ! scope[cmd] ) { + throw new Error(`Unknown command: ${cmd}`); + } + await scope[cmd]({ scope, args }); + } + } +} + +module.exports = { + BootScriptService +}; diff --git a/src/backend/src/services/ShutdownService.js b/src/backend/src/services/ShutdownService.js new file mode 100644 index 00000000..07a2a947 --- /dev/null +++ b/src/backend/src/services/ShutdownService.js @@ -0,0 +1,11 @@ +const BaseService = require("./BaseService"); + +class ShutdownService extends BaseService { + shutdown ({ reason, code } = {}) { + this.log.info(`Puter is shutting down: ${reason ?? 'no reason provided'}`); + process.stdout.write('\x1B[0m\r\n'); + process.exit(code ?? 0); + } +} + +module.exports = { ShutdownService }; diff --git a/src/backend/src/services/database/SqliteDatabaseAccessService.js b/src/backend/src/services/database/SqliteDatabaseAccessService.js index 3385970f..5458cbf3 100644 --- a/src/backend/src/services/database/SqliteDatabaseAccessService.js +++ b/src/backend/src/services/database/SqliteDatabaseAccessService.js @@ -18,6 +18,7 @@ */ const { es_import_promise } = require("../../fun/dev-console-ui-utils"); const { surrounding_box } = require("../../fun/dev-console-ui-utils"); +const { Context } = require("../../util/context"); const { CompositeError } = require("../../util/errorutil"); const structutil = require("../../util/structutil"); const { BaseDatabaseAccessService } = require("./BaseDatabaseAccessService"); @@ -44,7 +45,14 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService { this.db = new Database(this.config.path); // Database upgrade logic - const TARGET_VERSION = 24; + const HIGHEST_VERSION = 24; + const TARGET_VERSION = (() => { + const args = Context.get('args'); + if ( args['database-target-version'] ) { + return parseInt(args['database-target-version']); + } + return HIGHEST_VERSION; + })(); const [{ user_version }] = do_setup ? [{ user_version: -1 }] @@ -53,105 +61,93 @@ class SqliteDatabaseAccessService extends BaseDatabaseAccessService { const upgrade_files = []; - if ( user_version === -1 ) { - upgrade_files.push('0001_create-tables.sql'); - upgrade_files.push('0002_add-default-apps.sql'); - } + const available_migrations = [ + [-1, [ + '0001_create-tables.sql', + '0002_add-default-apps.sql', + ]], + [0, [ + '0003_user-permissions.sql', + ]], + [1, [ + '0004_sessions.sql', + ]], + [2, [ + '0005_background-apps.sql', + ]], + [3, [ + '0006_update-apps.sql', + ]], + [4, [ + '0007_sessions.sql', + ]], + [5, [ + '0008_otp.sql', + ]], + [6, [ + '0009_app-prefix-fix.sql', + ]], + [7, [ + '0010_add-git-app.sql', + ]], + [8, [ + '0011_notification.sql', + ]], + [9, [ + '0012_appmetadata.sql', + ]], + [10, [ + '0013_protected-apps.sql', + ]], + [11, [ + '0014_share.sql', + ]], + [12, [ + '0015_group.sql', + ]], + [13, [ + '0016_group-permissions.sql', + ]], + [14, [ + '0017_publicdirs.sql', + ]], + [15, [ + '0018_fix-0003.sql', + ]], + [16, [ + '0019_fix-0016.sql', + ]], + [17, [ + '0020_dev-center.sql', + ]], + [18, [ + '0021_app-owner-id.sql', + ]], + [19, [ + '0022_dev-center-max.sql', + ]], + [20, [ + '0023_fix-kv.sql', + ]], + [21, [ + '0024_default-groups.sql', + ]], + [22, [ + '0025_system-user.dbmig.js', + ]], + [23, [ + '0026_user-groups.dbmig.js', + ]], + ]; - if ( user_version <= 0 ) { - upgrade_files.push('0003_user-permissions.sql'); - } - - if ( user_version <= 1 ) { - upgrade_files.push('0004_sessions.sql'); - } - - if ( user_version <= 2 ) { - upgrade_files.push('0005_background-apps.sql'); - } - - if ( user_version <= 3 ) { - upgrade_files.push('0006_update-apps.sql'); - } - - if ( user_version <= 4 ) { - upgrade_files.push('0007_sessions.sql'); - } - - if ( user_version <= 5 ) { - upgrade_files.push('0008_otp.sql'); - } - - if ( user_version <= 6 ) { - upgrade_files.push('0009_app-prefix-fix.sql'); - } - - if ( user_version <= 7 ) { - upgrade_files.push('0010_add-git-app.sql'); - } - - if ( user_version <= 8 ) { - upgrade_files.push('0011_notification.sql'); - } - - if ( user_version <= 9 ) { - upgrade_files.push('0012_appmetadata.sql'); - } - - if ( user_version <= 10 ) { - upgrade_files.push('0013_protected-apps.sql'); - } - - if ( user_version <= 11 ) { - upgrade_files.push('0014_share.sql'); - } - - if ( user_version <= 12 ) { - upgrade_files.push('0015_group.sql'); - } - - if ( user_version <= 13 ) { - upgrade_files.push('0016_group-permissions.sql'); - } - - if ( user_version <= 14 ) { - upgrade_files.push('0017_publicdirs.sql'); - } - - if ( user_version <= 15 ) { - upgrade_files.push('0018_fix-0003.sql'); - } - - if ( user_version <= 16 ) { - upgrade_files.push('0019_fix-0016.sql'); - } - - if ( user_version <= 17 ) { - upgrade_files.push('0020_dev-center.sql'); - } - - if ( user_version <= 18 ) { - upgrade_files.push('0021_app-owner-id.sql'); - } - - if ( user_version <= 19 ) { - upgrade_files.push('0022_dev-center-max.sql'); - } - - if ( user_version <= 20 ) { - upgrade_files.push('0023_fix-kv.sql'); - } - - if ( user_version <= 21 ) { - upgrade_files.push('0024_default-groups.sql'); - } - - if ( user_version <= 22 ) { - upgrade_files.push('0025_system-user.dbmig.js'); - } - - if ( user_version <= 23 ) { - upgrade_files.push('0026_user-groups.dbmig.js'); + for ( const [v_lt_or_eq, files] of available_migrations ) { + if ( v_lt_or_eq + 1 >= TARGET_VERSION && TARGET_VERSION !== HIGHEST_VERSION ) { + this.log.noticeme(`Early exit: target version set to ${TARGET_VERSION}`); + break; + } + if ( user_version <= v_lt_or_eq ) { + upgrade_files.push(...files); + } } if ( upgrade_files.length > 0 ) { diff --git a/src/backend/src/services/runtime-analysis/AlarmService.js b/src/backend/src/services/runtime-analysis/AlarmService.js index 69711b96..177ca2ce 100644 --- a/src/backend/src/services/runtime-analysis/AlarmService.js +++ b/src/backend/src/services/runtime-analysis/AlarmService.js @@ -28,6 +28,7 @@ const { generate_identifier } = require('../../util/identifier.js'); const { stringify_log_entry } = require('./LogService.js'); const BaseService = require('../BaseService.js'); const { split_lines } = require('../../util/stdioutil.js'); +const { Context } = require('../../util/context.js'); class AlarmService extends BaseService { async _construct () { @@ -251,6 +252,15 @@ class AlarmService extends BaseService { svc_devConsole.add_widget(this.alarm_widget); } + const args = Context.get('args'); + if ( args['quit-on-alarm'] ) { + const svc_shutdown = this.services.get('shutdown'); + svc_shutdown.shutdown({ + reason: '--quit-on-alarm is set', + code: 1, + }); + } + if ( alarm.no_alert ) return; const severity = alarm.severity ?? 'critical'; diff --git a/tools/migrations-test/main.js b/tools/migrations-test/main.js new file mode 100644 index 00000000..5531867d --- /dev/null +++ b/tools/migrations-test/main.js @@ -0,0 +1,139 @@ +const path_ = require('node:path'); +const fs = require('node:fs'); +const { spawnSync } = require('node:child_process'); +const prompt = require('prompt-sync')({sigint: true}); + +const ind_str = () => Array(ind).fill(' --').join(''); + +let ind = 0; + +const log = { + // log with unicode warning symbols in yellow + warn: (msg) => { + console.log(`\x1b[33;1m[!]${ind_str()} ${msg}\x1b[0m`); + }, + crit: (msg) => { + console.log(`\x1b[31;1m[!]${ind_str()} ${msg}\x1b[0m`); + }, + info: (msg) => { + console.log(`\x1B[36;1m[i]\x1B[0m${ind_str()} ${msg}`); + }, + named: (name, value) => { + console.log(`\x1B[36;1m[i]${ind_str()} ${name}\x1B[0m ${value}`); + }, + error: e => { + if ( e instanceof UserError ) { + log.crit(e.message); + } else { + console.error(e); + } + }, + indent () { ind++; }, + dedent () { ind--; }, + heading (title) { + const circle = '🔵'; + console.log(`\n\x1b[36;1m${circle} ${title} ${circle}\x1b[0m`); + } +}; + +const areyousure = (message, options = {}) => { + const { crit } = options; + const logfn = crit ? log.crit : log.warn; + + logfn(message); + const answer = prompt(`\x1B[35;1m[?]\x1B[0m ${ options?.prompt ?? 'Are you sure?' } (y/n): `); + if ( answer !== 'y' ) { + + if ( options.fail_hint ) { + log.info(options.fail_hint); + } + + console.log(`\x1B[31;21;1mAborted.\x1B[0m`); + process.exit(1); + } +} + +if ( ! fs.existsSync('.is_puter_repository') ) { + throw new Error('This script must be run from the root of a puter repository'); +} + +areyousure( + 'This script will delete all data in the database. Are you sure you want to proceed?', + { crit: true } +) + +let backup_created = false; + +const DBPATH = 'volatile/runtime/puter-database.sqlite'; +const delete_db = () => { + if ( ! fs.existsSync(DBPATH) ) { + log.info('No database file to remove'); + // no need to create a backup if the database doesn't exist + backup_created = true; + return; + } + if ( ! backup_created ) { + log.info(`Creating a backup of the database...`); + const RANDOM = Math.floor(Math.random() * 1000000); + const DATE = new Date().toISOString().replace(/:/g, '-'); + fs.renameSync(DBPATH, `${DBPATH}_${DATE}_${RANDOM}.bak`); + backup_created = true; + return; + } + log.info('Removing database file'); + fs.unlinkSync(DBPATH); +} + +const pwd = process.cwd(); +const boot_script_path = path_.join(pwd, 'tools/migrations-test/noop.puter.json'); + +const launch_puter = (args) => { + const ret = spawnSync( + 'node', + ['tools/run-selfhosted.js', ...args], + { + stdio: 'inherit', + env: { + ...process.env, + NO_VAR_RUNTIME: '1', + }, + } + ); + ret.ok = ret.status === 0; + return ret; +}; + +{ + delete_db(); + log.info(`Test case: fresh install`); + if ( ! launch_puter([ + '--quit-on-alarm', + `--boot-script=${boot_script_path}`, + ]).ok ) { + log.crit('Migration to v21 raised alarm'); + process.exit(1); + } +} +{ + delete_db(); + log.info(`Test case: migrate to 21, then migrate to 24`); + if ( ! launch_puter([ + `--database-target-version=21`, + '--quit-on-alarm', + `--boot-script=${boot_script_path}`, + ]).ok ) { + log.crit('Migration to v21 raised alarm'); + process.exit(1); + } + if ( ! launch_puter([ + `--database-target-version=24`, + '--quit-on-alarm', + `--boot-script=${boot_script_path}`, + ]).ok ) { + log.crit('Migration to v24 raised alarm'); + process.exit(1); + } +} + +log.info('No migration scripts produced any obvious errors.'); +log.warn('This is not a substitute for release candidate migration testing!'); diff --git a/tools/migrations-test/noop.puter.json b/tools/migrations-test/noop.puter.json new file mode 100644 index 00000000..8afc8625 --- /dev/null +++ b/tools/migrations-test/noop.puter.json @@ -0,0 +1,3 @@ +[ + ["end-puter-process", { "reason": "migrations test" }] +] diff --git a/tools/migrations-test/package.json b/tools/migrations-test/package.json new file mode 100644 index 00000000..e960d4c0 --- /dev/null +++ b/tools/migrations-test/package.json @@ -0,0 +1,15 @@ +{ + "name": "migrations-test", + "version": "1.0.0", + "main": "main.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "AGPL-3.0-only", + "description": "", + "dependencies": { + "commander": "^12.1.0" + } +}