From 9c09ee3b7106e9df39ace68d61f6959daee7b04a Mon Sep 17 00:00:00 2001 From: Nick O'Leary Date: Fri, 12 Feb 2021 18:14:13 +0000 Subject: [PATCH] Rework Function node module integration --- .../@node-red/editor-api/lib/admin/index.js | 5 - .../@node-red/editor-api/lib/admin/nodes.js | 40 - .../editor-client/locales/en-US/editor.json | 1 + .../@node-red/editor-client/src/js/red.js | 14 + .../editor-client/src/js/ui/common/tabs.js | 4 +- .../editor-client/src/js/ui/palette-editor.js | 68 +- .../editor-client/src/js/ui/utils.js | 65 +- .../editor-client/src/sass/base.scss | 7 + .../nodes/core/function/10-function.html | 807 ++++++------------ .../nodes/core/function/10-function.js | 312 +++---- .../nodes/locales/en-US/messages.json | 2 +- .../@node-red/registry/lib/externalModules.js | 209 +++++ .../@node-red/registry/lib/index.js | 4 + .../@node-red/registry/lib/registry.js | 19 +- .../@node-red/registry/lib/util.js | 11 +- .../@node-red/runtime/lib/api/nodes.js | 42 - .../@node-red/runtime/lib/flows/index.js | 78 +- .../@node-red/runtime/lib/nodes/index.js | 16 +- .../@node-red/runtime/lib/nodes/npmModule.js | 460 ---------- .../registry/lib/externalModules_spec.js | 20 + 20 files changed, 800 insertions(+), 1384 deletions(-) create mode 100644 packages/node_modules/@node-red/registry/lib/externalModules.js delete mode 100644 packages/node_modules/@node-red/runtime/lib/nodes/npmModule.js create mode 100644 test/unit/@node-red/registry/lib/externalModules_spec.js diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/index.js b/packages/node_modules/@node-red/editor-api/lib/admin/index.js index 5de09acad..34c47b2cb 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/index.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/index.js @@ -67,11 +67,6 @@ module.exports = { adminApp.get(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.read"),nodes.getSet,apiUtil.errorHandler); adminApp.put(/^\/nodes\/((@[^\/]+\/)?[^\/]+)\/([^\/]+)$/,needsPermission("nodes.write"),nodes.putSet,apiUtil.errorHandler); - // NPM Modules - adminApp.get("/modules", needsPermission("nodes.read"), nodes.listNPMModules, apiUtil.errorHandler); - adminApp.delete("/modules/:spec", needsPermission("nodes.write"), nodes.uninstallNPMModule, apiUtil.errorHandler); - adminApp.post("/modules", needsPermission("nodes.write"), nodes.updateNPMModule, apiUtil.errorHandler); - // Context adminApp.get("/context/:scope(global)",needsPermission("context.read"),context.get,apiUtil.errorHandler); adminApp.get("/context/:scope(global)/*",needsPermission("context.read"),context.get,apiUtil.errorHandler); diff --git a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js index 77efedd11..058053a29 100644 --- a/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js +++ b/packages/node_modules/@node-red/editor-api/lib/admin/nodes.js @@ -191,45 +191,5 @@ module.exports = { runtimeAPI.nodes.getIconList(opts).then(function(list) { res.json(list); }); - }, - - listNPMModules: function(req, res) { - var opts = { - user: req.user, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.listNPMModules(opts).then(function(list) { - res.json(list); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - });; - }, - - uninstallNPMModule: function(req, res) { - var opts = { - user: req.user, - spec: req.params.spec, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.uninstallNPMModule(opts).then(function(result) { - res.json(result); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - }); - }, - - updateNPMModule: function(req, res) { - var body = req.body; - var opts = { - user: req.user, - spec: body.spec, - update: body.update, - req: apiUtils.getRequestLogObject(req) - } - runtimeAPI.nodes.updateNPMModule(opts).then(function(result) { - res.json(result); - }).catch(err => { - apiUtils.rejectHandler(req,res,err); - }); } }; diff --git a/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json b/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json index 7894f1ed2..e4c585811 100755 --- a/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json +++ b/packages/node_modules/@node-red/editor-client/locales/en-US/editor.json @@ -142,6 +142,7 @@ "nodeActionDisabled": "node actions disabled", "nodeActionDisabledSubflow": "node actions disabled within subflow", "missing-types": "

Flows stopped due to missing node types.

", + "missing-modules": "

Flows stopped due to missing modules.

", "safe-mode":"

Flows stopped in safe mode.

You can modify your flows and deploy the changes to restart.

", "restartRequired": "Node-RED must be restarted to enable upgraded modules", "credentials_load_failed": "

Flows stopped as the credentials could not be decrypted.

The flow credential file is encrypted, but the project's encryption key is missing or invalid.

", diff --git a/packages/node_modules/@node-red/editor-client/src/js/red.js b/packages/node_modules/@node-red/editor-client/src/js/red.js index 97159e6d2..e6eb381ff 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/red.js +++ b/packages/node_modules/@node-red/editor-client/src/js/red.js @@ -248,6 +248,7 @@ var RED = (function() { id: notificationId } if (notificationId === "runtime-state") { + RED.events.emit("runtime-state",msg); if (msg.error === "safe-mode") { options.buttons = [ { @@ -280,6 +281,16 @@ var RED = (function() { } ] } + } else if (msg.error === "missing-modules") { + text+=""; + options.buttons = [ + { + text: RED._("common.label.close"), + click: function() { + persistentNotifications[notificationId].hideNotification(); + } + } + ] } else if (msg.error === "credentials_load_failed") { if (RED.settings.theme("projects.enabled",false)) { // projects enabled @@ -370,6 +381,9 @@ var RED = (function() { } else if (persistentNotifications.hasOwnProperty(notificationId)) { persistentNotifications[notificationId].close(); delete persistentNotifications[notificationId]; + if (notificationId === 'runtime-state') { + RED.events.emit("runtime-state",msg); + } } }); RED.comms.subscribe("status/#",function(topic,msg) { diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js index 81cd407eb..1d6c75c32 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/common/tabs.js @@ -509,8 +509,8 @@ RED.tabs = (function() { li.attr('id',"red-ui-tab-"+(tab.id.replace(".","-"))); li.data("tabId",tab.id); - if (options.maximumTabWidth) { - li.css("maxWidth",options.maximumTabWidth+"px"); + if (options.maximumTabWidth || tab.maximumTabWidth) { + li.css("maxWidth",(options.maximumTabWidth || tab.maximumTabWidth) +"px"); } var link = $("",{href:"#"+tab.id, class:"red-ui-tab-label"}).appendTo(li); if (tab.icon) { diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js b/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js index 3ff3e7436..89df706f2 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/palette-editor.js @@ -331,7 +331,7 @@ RED.palette.editor = (function() { if (v.modules) { var a = false; v.modules = v.modules.filter(function(m) { - if (checkModuleAllowed(m.id,m.version,installAllowList,installDenyList)) { + if (RED.utils.checkModuleAllowed(m.id,m.version,installAllowList,installDenyList)) { loadedIndex[m.id] = m; m.index = [m.id]; if (m.keywords) { @@ -445,68 +445,6 @@ RED.palette.editor = (function() { var installAllowList = ['*']; var installDenyList = []; - function parseModuleList(list) { - list = list || ["*"]; - return list.map(function(rule) { - var m = /^(.+?)(?:@(.*))?$/.exec(rule); - var wildcardPos = m[1].indexOf("*"); - wildcardPos = wildcardPos===-1?Infinity:wildcardPos; - - return { - module: new RegExp("^"+m[1].replace(/\*/g,".*")+"$"), - version: m[2], - wildcardPos: wildcardPos - } - }) - } - - function checkAgainstList(module,version,list) { - for (var i=0;i deniedRule.wildcardPos - } else { - // First wildcard in same position. - // Go with the longer matching rule. This isn't going to be 100% - // right, but we are deep into edge cases at this point. - return allowedRule.module.toString().length > deniedRule.module.toString().length - } - return false; - } - function init() { if (RED.settings.get('externalModules.palette.allowInstall', true) === false) { return; @@ -517,8 +455,8 @@ RED.palette.editor = (function() { installAllowList = settingsAllowList; installDenyList = settingsDenyList } - installAllowList = parseModuleList(installAllowList); - installDenyList = parseModuleList(installDenyList); + installAllowList = RED.utils.parseModuleList(installAllowList); + installDenyList = RED.utils.parseModuleList(installDenyList); createSettingsPane(); diff --git a/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js b/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js index 230f561f9..5be2d6c40 100644 --- a/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js +++ b/packages/node_modules/@node-red/editor-client/src/js/ui/utils.js @@ -1122,6 +1122,67 @@ RED.utils = (function() { return '#'+'000000'.slice(0, 6-s.length)+s; } + function parseModuleList(list) { + list = list || ["*"]; + return list.map(function(rule) { + var m = /^(.+?)(?:@(.*))?$/.exec(rule); + var wildcardPos = m[1].indexOf("*"); + wildcardPos = wildcardPos===-1?Infinity:wildcardPos; + + return { + module: new RegExp("^"+m[1].replace(/\*/g,".*")+"$"), + version: m[2], + wildcardPos: wildcardPos + } + }) + } + + function checkAgainstList(module,version,list) { + for (var i=0;i deniedRule.wildcardPos + } else { + // First wildcard in same position. + // Go with the longer matching rule. This isn't going to be 100% + // right, but we are deep into edge cases at this point. + return allowedRule.module.toString().length > deniedRule.module.toString().length + } + return false; + } return { createObjectElement: buildMessageElement, getMessageProperty: getMessageProperty, @@ -1141,6 +1202,8 @@ RED.utils = (function() { sanitize: sanitize, renderMarkdown: renderMarkdown, createNodeIcon: createNodeIcon, - getDarkerColor: getDarkerColor + getDarkerColor: getDarkerColor, + parseModuleList: parseModuleList, + checkModuleAllowed: checkModuleAllowed } })(); diff --git a/packages/node_modules/@node-red/editor-client/src/sass/base.scss b/packages/node_modules/@node-red/editor-client/src/sass/base.scss index 391ddf83f..0a2d7407e 100644 --- a/packages/node_modules/@node-red/editor-client/src/sass/base.scss +++ b/packages/node_modules/@node-red/editor-client/src/sass/base.scss @@ -143,6 +143,13 @@ background-size: contain } + .red-ui-font-code { + font-family: $monospace-font; + font-size: $primary-font-size; + color: $info-text-code-color; + white-space: nowrap; + } + code { font-family: $monospace-font; font-size: $primary-font-size; diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.html b/packages/node_modules/@node-red/nodes/core/function/10-function.html index e28413790..5b0e8da44 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.html +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.html @@ -1,20 +1,67 @@ diff --git a/packages/node_modules/@node-red/nodes/core/function/10-function.js b/packages/node_modules/@node-red/nodes/core/function/10-function.js index 1b53374c2..10ffc1189 100644 --- a/packages/node_modules/@node-red/nodes/core/function/10-function.js +++ b/packages/node_modules/@node-red/nodes/core/function/10-function.js @@ -91,13 +91,17 @@ module.exports = function(RED) { function FunctionNode(n) { var libs = n.libs || []; n.modules = libs.map(x => x.spec).filter(x => (x && (x !== ""))); - var loadPromise = RED.nodes.createNode(this,n); + RED.nodes.createNode(this,n); var node = this; node.name = n.name; node.func = n.func; node.ini = n.initialize ? n.initialize.trim() : ""; node.fin = n.finalize ? n.finalize.trim() : ""; - node.libs = libs; + node.libs = libs || []; + + if (RED.settings.functionExternalModules === false && node.libs.length > 0) { + throw new Error("Function node not allowed to load external modules"); + } var handleNodeDoneCall = true; @@ -287,173 +291,173 @@ module.exports = function(RED) { } }); - // wait for module installation - loadPromise.catch(()=>{ - }).finally(function () { - if (node.hasOwnProperty("libs")) { - var modules = node.libs; - modules.forEach(module => { - var vname = module.hasOwnProperty("vname") ? module.vname : null; - if (vname && (vname !== "")) { - sandbox[vname] = null; - try { - var spec = module.spec; - if (spec && (spec !== "")) { - var lib = RED.require(module.spec); - sandbox[vname] = lib; + if (node.hasOwnProperty("libs")) { + var modules = node.libs; + modules.forEach(module => { + var vname = module.hasOwnProperty("var") ? module.var : null; + if (vname && (vname !== "")) { + sandbox[vname] = null; + try { + var spec = module.spec; + if (spec && (spec !== "")) { + var lib = RED.require(module.spec); + sandbox[vname] = lib; + } + } + catch (e) { + console.log(e); + node.warn("failed to load library: "+ module.spec); + } + } + }); + } + var context = vm.createContext(sandbox); + try { + var iniScript = null; + var iniOpt = null; + if (node.ini && (node.ini !== "")) { + var iniText = ` + (async function(__send__) { + var node = { + id:__node__.id, + name:__node__.name, + log:__node__.log, + error:__node__.error, + warn:__node__.warn, + debug:__node__.debug, + trace:__node__.trace, + status:__node__.status, + send: function(msgs, cloneMsg) { + __node__.send(__send__, RED.util.generateId(), msgs, cloneMsg); + } + }; + `+ node.ini +` + })(__initSend__);`; + iniOpt = createVMOpt(node, " setup"); + iniScript = new vm.Script(iniText, iniOpt); + } + node.script = vm.createScript(functionText, createVMOpt(node, "")); + if (node.fin && (node.fin !== "")) { + var finText = "(function () {\n"+node.fin +"\n})();"; + finOpt = createVMOpt(node, " cleanup"); + finScript = new vm.Script(finText, finOpt); + } + var promise = Promise.resolve(); + if (iniScript) { + context.__initSend__ = function(msgs) { node.send(msgs); }; + promise = iniScript.runInContext(context, iniOpt); + } + + processMessage = function (msg, send, done) { + var start = process.hrtime(); + context.msg = msg; + context.__send__ = send; + context.__done__ = done; + + node.script.runInContext(context); + context.results.then(function(results) { + sendResults(node,send,msg._msgid,results,false); + if (handleNodeDoneCall) { + done(); + } + + var duration = process.hrtime(start); + var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; + node.metric("duration", msg, converted); + if (process.env.NODE_RED_FUNCTION_TIME) { + node.status({fill:"yellow",shape:"dot",text:""+converted}); + } + }).catch(err => { + if ((typeof err === "object") && err.hasOwnProperty("stack")) { + //remove unwanted part + var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); + err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); + var stack = err.stack.split(/\r?\n/); + + //store the error in msg to be used in flows + msg.error = err; + + var line = 0; + var errorMessage; + if (stack.length > 0) { + while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { + line++; + } + + if (line < stack.length) { + errorMessage = stack[line]; + var m = /:(\d+):(\d+)$/.exec(stack[line+1]); + if (m) { + var lineno = Number(m[1])-1; + var cha = m[2]; + errorMessage += " (line "+lineno+", col "+cha+")"; + } } } - catch (e) { - node.warn("failed to load library: "+ module.spec); + if (!errorMessage) { + errorMessage = err.toString(); } + done(errorMessage); + } + else if (typeof err === "string") { + done(err); + } + else { + done(JSON.stringify(err)); } }); } - var context = vm.createContext(sandbox); - try { - var iniScript = null; - var iniOpt = null; - if (node.ini && (node.ini !== "")) { - var iniText = ` - (async function(__send__) { - var node = { - id:__node__.id, - name:__node__.name, - log:__node__.log, - error:__node__.error, - warn:__node__.warn, - debug:__node__.debug, - trace:__node__.trace, - status:__node__.status, - send: function(msgs, cloneMsg) { - __node__.send(__send__, RED.util.generateId(), msgs, cloneMsg); - } - }; - `+ node.ini +` - })(__initSend__);`; - iniOpt = createVMOpt(node, " setup"); - iniScript = new vm.Script(iniText, iniOpt); + + node.on("close", function() { + if (finScript) { + try { + finScript.runInContext(context, finOpt); + } + catch (err) { + node.error(err); + } } - node.script = vm.createScript(functionText, createVMOpt(node, "")); - if (node.fin && (node.fin !== "")) { - var finText = "(function () {\n"+node.fin +"\n})();"; - finOpt = createVMOpt(node, " cleanup"); - finScript = new vm.Script(finText, finOpt); + while (node.outstandingTimers.length > 0) { + clearTimeout(node.outstandingTimers.pop()); } - var promise = Promise.resolve(); - if (iniScript) { - context.__initSend__ = function(msgs) { node.send(msgs); }; - promise = iniScript.runInContext(context, iniOpt); + while (node.outstandingIntervals.length > 0) { + clearInterval(node.outstandingIntervals.pop()); } + if (node.clearStatus) { + node.status({}); + } + }); - processMessage = function (msg, send, done) { - var start = process.hrtime(); - context.msg = msg; - context.__send__ = send; - context.__done__ = done; - - node.script.runInContext(context); - context.results.then(function(results) { - sendResults(node,send,msg._msgid,results,false); - if (handleNodeDoneCall) { - done(); - } - - var duration = process.hrtime(start); - var converted = Math.floor((duration[0] * 1e9 + duration[1])/10000)/100; - node.metric("duration", msg, converted); - if (process.env.NODE_RED_FUNCTION_TIME) { - node.status({fill:"yellow",shape:"dot",text:""+converted}); - } - }).catch(err => { - if ((typeof err === "object") && err.hasOwnProperty("stack")) { - //remove unwanted part - var index = err.stack.search(/\n\s*at ContextifyScript.Script.runInContext/); - err.stack = err.stack.slice(0, index).split('\n').slice(0,-1).join('\n'); - var stack = err.stack.split(/\r?\n/); - - //store the error in msg to be used in flows - msg.error = err; - - var line = 0; - var errorMessage; - if (stack.length > 0) { - while (line < stack.length && stack[line].indexOf("ReferenceError") !== 0) { - line++; - } - - if (line < stack.length) { - errorMessage = stack[line]; - var m = /:(\d+):(\d+)$/.exec(stack[line+1]); - if (m) { - var lineno = Number(m[1])-1; - var cha = m[2]; - errorMessage += " (line "+lineno+", col "+cha+")"; - } - } - } - if (!errorMessage) { - errorMessage = err.toString(); - } - done(errorMessage); - } - else if (typeof err === "string") { - done(err); - } - else { - done(JSON.stringify(err)); - } + promise.then(function (v) { + var msgs = messages; + messages = []; + while (msgs.length > 0) { + msgs.forEach(function (s) { + processMessage(s.msg, s.send, s.done); }); + msgs = messages; + messages = []; } + state = RESOLVED; + }).catch((error) => { + messages = []; + state = ERROR; + node.error(error); + }); - node.on("close", function() { - if (finScript) { - try { - finScript.runInContext(context, finOpt); - } - catch (err) { - node.error(err); - } - } - while (node.outstandingTimers.length > 0) { - clearTimeout(node.outstandingTimers.pop()); - } - while (node.outstandingIntervals.length > 0) { - clearInterval(node.outstandingIntervals.pop()); - } - if (node.clearStatus) { - node.status({}); - } - }); - - promise.then(function (v) { - var msgs = messages; - messages = []; - while (msgs.length > 0) { - msgs.forEach(function (s) { - processMessage(s.msg, s.send, s.done); - }); - msgs = messages; - messages = []; - } - state = RESOLVED; - }).catch((error) => { - messages = []; - state = ERROR; - node.error(error); - }); - - } - catch(err) { - // eg SyntaxError - which v8 doesn't include line number information - // so we can't do better than this - updateErrorInfo(err); - node.error(err); - } - }); + } + catch(err) { + // eg SyntaxError - which v8 doesn't include line number information + // so we can't do better than this + updateErrorInfo(err); + node.error(err); + } } RED.nodes.registerType("function",FunctionNode, { - dynamicModuleList: "modules" + dynamicModuleList: "libs", + settings: { + functionExternalModules: { value: true, exportable: true } + } }); RED.library.register("functions"); }; diff --git a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json index a2e529d16..20180f33d 100755 --- a/packages/node_modules/@node-red/nodes/locales/en-US/messages.json +++ b/packages/node_modules/@node-red/nodes/locales/en-US/messages.json @@ -220,7 +220,7 @@ "finalize": "// Code added here will be run when the\n// node is being stopped or re-deployed.\n" }, "require": { - "var": "name", + "var": "variable", "module": "module" }, "error": { diff --git a/packages/node_modules/@node-red/registry/lib/externalModules.js b/packages/node_modules/@node-red/registry/lib/externalModules.js new file mode 100644 index 000000000..6486ce8b3 --- /dev/null +++ b/packages/node_modules/@node-red/registry/lib/externalModules.js @@ -0,0 +1,209 @@ +// This module handles the management of modules required by the runtime and flows. +// Essentially this means keeping track of what extra modules a flow requires, +// ensuring those modules are installed and providing a standard way for nodes +// to require those modules safely. + +const fs = require("fs-extra"); +const registryUtil = require("./util"); +const path = require("path"); +const exec = require("@node-red/util").exec; +const log = require("@node-red/util").log; + +const BUILTIN_MODULES = require('module').builtinModules; +const EXTERNAL_MODULES_DIR = "externalModules"; + +// TODO: outsource running npm to a plugin +const NPM_COMMAND = (process.platform === "win32") ? "npm.cmd" : "npm"; + +let registeredTypes = {}; +let settings; + +let knownExternalModules = {}; + +let installEnabled = true; +let installAllowList = ['*']; +let installDenyList = []; + +function getInstallDir() { + return path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", "externalModules")); +} + +async function refreshExternalModules() { + const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR)); + try { + const pkgFile = JSON.parse(await fs.readFile(path.join(externalModuleDir,"package.json"),"utf-8")); + knownExternalModules = pkgFile.dependencies; + } catch(err) { + } +} + +function init(_settings) { + settings = _settings; + path.resolve(settings.userDir || process.env.NODE_RED_HOME || "."); + + if (settings.externalModules && settings.externalModules.modules) { + if (settings.externalModules.modules.allowList || settings.externalModules.modules.denyList) { + installAllowList = settings.externalModules.modules.allowList; + installDenyList = settings.externalModules.modules.denyList; + } + if (settings.externalModules.modules.hasOwnProperty("allowInstall")) { + installEnabled = settings.externalModules.modules.allowInstall + } + } + installAllowList = registryUtil.parseModuleList(installAllowList); + installDenyList = registryUtil.parseModuleList(installDenyList); +} + +function register(type, dynamicModuleListProperty) { + registeredTypes[type] = dynamicModuleListProperty; +} + +function requireModule(module) { + if (!registryUtil.checkModuleAllowed( module, null,installAllowList,installDenyList)) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + throw e; + } + if (BUILTIN_MODULES.indexOf(module) !== -1) { + return require(module); + } + if (!knownExternalModules[module]) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + throw e; + } + const externalModuleDir = path.resolve(path.join(settings.userDir || process.env.NODE_RED_HOME || ".", EXTERNAL_MODULES_DIR)); + const moduleDir = path.join(externalModuleDir,"node_modules",module); + return require(moduleDir); +} + +function parseModuleName(module) { + var match = /((?:@[^/]+\/)?[^/@]+)(?:@([\s\S]+))?/.exec(module); + if (match) { + return { + spec: module, + module: match[1], + version: match[2], + builtin: BUILTIN_MODULES.indexOf(match[1]) !== -1, + known: !!knownExternalModules[match[1]] + } + } + return null; +} + +function isInstalled(moduleDetails) { + return moduleDetails.builtin || moduleDetails.known; +} + +async function checkFlowDependencies(flowConfig) { + await refreshExternalModules(); + + const checkedModules = {}; + const promises = []; + const errors = []; + + flowConfig.forEach(n => { + if (registeredTypes[n.type]) { + let nodeModules = n[registeredTypes[n.type]] || []; + if (!Array.isArray(nodeModules)) { + nodeModules = [nodeModules] + } + nodeModules.forEach(module => { + if (typeof module !== 'string') { + module = module.module || ""; + } + if (module) { + let moduleDetails = parseModuleName(module) + if (moduleDetails && checkedModules[moduleDetails.module] === undefined) { + checkedModules[moduleDetails.module] = isInstalled(moduleDetails) + if (!checkedModules[moduleDetails.module]) { + if (installEnabled) { + promises.push(installModule(moduleDetails).catch(err => { + errors.push({module: moduleDetails,error:err}); + })) + } else if (!installEnabled) { + const e = new Error("Module install disabled - externalModules.modules.allowInstall=false"); + e.code = "install_not_allowed"; + errors.push({module: moduleDetails,error:e}); + } + } else if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) { + const e = new Error("Module not allowed"); + e.code = "module_not_allowed"; + errors.push({module: moduleDetails,error:e}); + } + } + } + }) + } + }) + + return Promise.all(promises).then(() => { + if (errors.length > 0) { + throw errors; + } + }) +} + + +async function ensureModuleDir() { + const installDir = getInstallDir(); + + if (!fs.existsSync(installDir)) { + await fs.ensureDir(installDir); + } + const pkgFile = path.join(installDir,"package.json"); + if (!fs.existsSync(pkgFile)) { + await fs.writeFile(path.join(installDir,"package.json"),`{ + "name": "Node-RED External Modules", + "version": "1.0.0", + "private": true, + "dependencies": {} +}`) + } +} + +async function installModule(moduleDetails) { + let installSpec = moduleDetails.module; + if (!registryUtil.checkModuleAllowed( moduleDetails.module, moduleDetails.version,installAllowList,installDenyList)) { + const e = new Error("Install not allowed"); + e.code = "install_not_allowed"; + throw e; + } + if (moduleDetails.version) { + installSpec = installSpec+"@"+moduleDetails.version; + } + log.info(log._("server.install.installing",{name: moduleDetails.module,version: moduleDetails.version||"latest"})); + const installDir = getInstallDir(); + + await ensureModuleDir(); + + var args = ["install", installSpec, "--production"]; + return exec.run(NPM_COMMAND, args, { + cwd: installDir + },true).then(result => { + log.info("successfully installed: "+installSpec); + }).catch(result => { + var output = result.stderr; + var e; + var lookForVersionNotFound = new RegExp("version not found: ","m"); + if (/E404/.test(output) || /ETARGET/.test(output)) { + log.error(log._("server.install.install-failed-not-found",{name:installSpec})); + e = new Error("Module not found"); + e.code = 404; + throw e; + } else { + log.error(log._("server.install.install-failed-long",{name:installSpec})); + log.error("------------------------------------------"); + log.error(output); + log.error("------------------------------------------"); + throw new Error(log._("server.install.install-failed")); + } + }) +} + +module.exports = { + init: init, + register: register, + checkFlowDependencies: checkFlowDependencies, + require: requireModule +} \ No newline at end of file diff --git a/packages/node_modules/@node-red/registry/lib/index.js b/packages/node_modules/@node-red/registry/lib/index.js index 03f979424..ef091992a 100644 --- a/packages/node_modules/@node-red/registry/lib/index.js +++ b/packages/node_modules/@node-red/registry/lib/index.js @@ -28,6 +28,7 @@ var registry = require("./registry"); var loader = require("./loader"); var installer = require("./installer"); var library = require("./library"); +const externalModules = require("./externalModules") /** * Initialise the registry with a reference to a runtime object @@ -42,6 +43,7 @@ function init(runtime) { loader.init(runtime); registry.init(runtime.settings,loader); library.init(); + externalModules.init(runtime.settings); } /** @@ -297,6 +299,8 @@ module.exports = { */ getNodeExampleFlowPath: library.getExampleFlowPath, + checkFlowDependencies: externalModules.checkFlowDependencies, + deprecated: require("./deprecated") }; diff --git a/packages/node_modules/@node-red/registry/lib/registry.js b/packages/node_modules/@node-red/registry/lib/registry.js index 730280d72..f2ea23b2f 100644 --- a/packages/node_modules/@node-red/registry/lib/registry.js +++ b/packages/node_modules/@node-red/registry/lib/registry.js @@ -21,6 +21,7 @@ var fs = require("fs"); var library = require("./library"); const {events} = require("@node-red/util") var subflows = require("./subflow"); +var externalModules = require("./externalModules") var settings; var loader; @@ -28,6 +29,7 @@ var nodeConfigCache = {}; var moduleConfigs = {}; var nodeList = []; var nodeConstructors = {}; +var nodeOptions = {}; var subflowModules = {}; var nodeTypeToId = {}; @@ -36,12 +38,7 @@ var moduleNodes = {}; function init(_settings,_loader) { settings = _settings; loader = _loader; - moduleNodes = {}; - nodeTypeToId = {}; - nodeConstructors = {}; - subflowModules = {}; - nodeList = []; - nodeConfigCache = {}; + clear(); } function load() { @@ -234,6 +231,7 @@ function removeNode(id) { if (typeId === id) { delete subflowModules[t]; delete nodeConstructors[t]; + delete nodeOptions[t]; delete nodeTypeToId[t]; } }); @@ -411,7 +409,7 @@ function getCaller(){ return stack[0].getFileName(); } -function registerNodeConstructor(nodeSet,type,constructor) { +function registerNodeConstructor(nodeSet,type,constructor,options) { if (nodeConstructors.hasOwnProperty(type)) { throw new Error(type+" already registered"); } @@ -431,6 +429,12 @@ function registerNodeConstructor(nodeSet,type,constructor) { } nodeConstructors[type] = constructor; + nodeOptions[type] = options; + if (options) { + if (options.dynamicModuleList) { + externalModules.register(type,options.dynamicModuleList); + } + } events.emit("type-registered",type); } @@ -525,6 +529,7 @@ function clear() { moduleConfigs = {}; nodeList = []; nodeConstructors = {}; + nodeOptions = {}; subflowModules = {}; nodeTypeToId = {}; } diff --git a/packages/node_modules/@node-red/registry/lib/util.js b/packages/node_modules/@node-red/registry/lib/util.js index 15eca575b..d6433f6f4 100644 --- a/packages/node_modules/@node-red/registry/lib/util.js +++ b/packages/node_modules/@node-red/registry/lib/util.js @@ -17,6 +17,7 @@ const path = require("path"); const semver = require("semver"); const {events,i18n,log} = require("@node-red/util"); + var runtime; function copyObjectProperties(src,dst,copyList,blockList) { @@ -45,13 +46,8 @@ function requireModule(name) { var relPath = path.relative(__dirname, moduleInfo.path); return require(relPath); } else { - var npm = runtime.nodes.loadNPMModule(name); - if (npm) { - return npm; - } - var err = new Error(`Cannot find module '${name}'`); - err.code = "MODULE_NOT_FOUND"; - throw err; + // Require it here to avoid the circular dependency + return require("./externalModules").require(name); } } @@ -129,7 +125,6 @@ function checkAgainstList(module,version,list) { } function checkModuleAllowed(module,version,allowList,denyList) { - // console.log("checkModuleAllowed",module,version);//,allowList,denyList) if (!allowList && !denyList) { // Default to allow return true; diff --git a/packages/node_modules/@node-red/runtime/lib/api/nodes.js b/packages/node_modules/@node-red/runtime/lib/api/nodes.js index 3e74e3a97..556e57df9 100644 --- a/packages/node_modules/@node-red/runtime/lib/api/nodes.js +++ b/packages/node_modules/@node-red/runtime/lib/api/nodes.js @@ -447,47 +447,5 @@ var api = module.exports = { } else { return null } - }, - - /** - * Gets list of NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - list of installed NPM modules - * @memberof @node-red/runtime_nodes - */ - listNPMModules: async function(opts) { - var promise = runtime.nodes.listNPMModules(); - return promise; - }, - - /** - * Uninstall NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - object for request result - * @memberof @node-red/runtime_nodes - */ - uninstallNPMModule: async function(opts) { - var spec = opts.spec; - var promise = runtime.nodes.uninstallNPMModule(spec); - return promise; - }, - - /** - * Update NPM modules - * @param {Object} opts - * @param {User} opts.user - the user calling the api - * @param {Object} opts.req - the request to log (optional) - * @return {Promise} - object for request result - * @memberof @node-red/runtime_nodes - */ - updateNPMModule: async function(opts) { - var spec = opts.spec; - var isUpdate = opts.update; - var promise = runtime.nodes.updateNPMModule(spec, isUpdate); - return promise; } } diff --git a/packages/node_modules/@node-red/runtime/lib/flows/index.js b/packages/node_modules/@node-red/runtime/lib/flows/index.js index b31dc7755..bdcb9ae75 100644 --- a/packages/node_modules/@node-red/runtime/lib/flows/index.js +++ b/packages/node_modules/@node-red/runtime/lib/flows/index.js @@ -187,35 +187,35 @@ function setFlows(_config,_credentials,type,muteLog,forceStart,user) { }); } - return configSavePromise - .then(function(flowRevision) { - if (!isLoad) { - log.debug("saved flow revision: "+flowRevision); - } - activeConfig = { - flows:config, - rev:flowRevision - }; - activeFlowConfig = newFlowConfig; - if (forceStart || started) { - // Flows are running (or should be) - // Stop the active flows (according to deploy type and the diff) - return stop(type,diff,muteLog).then(() => { - // Once stopped, allow context to remove anything no longer needed - return context.clean(activeFlowConfig) - }).then(() => { - // Start the active flows - start(type,diff,muteLog).then(() => { - events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); - }); - // Return the new revision asynchronously to the actual start - return flowRevision; - }).catch(function(err) { }) - } else { - events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); - } - }); + return configSavePromise.then(flowRevision => { + if (!isLoad) { + log.debug("saved flow revision: "+flowRevision); + } + activeConfig = { + flows:config, + rev:flowRevision + }; + activeFlowConfig = newFlowConfig; + if (forceStart || started) { + // Flows are running (or should be) + + // Stop the active flows (according to deploy type and the diff) + return stop(type,diff,muteLog).then(() => { + // Once stopped, allow context to remove anything no longer needed + return context.clean(activeFlowConfig) + }).then(() => { + // Start the active flows + start(type,diff,muteLog).then(() => { + events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); + }); + // Return the new revision asynchronously to the actual start + return flowRevision; + }).catch(function(err) { }) + } else { + events.emit("runtime-event",{id:"runtime-deploy",payload:{revision:flowRevision},retain: true}); + } + }); } function getNode(id) { @@ -246,7 +246,7 @@ function getFlows() { return activeConfig; } -function start(type,diff,muteLog) { +async function start(type,diff,muteLog) { type = type||"full"; started = true; var i; @@ -271,7 +271,21 @@ function start(type,diff,muteLog) { log.info(" "+settings.userDir); } events.emit("runtime-event",{id:"runtime-state",payload:{error:"missing-types", type:"warning",text:"notification.warnings.missing-types",types:activeFlowConfig.missingTypes},retain:true}); - return Promise.resolve(); + return; + } + + try { + await typeRegistry.checkFlowDependencies(activeConfig.flows); + } catch(err) { + log.info("Failed to load external modules required by this flow:"); + const missingModules = []; + for (i=0;i (item.name === name)); - return item; -} - -function setInProgress(name) { - inProgress[name] = true; -} - -function clearInProgress(name) { - inProgress[name] = false; -} - -/** - * Install NPM module - * @param {string} module - module specification - */ -function installNPM(module) { - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - if (!pkg) { - var args = ["install", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - pkg = infoNPM(name); - var spec = name +(pkg ? "@"+pkg.version : ""); - log.info("successfully installed: "+spec); - var meta = loadMetadata(); - var item = { - name: name, - spec: module, - status: "installed", - }; - meta.modules.push(item); - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to install: "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to install: "+name - log.warn(msg); - reject(msg); - }); - } - else { - var meta = loadMetadata(); - if (!findModule(meta, name)) { - var item = { - name: name, - spec: module, - status: "preinstalled", - }; - meta.modules.push(item); - saveMetadata(meta); - } - clearInProgress(name); - var spec = name +(pkg ? ("@"+pkg.version) : ""); - log.info("already installed: "+spec); - } - resolve(true); - }); -} - -/** - * Check allowance of NPM module installation - * @param {string} name - module specification - */ -function isAllowed(name) { - if (!allowInstall) { - return false; - } - var [module, ver] = moduleName(name); - var aList = util.parseModuleList(allowList); - var dList = util.parseModuleList(denyList); - return util.checkModuleAllowed(module, ver, aList, dList); -} - -/** - * Check and install NPM module according to dynamic module specification - * @param {Object} node - node object - */ -function checkInstall(node) { - var name = null; - if(moduleProp.hasOwnProperty(node.type)) { - name = moduleProp[node.type]; - } - var promises = []; - if (name && node.hasOwnProperty(name)) { - var modules = node[name]; - modules.forEach(module => { - var name = module; - if ((typeof module === "object") && - module && - module.hasOwnProperty("name")) { - name = module.name; - } - if (isAllowed(name)) { - var [n, v] = moduleName(name); - setInProgress(name); - promises.push(installNPM(name)); - } - else { - log.info("installation not allowed: "+name); - } - }); - } - return Promise.all(promises); -} - -/** - * Load NPM module - * @param {string} module - module to load - */ -function load(module) { - try { - var [name, ver] = moduleName(module); - var path = modulePath("node_modules", name); - var npm = require(path); - return npm; - } - catch (e) { - return null; - } -} - -/** - * Get list of installed modules - */ -function listModules() { - return new Promise((resolve, reject) => { - var meta = loadMetadata(); - var modules = meta.modules; - modules.forEach(item => { - var name = item.name; - var info = infoNPM(name); - if (info) { - item.version = info.version; - } - item.inProgress = ((name in inProgress) && inProgress[name]); - }); - Object.keys(inProgress).forEach(name => { - if (inProgress[name] && - !modules.find(item => (item.name === name))) { - modules.push({ - name: name, - spec: name, - state: "inprogress", - inProgress: true - }); - } - }); - resolve(meta.modules); - }); -} - -/** - * Uninstall NPM modules - */ -function uninstall(module) { - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - var meta = loadMetadata(); - var item = findModule(meta, name); - if (pkg && item) { - if (item.status === "preinstalled") { - clearInProgress(name); - var msg = "can't uninstall preinstalled: "+name; - log.warn(msg); - reject(msg); - } - else { - var args = ["uninstall", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - log.info("successfully uninstalled: "+name); - var meta = loadMetadata(); - var items = meta.modules.filter(item => (item.name !== name)); - meta.modules = items; - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to uninstall: "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to uninstall: "+name; - log.warn(msg); - reject(msg); - }); - } - } - else { - clearInProgress(name); - var msg = "module not installed: "+name; - log.info(msg); - reject(msg); - } - }); -} - -/** - * Update NPM modules - */ -function update(module, isUpdate) { - var act = (isUpdate ? "updated": "install") - var acted = (isUpdate ? "updated": "installed") - var [name, ver] = moduleName(module); - setInProgress(name); - return new Promise((resolve, reject) => { - var pkg = infoNPM(name); - if (!pkg || isUpdate) { - var args = ["install", module]; - var dir = modulePath(); - return exec.run(npmCommand, args, { - cwd: dir - }, true).then(result => { - if (result && (result.code === 0)) { - pkg = infoNPM(name); - var spec = name +(pkg ? "@"+pkg.version : ""); - log.info("successfully "+acted+": "+spec); - var meta = loadMetadata(); - var items = meta.modules.filter(item => (item.name !== name)); - var item = { - name: name, - spec: module, - status: "installed", - }; - items.push(item); - meta.modules = items; - saveMetadata(meta); - clearInProgress(name); - resolve(true); - } - else { - clearInProgress(name); - var msg = "failed to "+act+": "+name; - log.warn(msg); - reject(msg); - } - }).catch(e => { - clearInProgress(name); - var msg = "failed to "+act+": "+name; - log.warn(msg); - reject(msg); - }); - } - else { - clearInProgress(name); - var msg = "not "+acted+": "+name; - log.info(msg); - reject(msg); - } - }); -} - -api = { - init: init, - register: register, - checkInstall: checkInstall, - load: load, - list: listModules, - uninstall: uninstall, - update: update -}; -module.exports = api; diff --git a/test/unit/@node-red/registry/lib/externalModules_spec.js b/test/unit/@node-red/registry/lib/externalModules_spec.js new file mode 100644 index 000000000..a08c30ed8 --- /dev/null +++ b/test/unit/@node-red/registry/lib/externalModules_spec.js @@ -0,0 +1,20 @@ + // init: init, + // register: register, + // checkFlowDependencies: checkFlowDependencies, + // require: requireModule + // + +const should = require("should"); + +const fs = require("fs"); +const path = require("path"); + +var NR_TEST_UTILS = require("nr-test-utils"); + +var externalModules = NR_TEST_UTILS.require("@node-red/registry/lib/externalModules"); + + +describe("externalModules api", function() { + + +}); \ No newline at end of file