From 302c5cfe09f502a98717c927da49c4bdd4bfb23c Mon Sep 17 00:00:00 2001 From: Dave Conway-Jones Date: Fri, 12 Feb 2021 16:55:41 +0000 Subject: [PATCH] CSV node - handle commas in msg.columns if quoted. and add more tests To close #2860 --- .../@node-red/nodes/core/parsers/70-CSV.js | 29 +++++++-------- test/nodes/core/parsers/70-CSV_spec.js | 35 +++++++++++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js index aab27a55e..5e2576491 100644 --- a/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js +++ b/packages/node_modules/@node-red/nodes/core/parsers/70-CSV.js @@ -18,7 +18,7 @@ module.exports = function(RED) { "use strict"; function CSVNode(n) { RED.nodes.createNode(this,n); - this.template = (n.temp || "").split(","); + this.template = (n.temp || ""); this.sep = (n.sep || ',').replace("\\t","\t").replace("\\n","\n").replace("\\r","\r"); this.quo = '"'; this.ret = (n.ret || "\n").replace("\\n","\n").replace("\\r","\r"); @@ -38,16 +38,12 @@ module.exports = function(RED) { if (this.hdrout === true) { this.hdrout = "all"; } var tmpwarn = true; var node = this; + var re = new RegExp(',(?=(?:(?:[^"]*"){2})*[^"]*$)','g'); // pass in an array of column names to be trimed, de-quoted and retrimed var clean = function(col) { - for (var t = 0; t < col.length; t++) { - col[t] = col[t].trim(); // remove leading and trailing whitespace - if (col[t].charAt(0) === '"' && col[t].charAt(col[t].length -1) === '"') { - // remove leading and trailing quotes (if they exist) - and remove whitepace again. - col[t] = col[t].substr(1,col[t].length -2).trim(); - } - } + col = col.trim().split(re) || [""]; + col = col.map(x => x.replace(/"/g,'').trim()); if ((col.length === 1) && (col[0] === "")) { node.goodtmpl = false; } else { node.goodtmpl = true; } return col; @@ -67,13 +63,14 @@ module.exports = function(RED) { if (node.hdrout !== "none" && node.hdrSent === false) { if ((node.template.length === 1) && (node.template[0] === '')) { if (msg.hasOwnProperty("columns")) { - node.template = clean((msg.columns || "").split(",")); + node.template = clean(msg.columns || ""); } else { node.template = Object.keys(msg.payload[0]); } } - ou += node.template.join(node.sep) + node.ret; + // ou += node.template.join(node.sep) + node.ret; + ou += node.template.map(v => v.indexOf(node.sep)!==-1 ? '"'+v+'"' : v).join(node.sep) + node.ret; if (node.hdrout === "once") { node.hdrSent = true; } } for (var s = 0; s < msg.payload.length; s++) { @@ -93,7 +90,7 @@ module.exports = function(RED) { } else { if ((node.template.length === 1) && (node.template[0] === '') && (msg.hasOwnProperty("columns"))) { - node.template = clean((msg.columns || "").split(",")); + node.template = clean(msg.columns || "")//.split(",")); } if ((node.template.length === 1) && (node.template[0] === '')) { /* istanbul ignore else */ @@ -144,7 +141,7 @@ module.exports = function(RED) { } } msg.payload = ou; - msg.columns = node.template.join(','); + msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).join(','); if (msg.payload !== '') { send(msg); } done(); } @@ -179,7 +176,7 @@ module.exports = function(RED) { if ((node.hdrin === true) && first) { // if the template is in the first line if ((line[i] === "\n")||(line[i] === "\r")||(line.length - i === 1)) { // look for first line break if (line.length - i === 1) { tmp += line[i]; } - node.template = clean(tmp.split(node.sep)); + node.template = clean(tmp); first = false; } else { tmp += line[i]; } @@ -255,14 +252,14 @@ module.exports = function(RED) { } if (msg.parts.index + 1 === msg.parts.count) { msg.payload = node.store; - msg.columns = node.template.filter(val => val).join(','); + msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); delete msg.parts; send(msg); node.store = []; } } else { - msg.columns = node.template.filter(val => val).join(','); + msg.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); send(msg); // finally send the array } } @@ -270,7 +267,7 @@ module.exports = function(RED) { var len = a.length; for (var i = 0; i < len; i++) { var newMessage = RED.util.cloneMessage(msg); - newMessage.columns = node.template.filter(val => val).join(','); + newMessage.columns = node.template.map(v => v.indexOf(',')!==-1 ? '"'+v+'"' : v).filter(v => v).join(','); newMessage.payload = a[i]; if (!has_parts) { newMessage.parts = { diff --git a/test/nodes/core/parsers/70-CSV_spec.js b/test/nodes/core/parsers/70-CSV_spec.js index a6c24f202..4c0d6f00b 100644 --- a/test/nodes/core/parsers/70-CSV_spec.js +++ b/test/nodes/core/parsers/70-CSV_spec.js @@ -18,6 +18,7 @@ var should = require("should"); var csvNode = require("nr-test-utils").require("@node-red/nodes/core/parsers/70-CSV.js"); var helper = require("node-red-node-test-helper"); +const { c } = require("tar"); describe('CSV node', function() { @@ -136,6 +137,40 @@ describe('CSV node', function() { }); }); + it('should allow commas and spaces in the template', function(done) { + var flow = [ { id:"n1", type:"csv", temp:"a,b b,\"c,c\",\" d, d \"", wires:[["n2"]] }, + {id:"n2", type:"helper"} ]; + helper.load(csvNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); + check_parts(msg, 0, 1); + done(); + }); + var testString = "1,2,3,4"+String.fromCharCode(10); + n1.emit("input", {payload:testString}); + }); + }); + + it('should allow passing in a tempalte as first line of CSV', function(done) { + var flow = [ { id:"n1", type:"csv", temp:"", hdrin:true, wires:[["n2"]] }, + {id:"n2", type:"helper"} ]; + helper.load(csvNode, flow, function() { + var n1 = helper.getNode("n1"); + var n2 = helper.getNode("n2"); + n2.on("input", function(msg) { + msg.should.have.property('payload', { a: 1, "b b":2, "c,c":3, "d, d": 4 }); + msg.should.have.property('columns', 'a,b b,"c,c","d, d"'); + check_parts(msg, 0, 1); + done(); + }); + var testString = 'a,b b,"c,c"," d, d "'+"\n"+"1,2,3,4"+String.fromCharCode(10); + n1.emit("input", {payload:testString}); + }); + }); + it('should leave numbers starting with 0, e and + as strings (except 0.)', function(done) { var flow = [ { id:"n1", type:"csv", temp:"a,b,c,d,e,f,g", wires:[["n2"]] }, {id:"n2", type:"helper"} ];