How to escape special chars in JSON

Got some special chars in JSON that needed to be escapes.

var aJS = {txt : "012345\nabcde\näöüß"};
var s_aJS5 = E.toJS(aJS);
// "{txt:\"012345\\nabcde\\n\\xE4\\xF6\\xFC­\\xDF\"}";

// but in some cases additional encoding is required 
var s_aJS5_e = E.toJS(aJS).replace(/\\n/g,'\\\\n').repl­ace(/\\x/g,'\\\\x'); 
//  "{txt:\"012345\\\\nabcde\\\\n\\\\xE4\\\\­xF6\\\\xFC\\\\xDF\"}";

Is there a Espruino function for this, or could this be another candidate for StringDecoders?

I'm not sure I understand the issue here? E.toJS should escape everything enough for JSON. If you want to double-escape everything then you can always just run it through E.toJS twice?

I try so send JSON5 via nodejs and ble with this sample code https://www.espruino.com/Interfacing and ran into syntax error or eval issues.

So you're trying to parse the data you received on the PC with JSON.parse?

If so, if you're using E.toJS you actually need to use a JSON5 parser

Yes, PC sends JSON5 data to nRF device running on Espruino.
It is all about Umlaute and \n in stings in JSON that cause syntax and eval issues if not proper escaped.

Sorry, I'm not sure I understand...

Yes, PC sends JSON5 data to nRF device running on Espruino.

So it's Espruino that has trouble decoding the data from the PC? Or the PC that has trouble decoding data from Espruino?

Because if you're using E.toJS I assume it's the PC having trouble?

Sorry, I'm not sure I understand...

Sample code on Mac that is causing no issues when sending to a Espruino device after two time JSON5.stringify() like this:

var noble = require('@abandonware/noble');
const JSON5 = require('json5');
var  aJS = {txt : "012345\nabcde\näöüß"};
var  saJS5 = JSON5.stringify(aJS);    
// {txt:'012345\nabcde\näöüß'}
var sa2JS5 =  JSON5.stringify(saJS5);
// {txt:'012345\\nabcde\\näöüß'}
var COMMAND = `\x03\x10transfer(${sa2JS5})\n`;
// transfer("{txt:'012345\\nabcde\\näöüß'}"­)

// use  https://www.espruino.com/Interfacing  Node.js / JavaScript
// added some seconds before calling disconnect to catch more data

sample code on PuckJS

transfer = function (saJS5) {
        var aJS5;
        try {
            aJS5 = eval('('+saJS5+')');
        } catch(e){
            console.log("error in eval:",e);
        }
}
 

Sample code on PuckJS that is causing issues if not escaped when sending to a Espruino device after two times E.toJS

var aJS5 = {txt : "012345\nabcde\näöüß"};
var saJS5 = E.toJS(aJS5);  
//"{txt:\"012345\\nabcde\\n\\xE4\\xF6\\x­FC\\xDF\"}"
var sa2JS5 =  E.toJS(saJS5);
// "{txt:\"012345\\nabcde\\n\\xE4\\xF6\\xFC­\\xDF\"}"  // same
var COMMAND = `\x03\x10transfer(${sa2JS5})\n`;
// transfer("{txt:\"012345\\nabcde\\n\\xE4\­\xF6\\xFC\\xDF\"}")  <- cause syntax error


var saJS5e =  E.toJS(aJS5).replace(/\\n/g,'\\\\n').rep­lace(/\\x/g,'\\\\x'); 
// "{txt:\"012345\\\\nabcde\\\\n\\\\xE4\\\\­xF6\\\\xFC\\\\xDF\"}";
var COMMAND = `\x03\x10transfer(${saJS5e})\n`;
// "\3\x10transfer({txt:\"012345\\\\nabcde\­\\\n\\\\xE4\\\\xF6\\\\xFC\\\\xDF\"})\n";­ // <- no error

I just tried:

var aJS5 = {txt : "012345\nabcde\näöüß"};
var saJS5 = E.toJS(aJS5);  
//"{txt:\"012345\\nabcde\\n\\xE4\\xF6\\x­FC\\xDF\"}"
var sa2JS5 =  E.toJS(saJS5);
// "{txt:\"012345\\nabcde\\n\\xE4\\xF6\\xFC­\\xDF\"}"  // same
var COMMAND = `transfer(${sa2JS5})\n`;
// note I took the first 2 control chars out
eval(COMMAND); // this work as-is

// Exact code copied from your example that you say doesn't work
transfer("{txt:\"012345\\nabcde\\n\\xE4\­\xF6\\xFC\\xDF\"}") // works fine

>print(COMMAND);
transfer("{txt:\"012345\\nabcde\\n\\xE4\­\xF6\\xFC\\xDF\"}")
=undefined
//copy and paste
>transfer("{txt:\"012345\\nabcde\\n\\xE4­\\xF6\\xFC\\xDF\"}") // also works fine

in the emulator

So I don't really see what the problem is here?

I guess there is maybe some problem with how you are sending this? It seems to me from your replace of \ with \\ that somehow you're trying to evaluate/parse the string one more times than was intended

I guess there is maybe some problem with how you are sending this? It seems to me from your replace of \ with \ that somehow you're trying to evaluate/parse the string one more times than was intended

Wow - many thanks

code on Puck:

transfer = function (saJS5) {
        var aJS5;
        try {
            aJS5 = eval('('+saJS5+')');
            print("aJS5:",aJS5);
        } catch(e){
            console.log("error in eval:",e);
        }
};

I tried this:

transfer
COMMAND: transfer(""{txt:'012345\\nabcde\\näöüß'}­"")  // <- using extra " is the issue

Noble: stateChange -> poweredOn
Found:  Puck.js f5b9
BT> Connecting
BT> Connected
Received "<- Serial1\r\n>"
write: transfer(""{txt:'012345\\nabcde\\näöüß'}­"")   

Received "Uncaught SyntaxError"
Received ": Got '{' expected '"
Received ",'\r\n at line 1 col 1"
Received "2\r\ntransfer(\"\"{txt:'"
Received "012345\\\\nabcde\\\\n ?["
Received "228] ?[246] ?[252] ?"
Received "[223]'}\"\")\r\n        "
Received "                    "
Received "       ^\r\n>"
Disconnected

removing the extra " and it works fine

COMMAND: transfer("{txt:'012345\\nabcde\\näöüß'}"­)

Noble: stateChange -> poweredOn
Found:  Puck.js f5b9
BT> Connecting
BT> Connected
Received "<- Serial1\r\n>"
write: transfer("{txt:'012345\\nabcde\\näöüß'}"­)

Received "aJS5: { \r\n  \"txt\": \""
Received "012345\\nabcde\\n\\xE4\\"
Received "xF6\\xFC\\xDF\"\r\n }\r\n>"
Disconnected

Ok, let's look at JSON.

// receiving device
transfer = function (sJS) {
        try {
            print(JSON.parse(sJS));
        } catch(e){
            print(e);
       }
}

// sender device
bleSendJS = function(_id, _cmd) {
    print('_cmd:',_cmd);
    NRF.requestDevice({ filters: [{ id: id }] }).then(function(device) {
        return require("ble_uart").connect(device);
    }).then(function(uart) {
        uart.on('data', function(d) { print("Got:" + JSON.stringify(d)); });
        //uart.write(`\x03\x10transfer('${_cmd}'­)\n`);
        uart.write("\x03\x10transfer('"+_cmd+"')­\n");
        setTimeout(function() {
            uart.disconnect();
            console.log("Disconnected");
        }, 1E3);
    });
};

var aJS = JSON.stringify({txt : "012345\nabcde\näöüß"}),
        aJSe = JSON.stringify({txt : "012345\nabcde\näöüß"}).replace(/\\n/g, '\\\\n').replace(/\\u/g, '\\\\u');
    id = "ed:4f:05:32:73:ad random";

bleSendJS(id,aJS);
/* output
_cmd: {"txt":"012345\nabcde\n\u00E4\u00F6\u00F­C\u00DF"}
Got:"<- Serial1\r\n>"
Got:"SyntaxError: SyntaxE"
Got:"rror: Expecting a va"
Got:"lid value, got UNFIN"
Got:"ISHED STRING\r\n>"
Disconnected
*/

setTimeout(()=>{bleSendJS(id,aJSe);},2E4­);
/* output
_cmd: {"txt":"012345\\nabcde\\n\\u00E4\\u00F6\­\u00FC\\u00DF"}
Got:"<- Serial1\r\n>"
Got:"{ \r\n  \"txt\": \"012345"
Got:"\\nabcde\\n\\xE4\\xF6\\xF"
Got:"C\\xDF\"\r\n }\r\n>"
Disconnected

*/

Hope you have an idea why the extra escaping is needed?

        //uart.write(`\x03\x10transfer('${_cmd}'­)\n`);
        uart.write("\x03\x10transfer('"+_cmd+"')­\n");

Both of those are adding extra quotes, just in different ways?

What about:

        uart.write(`\x03\x10transfer(${_cmd})\n`­);

Both of those are adding extra quotes, just in different ways?

yes and both only work when sending aJSe.

What about ....

bleSendJS(id,aJS);
/* output
_cmd: {"txt":"012345\nabcde\n\u00E4\u00F6\u00F­C\u00DF"}
Got:"<- Serial1\r\n>"
Got:"SyntaxError: SyntaxE"
Got:"rror: Expecting a va"
Got:"lid value, got ID\r\n>"
Disconnected
*/
setTimeout(()=>{bleSendJS(id,aJSe);},2E4­);
/* output
_cmd: {"txt":"012345\\nabcde\\n\\u00E4\\u00F6\­\u00FC\\u00DF"}
Got:"<- Serial1\r\n>"
Got:"SyntaxError: SyntaxE"
Got:"rror: Expecting a va"
Got:"lid value, got ID\r\n>"
Disconnected

I don't understand. I ran this on a Pixl, talking to a Puck.js, and it works great using print:

function bleSendJS(_id, _cmd) {
  print('_cmd:',_cmd);
  NRF.requestDevice({ filters: [{ id: id }] }).then(function(device) {
      return require("ble_uart").connect(device);
  }).then(function(uart) {
    uart.on('data', function(d) { print("Got:" + JSON.stringify(d)); });
    uart.write(`\x03\x10print(${_cmd})\n`);
    setTimeout(function() {
        uart.disconnect();
        console.log("Disconnected");
    }, 1E3);
  });
}

var aJS = JSON.stringify({txt : "012345\nabcde\näöüß"}),    
id =  "c8:c1:f1:47:c7:79 random";
bleSendJS(id,aJS);

Reports:

Got:"{ \r\n  \"txt\": \"012345"
Got:"\\nabcde\\n\\xE4\\xF6\\xF"
Got:"C\\xDF\"\r\n }\r\n>"

Using var aJS = E.toJS({txt : "012345\nabcde\näöüß"}) instead does exactly the same.

Is your transfer function really:

transfer = function (saJS5) {
        var aJS5;
        try {
            aJS5 = eval('('+saJS5+')');
        } catch(e){
            console.log("error in eval:",e);
        }
}

If so then the issue is that in transfer you're trying to evaluate something that isn't a string. It's already a properly decoded JSON object.

So...

transfer = function (saJS5) {
        var aJS5;
        try {
            aJS5 = eval('('+saJS5+')');
        } catch(e){
            console.log("error in eval:",e);
        }
}

// What you think you're doing
transfer("{txt:\"012345\\nabcde\\n\\xE4\­\xF6\\xFC\\xDF\"}") // ok

// what you were doing just by adding '' around a string
transfer('{"txt":"012345\nabcde\n\u00E4\­u00F6\u00FC\u00DF"}') // error

// what you're actually doing
transfer({"txt":"012345\nabcde\n\u00E4\u­00F6\u00FC\u00DF"}) // error

Wow, just copied the code from your response and it works - I guess this has to do with code pages differences. I write code with a Mac OS editor and load the file into thee WebIDE.

// writing in WebIDE 
var aJS = JSON.stringify({txt : "012345\nabcde\näöüß"}),  

// load file after using the Mac editor
var aJS = JSON.stringify({txt : "012345\nabcde\n����"}),

Maybe some quotes got effected as well .....

Many thanks, using replace was just a workaround and I am happy to remove them again.

2 Attachments

Ok, so if print() work, then JSON.parse() should work too, but it causes an error.

...
    //uart.write(`\x03\x10print(${_cmd})\n`)­;
    uart.write(`\x03\x10JSON.parse(${_cmd})\­n`);
...

Reports:

Got:"Uncaught SyntaxError"
Got:": Expecting a valid "
Got:"value, got ID\r\n at l"
Got:"ine 1 col 1\r\n[object"
Got:" Object]\r\n^\r\n>"

if print() work, then JSON.parse() should work too

Nope... because cmd is {"txt":"012345\nabcde\n\u00E4\u00F6\u00F­C\u00DF"}

So what you're doing is telling Espruino to do : JSON.parse({"txt":"012345\nabcde\n\u00E4­\u00F6\u00FC\u00DF"})

The JSON is already parsed by the command-line. There's no need to parse it at all - what you get is the actual JSON object you wanted in the first place!

The JSON is already parsed by the command-line. There's no need to parse it at all - what you get is the actual JSON object you wanted in the first place!

Thanks for your support and clarification!