Trying to stringify an object to JSON, but not enough memory to fit the whole thing in memory

How do I do this?

I am currently doing this:

				res.writeHead(200,head);
				res.write("{\"Temp\":");
				res.write(JSON.stringify(HISTORY.Temp));­
				res.write(",\r\"RH\":");
				res.write(JSON.stringify(HISTORY.RH));
				res.write(",\"Pressure\":");
				res.write(JSON.stringify(HISTORY.Pressur­e));
				res.write(",\"AirQual\":");
				res.write(JSON.stringify(HISTORY.AirQual­));
				res.write(",\"Clear\":");
				res.write(JSON.stringify(HISTORY.Clear))­;
				res.write(",\"Red\":");
				res.write(JSON.stringify(HISTORY.Red));
				res.write(",\"Green\":");
				res.write(JSON.stringify(HISTORY.Green))­;
				res.write(",\"Blue\":");
				res.write(JSON.stringify(HISTORY.Blue));­
				res.write("}");

It doesn't seem any better than just doing res.write(JSON.stringify(HISTORY));

either way, it runs out of memory halfway through (the main reason being that some of the arrays being stringified are floats, so they get printed with a billion decimal places). I see that there's the .drain callback, but I can't find any example about how to use this. How do I keep track of where in the process I am so I know what to stringify next?

Thanks!

@DrAzzy, I do not know how your HISTORY is organized. I assume it is time series... so I suggest to chunk by time slices and you manage those with minimal information by keeping two (2) time values: - not transmitted yet, in transmission (up to including this time), successfully transmitted (up to including this time). Also, it seems that you do not need all this pseudo-precision: limit to the number of digits you need at capturing time and stick these into the history. If this is not enough yet, apply some scaling - may be you get away with a constant for each of the types of measurement - and keep the actual value and integer. Those values you push into the history and then you take it from there.

I don't understand what you're suggesting?

HISTORY object contains 8 typed 48 element arrays (representing data every 30 minutes), 4 Float32Arrays and 4 Uint16Arrays. The time data is not retained - I push onto one end and drop the oldest value every time I update it (by looping over the arrays since I can't do push() with a typed array). This works.

The problem is that I need to stringify the whole thing and send it as JSON to the webpage that will display it as a nice neat graph. However, I get out of memory when I do this - in the above, I've commented out all except RH, Temp, and Clear - which works, but then I don't get to have all the data I want.
I've tried breaking it all up into separate documents, but this doesn't work either - still runs out of memory, I think because it's not waiting until everything gets sent before stuffing more into the bufer.

It looks like the drain() method is made just for this kind of thing - but I don't know how to use it and there are no examples :-(

@DrAzzy,

Ic. Sorry not being more understanding of the data structure as conveyed by the shared code.

Based on above details I would make an initial request that just delivers the frame and code for subsequent - serialized - xhr requests for each of the 8 time series. The initial request delivers the list with the detail - type of the time series. The xhrs have to be serialized: after first returns data, data is put into a received history object, second is fired, and so fort. After last, you move on with with complete composed received History object to render it graphically, as you already do now.

Before you though go thru this hassle, I'm not sure if there is not a streaming way... which you hint with the .drain() - because I recall that images can be sent, and they are a lot of data...

Since everything is JS and thus single threaded - at least logically - you could enforce the writes to the result to be broken up by each of the history types and also throttle them the following way:

                ...
                .....  
                writeBegin();
}                

function writeBegin(res) {
  res.writeHead(200,head);              
  var types = "Temp","RH","Pressure","AirQual","Clear"­,"Red","Green","Blue"];
  setTimeout(writeTypedHistory,10,res,0,ty­pes);
}

function writeTypedHistory(res,tdx,types) {
  var type = types[tdx];
  res.write((tdx === 0) ? '{"' : ',"');
  res.write(type);
  res.write('":');
  res.write(JSON.stringify(History[type]))­ 
  if (++tdx < types.length) {
    setTimeout(writeTypedHistory,500,tdx,typ­es);
  } else {
    res.write('}');
  }
}

function writeEnd(res) {
                .....
                ...

Currently, throttling is done by constant time... but it could be variable or even driven by available memory.

HUZZAH!!!!! Thank you!!

htypes=["Temp","RH","Pressure","AirQual"­,"Clear","Red","Green","Blue"];

function writeTypedHistory(res,tdx) {
  var typen = htypes[tdx];
  res.write((tdx === 0) ? '{"' : ',"');
  res.write(typen);
  res.write('":');
  res.write(JSON.stringify(HISTORY[typen])­);
  if (++tdx < htypes.length) {
    setTimeout(writeTypedHistory,200,res,tdx­);
  } else {
    res.write('}');
    res.end();
  }
}



function processrequest(req, res) {
	try {
		var par=url.parse(req.url,true);
		var q=par.query; 
		var path=par.pathname; 
		path=path.split('/');
		path=path[1];
	    switch(path) {
	    	case "history.json":
			res.writeHead(200,head);
			setTimeout(writeTypedHistory,10,res,0);
	    		break;
<snip>
	    }
	}
	catch (err){
		res.writeHead(500,head);
	    res.write("{status:\"error\",error:\""+e­rr.message+"\"}");
		res.end();
	}
}

@DrAzzy, URVVVW! ...I guess your uppercase allows me to use it too without being too loud... ;-)

Btw, if I would be more experienced in the use await, async function and Promise - the code could be a bit more in-line... The solution I sketched was the crowbar way and way before Promise existed.

At the Chrome Dev Summit 2018 in SF mid Nov I noticed the await in conjunction with JS in the browser driving a Midi interface; see The Web-Connected MIDI Trumpet Robot from @urish">Uri Shaked - Uri also on Espruino forum w/ @urish.

At one time, the project used a Pixle in an enclosed, pressurized container controlling the compressor over BLE to drive the trumpet via latex lips... see youtube https://www.youtube.com/watch?time_conti­nue=1&v=9hOYcU34KMs
taken from We tried to build a Robot that plays the Trumpet and Happily Failed. But unfortunately, it was not reliable enough, and the solution as described in the link was used: focus a speaker to the mouth piece and use the tropet w/ correct fingering as a resonating transmission media/tube...)

I do not know about the implementation of await in Espruino. It would make many things very easy. The loop could then be inline and make the code very readable.

@allObjects there has been some discussion regarding async/await in Espruino.

Right now, you can use a build-step and a transpiler such as rollup to achieve the same. It also has some other advantages that help reduce bundled code size, I used it in my In-Real-Life Chrome T-Rex Project.

You can find an example setup someone created here

There's actually an example on using drain on the Internet page: http://www.espruino.com/Internet#transfe­rring-large-amounts-of-data

var history = new Uint8Array(10000);

// ...

function onPageRequest(req, res) {
  res.writeHead(200, {'Content-Type': 'text/plain'});
  var n = 0;
  res.on('drain',function() {
    for (var i=0;i<10;i++)
      res.write(history[n++]+",");
    if (n>=history.length) res.end(" ];");
  });
  res.write("var data = [");
}
require('http').createServer(onPageReque­st).listen(8080);

It should be pretty easy to tweak your example to use that rather than setTimeout @DrAzzy - also, if you wanted you could use htypes=Object.keys(HISTORY) - so that then every field in your object gets enumerated. If you wanted to avoid having the counter you could do the above line and could then just remove items from htypes as they were output - and then stop when it was empty.

Unfortunately async/await isn't in Espruino yet - its efficient implementation is particularly painful in Espruino. However honestly using Promises wouldn't be that much more typing and it's probably a lot more obvious what's happening. The current solution works fine without them though.

Thanks - adjusted code to use drain; will see how it works when I get home tomorrow.

function getTypedHistString(tdx) {
	var ret="";
	var typen = htypes[tdx];
	if (tdx<4){
		var tstr="[";
		for (var i=0;i<48;i++) {
			tstr=tstr+HISTORY[typen][i].toFixed(1);
			if (i !=47) {
				tstr+=",";
			} 
		}
		tstr+="]";
		return tstr;
	} else {
		return JSON.stringify(HISTORY[typen]);
	}
}


function processrequest(req, res) {
	try {
		var par=url.parse(req.url,true);
		var q=par.query; 
		var path=par.pathname; 
		path=path.split('/');
		path=path[1];
		switch(path) {
			case "status.json":
			var r='{"lamp":{';
			for (var i=0;i<5;i++) {
				r+='"'+colors.substr(i*4,4)+'":'+STATUS.­LEDs[i].toFixed(2);
				if (i<4) {r+=",";}
			}
			r+='},\n"sensors":{"RH":'+STATUS.RH.toFi­xed(1)+',"Temp":'+STATUS.Temp.toFixed(1)­+',"Pressure":'+STATUS.Pressure.toFixed(­2)+',"Air Quality":'+STATUS.AirQual.toFixed(2)+'},­\n"Light":'+JSON.stringify(STATUS.Light)­+'}';
			res.writeHead(200,head);
			res.write(r);
			res.end();
			break;
			case "history.json":
			res.writeHead(200,head);
			var n=0;
			res.on('drain',function(){
				res.write('"'+htypes[n]+'"');
				res.write(":");
				res.write(getTypedHistString(n++));
				if (n==8) {res.end("}");} else {res.write(',\r\n');}
			});
			res.write("{");
			break;
			case "usermsg.cmd":
			if (query.msg && query.msg.length > 1) {
				usrmsg=query.msg;
				MnuS=10;
				setTimeout("uplcd();",100);
				res.writeHead(200,head);
				res.write("{status:\"ok\",message:\""+qu­ery.msg+"\"}");
			} else {
				res.writeHead(400,head);
				res.write("{status:\"error\",error:\"No message supplied\"}");
			}
			res.end();
			break;
			case "lamp.cmd":
			STATUS.LEDs[0]=q.BLUE===undefined ? 0:E.clip(q.BLUE,0,1);
			STATUS.LEDs[1]=q.COOL===undefined ? 0:E.clip(q.COOL,0,1);
			STATUS.LEDs[2]=q.WARM===undefined ? 0:E.clip(q.WARM,0,1);
			STATUS.LEDs[3]=q.YELL===undefined ? 0:E.clip(q.YELL,0,1);
			STATUS.LEDs[4]=q.RED===undefined ? 0:E.clip(q.RED,0,1);
			setTimeout("upled();",100);
			res.writeHead(200,head);
			res.write("{status:\"ok\",message:\"lamp­ state set\"}");
			res.end();
			break;
			case "code.run": //This is why I don't expose this board to requests from outside the LAN ;-) 
			if (query.code) {
				var x="";
				try {
					x=eval(query.code);
					res.writeHead(200,head);
					res.write("{status:\"ok\",result:\""+x+"­\"}");
				}
				catch (err) {
					res.writeHead(500,head);
					res.write("{status:\"error\",error:\""+e­rr.message+"\"}");
				}
			} else {
				res.writeHead(400,head);
				res.write("{status:\"error\",error:\"No code supplied.\"}");
			}
			res.end();
			break;
			default:
			res.writeHead(404,head);
			res.write("{status:\"error\",error:\"unk­nown command\"}");
			res.end();
		}
	}
	catch (err){
		res.writeHead(500,head);
		res.write("{status:\"error\",error:\""+e­rr.message+"\"}");
		res.end();
	}
}

Works like a charm!

I was thinking about the object.keys() approach - but I don't want to bite myself in the ass if I later add something to HISTORY object, since my code (as you can see) makes assumptions about which index number corresponds to which kind of data, since I have to do float arrays differently from integer arrays.

...line 49 confuses me...

res.write("{");? It should write { for the start of the JSON, then each drain event it writes key : value,\n and finally it'll write } and close the connection.

...I guess it is a left over, because it is sent after this draining loop completed and end has already been issued. I read this out of walking through the code: line 47 writes the closing and end after all sub objects have been written.

No, it is sent before the draining loop fires even though the code comes afterwards. The 'drain' event will only fire once the socket 'drains', which will happen once the { gets written.

...dddddooohhh... of course, it is the opening curly braces for the overall JSON object. Thanks, @dave_irvine to point this out... and it probably has to happen there, because if written before setting up the drain event handler, the drain even would already have passed... :/