How to save "big" data.

So I'm building this throwing app and I have option, that you can record every throws acceleration data, if you want. The record is 2 s long and every axis has a 200 points.
This is not yet a problem, the problem is how to save this data. In one game you can have 20-30 throws, so I can not store all the throws in a array. If I just push all throw data to a array, I will run out of memory. So that's not an option.

Now I'm trying to save every throw to json file and at the end of the game, the throw app will collect all the json files and write that to the one csv file.
But I will have the same problem: if I read all the json files, I need to temporarily hold the data somewhere, before I write the csv file. So thats not good option also.

Now I'm trying to out this "offset" option on the write file, maybe that is my solution. I will past my code in here. I know it doesn't work yet, because I misunderstood "offset" option in write command.
But before I continue, is this the right ( only?) solution or is there a better way?

var HZ = 100;
var SAMPLES = 2 * HZ;
var SCALE = 2000;
var accelx = new Int16Array(SAMPLES);
var accely = new Int16Array(SAMPLES);
var accelz = new Int16Array(SAMPLES);
var timestep = new Int16Array(SAMPLES);
var accelId = 0;
var throws_acc = [];



function SaveThrowJson(json_n){
  require("Storage").writeJSON("KyRec_" + (json_n + 1) + ".json", [accelx, accely, accelz, timestep]);
}

function RemoveOldJson(json_files){
  //let json_files = require("Storage").list(/^KyRec_.*.json$/);
  json_files.forEach(f_json => {
    require("Storage").erase(f_json);
  });
}

function SaveFile(){
  let storage = require("Storage");
  let csv_files_N = storage.list(/^kyAc_.*.csv$/).length;
  let json_files = storage.list(/^KyRec_.*.json$/);
  let csv = "";
  let date = new Date();
  let offset = 0;
  let offset_old = 0;
  let fn = date.getHours() + "-" + date.getMinutes() + "_" + date.getDate() + "-" + (date.getMonth() +1) + "-" + date.getFullYear().toString().substr(-2);
  fn = "kyAc_" + fn + "_" + (csv_files_N + 1) + ".csv";
  let json_data = "";
  //print(throws_acc);
  json_files.forEach(f_json => {
    json_data = storage.readJSON(f_json);
    csv = "";
    for (var i = 0; i < json_data[0].length; i++){
      csv += `${json_data[3][i]},${json_data[0][i]/SCALE},${json_data[1][i]/SCALE},${json_data[2][i]/SCALE}\n`;
      offset++;
    }
    csv += `${0},${0},${0},${0}\n`;
    offset++;
    if (offset_old == 0)
      storage.write(fn, csv);
    else
      storage.write(fn, csv, offset_old);
    print(offset_old);
    offset_old = offset;
  });
  //if (csv != "") storage.write(fn, csv);
  //    storage.write(file.name,txt,file.offset);
  //  file.offset += l;
  RemoveOldJson(json_files);
}


function recordStart() {"ram"
  console.log("start settings");
	accelId = 0;
	Bangle.accelWr(0x18, 0b01110100);
	Bangle.accelWr(0x1B, 0x03 | 0x40);
	Bangle.accelWr(0x18, 0b11110100);
	Bangle.setPollInterval(10);
  startRecord();
}

function recordStop() {"ram"
  console.log("end settings");
  Bangle.setPollInterval(80);
	Bangle.accelWr(0x18, 0b01101100);
	Bangle.accelWr(0x1B, 0x0);
	Bangle.accelWr(0x18, 0b11101100);
  if (settings.save_record) SaveFile();
  //print(accelx);
  //print(throws_acc);
}

function startRecord() {
	var stopped = false;
	var round_n = 1;
	g.clear(1);

	var start_time = getTime();
	var Throws_n = 0;
	var maxMag = 0;
	var aX = 0;
	var t = 0;
  var t_old = 0;
  var throw_time_limit = 0;
  var write_record = 0;
  var wrtie_time = getTime();
  let date = new Date();
  function add_round(){
    round_n++;
    total_throws = total_throws + Throws_n;
    Throws_n = 0;
    if (settings.save_record){
      let date = new Date();
      if (settings.save_record) SaveFile();
    }
  }
	function accelHandler(accel) {
		t = ~~(getTime() - start_time);
    aX = Math.abs(accel.x * 2);
    //console.log(aX);
    //print(getTime() - start_time);
    if (t != t_old){
      //print((t - t_old));
      t_old = t;
      render_layout();
    }
    //print(t -throw_time_limit, t, throw_time_limit);
		if (aX > settings.throw_g_lim && (t -throw_time_limit) > 3 ) {
      //console.log(aX);
      throw_time_limit = t;
			Throws_n++;
      render_layout();
      write_record = 1;
		}
    if (settings.save_record){
      if (write_record == 1 && (t -throw_time_limit) > 1) {
        SaveThrowJson(Throws_n);
        //json_n++;
        //print(Throws_n);
        write_record = 0;
        wrtie_time = getTime();
        accelx.fill(0);
        accely.fill(0);
        accelz.fill(0);
        timestep.fill(0);
      }
      accelx[accelId] = accel.x * SCALE * 2;
      accely[accelId] = accel.y * SCALE * 2;
      accelz[accelId] = accel.z * SCALE * 2;
      timestep[accelId] = (getTime() - wrtie_time)*1000;
      accelId++;
      //print(accelId);
      if (accelId == SAMPLES) accelId = 0;
    }
	}

	Bangle.on("accel", accelHandler);
}

If you're writing binary data into a file with an offset, you need to tell Espruino the size of the file you want first so it can allocate space - see https://www.espruino.com/Reference#l_Storage_write

What you were doing storing individual files for acceleration data looks good - I guess it's just the hit of loading the data back.

One thing you might want to consider is to encode the arrays as base64 and put that in JSON with btoa(accelx.buffer) and then new Int16Array(E.toArrayBuffer(atob(str))).

The reason is that Espruino stores arrays way more efficiently in IntXArrays, but when you write to JSON it has to convert those arrays into normal JS arrays, which use a lot more memory. If you write as base64 it avoids all that and will load everything far more efficiently.

True, I can save the temporary throw as base64 format.

The problem is that, if I want to use offset option, I have to know how big the csv file will be. I understood that if I don't define the file size in the beginning, then I can't use offset option, right? I think, I need to estimate its size and make it big enough.

I understood that if I don't define the file size in the beginning, then I can't use offset option, right?

Correct, yes. However if you were to write the raw buffer from the Uint16Array I guess you know its size? 3 buffers of 2*SAMPLES bytes?

But if you're planning on writing CSV maybe check out StorageFile - that provides a file that you can keep appending to (but it's text only, not binary): https://www.espruino.com/Reference#StorageFile

The problem with writing CSV is you have to be able to load it back in - if you do str.split(",") that's going to create a huge array of numbers anyway, which is pretty much the same as using JSON.parse

Thanks, the StrageFile solved that problem. Now back to testing.

Now I have made first throw app. Can you guys give me a feedback, how to make it smoother or faster or better? Here is link to my repo https://github.com/pinq-/BangleApps/blob/master/apps/Kyykka_throw/Kyykka.app.js