what is the recommended method to transfer (even large) data files onto a device?

Good morning!

I'm currently experimenting with large files of data.

What is the recommended way to transfer such (potentially large) files (even binary ones) onto a connected device?

I've already been able to transfer large binary files from the IDE to my Bangle.js by means of an (automatically created) script containing lots of Storage.write operations which I copied into the terminal(!) area of the IDE.

But I guess, there must be a much better approach?

You can use btoa to convert binary data to base64 string. And then atob to convert the base64 string back to binary. Depending on memory available on the device, you can just include the binary in a file.
For example the CCS811 firmware updater contains two originally 5K binaries in base64 format. The two binary, and the firmware updater code fits in ESP8266 or Pixl, so you can simply dump some binary in one source file.
Or if it's too big, (I guess, haven't tested, but should work) you can move the binaries to separate files, and require them just when you need it. And the IDE should just upload all the files and save to flash if you choose the "Save to flash" option.

plus heatshrink like used for images.

require("heatshrink").decompress()
require("heatshrink").compress()

Good morning!

Thanks for the hints, but I was looking for a better approach than posting JavaScript commands on the IDE, rather than

• encoding/decoding (like atob/btoa) or
  
• heatshrink, which I cannot use as this runs on the watch only (and I would need a counterpart on the PC - and, yes, I know the C source is available, but I have to use JavaScript only, not even WebASM)
  
  

What I found in the meantime is the puck.js library which Gordon also seems to use for his App Store web site.

Here is what I found out this morning.

The rest should just be plain (albeit resource-savvy) JavaScript programming

Oh, the File converter does* just that! Creates a heatshrink compressed data you can paste.

*: There is a bug / typo, so for now (already sent a PR) go there, open the console, and past & run this:

  function fileLoaded() {
    document.getElementById("result").value = "";
    document.getElementById("result").style.­display = "none";
    document.getElementById("size").innerTex­t = "Please Choose a file first";
    if (!bytes) return;

    var fileTypeSelect = document.getElementById("fileType");
    var fileType = fileTypeSelect.options[fileTypeSelect.se­lectedIndex].value;

    if (bytes.length>(20*1024)) {
      document.getElementById("size").innerTex­t = "File too long - must be less than 20kB";
      return;
    }

    var dqStr = "";
    var tmpStr = "";
    var lastCh = 0;
    for (var i=0;i<bytes.length;i++) {
      var ch = bytes[i];
      // templated string
      if (ch==92) tmpStr += "\\\\"; // escaping slash
      else if (ch==96) tmpStr += "\\\`"; // template quote
      else if (lastCh==36 && ch==126) tmpStr += "\\{" // ${
      else tmpStr += String.fromCharCode(ch);
      // double-quoted string
      if (ch==34) dqStr += "\\\"";
      else if (ch==9) dqStr += "\\t";
      else if (ch==10) dqStr += "\\n";
      else if (ch==13) dqStr += "\\r";
      else if (ch==92) dqStr += "\\\\";
      else if (ch>=32 && ch<127)
        dqStr += String.fromCharCode(ch);
      else { // hex code
        if (ch<64 && (i+1>=bytes.length || (bytes[i+1]<48/*0*/ || bytes[i+1]>55/*7*/)))
          dqStr += "\\"+ch.toString(8/*octal*/); // quick compactness hack
        else
          dqStr += "\\x"+(ch+256).toString(16).substr(-2); // hex
      }
      lastCh = ch;
    }

    var finalStr = "";
    switch (fileType) {
      case "b64" :
        finalStr = 'atob("'+btoa(String.fromCharCode.apply(­null, bytes))+'")';
        break;
      case "b64c" :
        finalStr = 'E.toString(require("heatshrink").decomp­ress(atob("'+btoa(String.fromCharCode.ap­ply(null, heatshrink.compress(bytes)))+'")))';
        break;
      case "b64b" :
        finalStr = 'var data = new Uint8Array('+bytes.length+');\r\n';
        var BSIZE = 128;
        for (var i=0;i<bytes.length;i+=BSIZE) {
          let d = btoa(String.fromCharCode.apply(null, bytes.slice(i,i+BSIZE)));
          finalStr += 'data.set(atob("'+d+'"),'+i+');\r\n';
        }
        break;
        case "b64f" :
          finalStr = 'var addr=....; // '+bytes.length+' bytes\n';
          var BSIZE = 256;
          for (var i=0;i<bytes.length;i+=BSIZE) {
            let d = btoa(String.fromCharCode.apply(null, bytes.slice(i,i+BSIZE)));
            finalStr += 'require("Flash").write(atob("'+d+'"), addr+'+i+');\r\n';
          }
          break;
      case "quoted" :
        finalStr = '"'+dqStr+'"';
        break;
      case "templated" :
        finalStr = '"'+tmpStr+'"';
        break;
      default: throw new Error("Unknown type!");
    }
    document.getElementById("size").innerTex­t = finalStr.length+" Characters";
    document.getElementById("result").style.­display = "";
    document.getElementById("result").value = finalStr;
  }

Thanks! I'll pull the changes in.

The plan is to have a 'file manager' inside the IDE to allow upload/download and maybe image conversion too - but unfortunately last week was spent just firefighting 1000+ emails/forum posts :)

Hopefully it'll come soon though