Flash storage Module, any js type , web pages, css, js and images

Posted on Sat 5th, March 2016

• #1

Wilberforce
I'm still refining this, however recent activity on gitter has prompted me to publish this now!

https://github.com/wilberforce/EspruinoDocs/tree/master/modules

There are 3 modules used.

The FlashStoreWrite module is used to populate flash with assets that you want access to in your real application. It is a key based storage system, that takes a key, and stores a js object against it. This can then be recovered using the FlashStore module.

The basic usage is:
```
var fs = new(require("FlashStoreWrite"))(0x7c000);
fs.item('example','store me!');
```
The FlashItem object is used for recovering the stored object, which has methods toString() and valueOf();

var str=fs.item('example').toString();

The flashItem also knows about the types of object so treats the differently.

This allows us to store things like web assets such as css and js files, images and have them served by a the http server object (using pipe so that not much memory) is consumed.

We can also store modules in flash, and load them on demand.

1 Attachment
- FlashStore.zip

• #2

Wilberforce

Complex types can be stored, they are automatically converted to JSON strings and back again:

var fs = new(require("FlashStoreWrite"))(0x7c000);

var config={start:4, end:9, data:'fishing'};

fs.item('config',config);

var rfs=new(require("FlashStore"))(0x7c000);
var my_config=rfs.item('config').valueOf();
console.log(my_config);

Flash map 4MB:512/512, manuf 0xe0 chip 0x4016
>echo(0);
{ "start": 4, "end": 9,
  "data": "fishing"
 }

• #3

Wilberforce

web assets can be stored:

fs.item('/favicon.ico').wget('http://www.espruino.com/favicon.ico');
fs.item('/images/logo.png').wget('http://www.espruino.com/images/logo.png');

I'll post a demo tomorrow on how to serve these in a web server, using the FlashItem.pipe method

• #4

allObjects

Excellent! - Very very cool... What about delete? ...and garbage collect? - see memory manager... just dealing with strings... so far cross developed in HTML5 with test bed a while ago... having reference objects so they would not eat up the RAM... and keying was next on my list... are next on list. Unfortunately I'm too busy with other stuff and laying low, Espruino-wise... :( ...the more I enjoy what you present! - I started the work when I came across FRAM/MRAM which does not have the EEPROM/FLASH issues. The memory was serially attached and the memory manager already follows this idea... I know that @Gordon has published something as well that takes the paging and equal/balanced(?) erasing/rewriting into consideration. @DrAzzy, we are not the only one who like to tool with memory!

• #5

Wilberforce

Here is the web server example:

var fs = new(require("FlashStoreWrite"))(0x7c000);

//fs.item('/images/logo.png').delete();

fs.item('/favicon.ico').wget('http://www.espruino.com/favicon.ico');
fs.item('/images/logo.png').wget('http://www.espruino.com/images/logo.png');

function Doc() {
    this.str='';
}

Doc.prototype.write=function(s) {
    this.str+=s;
  this.str+='\n';
};

Doc.prototype.toString=function(){return this.str;};

var document=new Doc();

  
// http://www.accessify.com/tools-and-wizards/developer-tools/html-javascript-convertor/
document.write("<html lang=\"en\">");
document.write("  <head>");
document.write("    <meta charset=\"utf-8\">");
document.write("    <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">");
document.write("    <meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">");
document.write("    <link rel=\"stylesheet\" href=\"http:\/\/maxcdn.bootstrapcdn.com\/bootstrap\/3.3.5\/css\/bootstrap.min.css\">");
document.write("  <\/head>");
document.write("  <body>");
document.write("    <div class=\"well well-lgt\">  ");
document.write("      <div class=\"panel-heading\">");
document.write("        <h3>Web Flash Server<\/h3>");
document.write("      <\/div>");
document.write("      <button class=\"btn btn-warning\" id=\"send\" onclick=\"action(this);\">Send to Espruino<\/button>");
document.write("      <ul id=\"list\" class=\"list-group\">");
document.write("        <li class=\"list-group-item\"><\/li>");
document.write("        <li class=\"list-group-item\"><img src=\"\/images\/logo.png\"><\/li>");
document.write("      <\/ul>");
document.write("    <\/div>");
document.write("    <script src=\"\/js\/app.js\"><\/script>");
document.write("  <\/body>");
document.write("<\/html>");
  
console.log( document.toString() );

fs.item('/',document.toString(),'text/html');

document.str='';

document.write("var count=1;");
document.write("function action(btn) {");
document.write("  btn.innerHTML = 'Send ' + count;");
document.write("  send_json(count);");
document.write("    count = count + 1;");
document.write("}");
document.write("function add(text) {");
document.write("  var ul = document.getElementById(\"list\");");
document.write("  var li = document.createElement(\"li\");");
document.write("  li.className = \"list-group-item\";");
document.write("  li.appendChild(document.createTextNode(text));");
document.write("  ul.appendChild(li);");
document.write("}");
document.write("function send_json(count) {");
document.write("  count = count + 1;");
document.write("  var xhttp = new XMLHttpRequest();");
document.write("  xhttp.onreadystatechange = function () {");
document.write("    if (xhttp.readyState == 4 && xhttp.status == 200) {");
document.write("      add(xhttp.responseText);");
document.write("    }");
document.write("  };");
document.write("  xhttp.open(\"GET\", \"\/json?count=\" + count, true);");
document.write("  xhttp.send();");
document.write("};");

fs.item('/js/app.js',document.toString(),'application/javascript');

print( process.memory());

delete document;

print( process.memory());

var fs = new(require("FlashStore"))(0x7c000);

require("http").createServer(function (request, response) {
    var u = url.parse(request.url, true);
    var q = fs.find(u.pathname);
    if (q) {
      console.log({match:u.pathname});
      q.pipe(response);
    } else {
      if ( u.pathname === '/json' ) {
         // response.writeHead(200);
        response.end(Date.now().toString());
        return;
      }
      console.log({   q : u.query,   p : u.pathname  });
      response.writeHead(404);
      response.end("404: Not found");
    }

  }).listen(80);
print(process.memory());

The webserver - using the FlashStore object to retrieve, it assumes you have a saved wifi.save() that is already connected:

var fs = new(require("FlashStore"))(0x7c000);

require("http").createServer(function (request, response) {
    var u = url.parse(request.url, true);
    var q = fs.find(u.pathname);
    if (q) {
      console.log({match:u.pathname});
      q.pipe(response);
    } else {
      if ( u.pathname === '/json' ) {
         // response.writeHead(200);
        response.end(Date.now().toString());
        return;
      }
      console.log({   q : u.query,   p : u.pathname  });
      response.writeHead(404);
      response.end("404: Not found");
    }

  }).listen(80);
print(process.memory());

The ico and image assets are saved, then the index.html root document and /js/app.js

The webserver code searches the store, and if the content is matched it s served from the flash, using the .pipe method.

When the button is clicked, a /json method is called, and this returning the current time back to the browser...

2 Attachments

• #6

Wilberforce
@allObjects

Excellent! - Very very cool... What about delete? ...and garbage collect?

At this stage I've decided to keep it very simple.
The likes of the ESP12 with 4Mb means that we have oodles of space.

This is the structure used to store items:
```
={ "addr": 507904,
  "flash": function () { [native code] },
  "page_size": 4096,
  "items": {
    "_root": { "page": 0, "next_page": 14, "length": 743,
      "mime": "application/json"
     },
    "DS18B20": { "page": 1, "length": 1307,
      "mime": "application/javascript; charset=utf-8"
     },
    "/": { "page": 2, "length": 751,
      "mime": "text/html"
     },
    "/js/app.js": { "page": 3, "length": 627,
      "mime": "application/javascript"
     },
    "/favicon.ico": { "page": 4, "length": 1150,
      "mime": "image/x-icon"
     },
    "clock": { "page": 7, "length": 413,
      "mime": "application/javascript; charset=utf-8"
     },
    "Clock": { "page": 8, "length": 413,
      "mime": "application/javascript; charset=utf-8"
     },
    "DS18S20": { "page": 9, "length": 2,
      "mime": "application/json"
     },
    "/images/logo.png": { "page": 10, "length": 4726,
      "mime": "image/png"
     },
    "config": { "page": 12, "length": 36,
      "mime": "application/json"
     },
    "/js.app.js": { "page": 13, "length": 624,
      "mime": "application/javascript"
     }
   }
 }
```
The magic is in the _root object, this is stored in JSON.stringify format in the first page of the flash, and is restored in the FileStorage constructor.

The page size is 4K - currently each object takes up at least on page. This seems wasteful, however the trade off is speed.
An existing object can get appended too - as long as it fits in the space allocated to it.

I did not think it was worth the extra trouble of storing intra page and managing indexes.

Given that you can store a complex js structure, you could pack everything you wanted in a structure and store that.

What about delete?

The fs.item('my_key').delete(); method removes an item. At this stage it is removed from the list of objects. The page is wasted, as the only management at this stage is the items._root.next_page.

A future enhancement could be to have an array of free pages, and this could be checked and used for a new write.

...and garbage collect?

In my case, I wanted to store web pages, and modules. Both of these are easy to re-populate, so rather than garbage collect and move pages around (with limited available memory), it is more straightforward to call fs.erase() and re-populate everything again.

I wanted the FileStore module to be as small as possible, so that the end application has as much memory without bloating the module with methods. This is why the FileStoreWrite inherits it's methods from FileStore and adds the writing methods.
• #7

Wilberforce
Compressed objects.

The item.wget() allows web assets to be fetched and stored:
```
fs.item('/css/bootstrap.min.css').wget('http://www.espruino.com/css/bootstrap.min.css',{compress:'gzip'});
```
With the compress options, the headers are modfied and the content is compressed, and this is what is sorted in flash.

When the webserver .pipe method retrieves the object, it is sent back compressed to the browser and the browser unzips on the fly. This means that we don't need gzip/ungzip libraries for espurino, but can store large libraries for the browser front end efficiently with no overhead.

It's not particularly quick - just under 9 secs to load.However it means a web server can be set up in AP mode, and not be connected to the internet, and have bootstrap.css!
• #8

Wilberforce
We can also cache Modules in Eprom, and load on demand:
```
var fs = new(require("FlashStoreWrite"))(0x7c000);
// Save module source in EEPROM
fs.item('clock').wget('http://www.espruino.com/modules/clock.min.js');
```
```
var fs = new(require("FlashStore"))(0x7c000);
// Retrieve module source, add to module cache and then use..
require( fs.item('clock').module() );
var Clock = require("clock").Clock;
var clk=new Clock(2014,4,15,23,45,0,0);
```
require( fs.item('clock').module() );
={ "Clock": function () { ... } }
var Clock = require("clock").Clock;
=function () { ... }
var clk=new Clock(2014,4,15,23,45,0,0);
={ "lastTime": 1457230986.79778003692, "date": { "ms": 1400197500000 } }

This means you can use require on the left hand side of the IDE on a pre-cached module, and then use that module.

• #9

Aleksandrs

I have litle bug i don`t understand ...

function startDevice(){
  require( fs.item('Generalscript').module() );
  var Generalscript = require("Generalscript").Generalscript;
  var gs = new Generalscript();
  gs.getVer();
}

Uncaught Error: Both arguments to addCached must be strings
 at line 1 col 41
Modules.addCached(this.k,this.toString()),this.k;
                                        ^
in function "module" called from line 1 col 42
require( fs.item('Generalscript').module() );
                                         ^
in function "startDevice" called from line 23 col 24
           startDevice();
                       ^
in function called from system

• #10

Wilberforce

You need to use the module in 2 stages.

The first is to use the write module to load your module into flash. You can add it as a string, or use the wget method to load it.
Once it is in flash you don't need this loader script.

Then use the flash item method to pull the string from flash and add to the module cache.

In your example you would not use the 2nd require, as the first would have already added it.

The error you are getting is due to the module not getting added to then flash with the write method first, so the retrieved string is empty so the module.add fails

• #11

MaBe

@Wilberforce - very cool - thanks for sharing !

downoaded the zip file and stored the js files in the module folder

running a ESP8266 with 1v89

tried this

var fs = new(require("FlashStoreWrite"))(0x7c000);
var config={start:4, end:9, data:'fishing'};
fs.item('config',config);

and got

>WARNING: Module "FlashStore" not found
WARNING: Module "FlashItem" not found
Uncaught Error: Constructor should be a function, but is Object
 at line 1 col 83
...ire('FlashItem');FlashStore.prototype._store=function(e,b,c,...
                              ^
 at line 1 col 2885
...(require('FlashStoreWrite'))(507904);var config={start:4,end...
                              ^
=undefined

Inside of FlashStoreWrite there are two require statements that seems to cause the Error.

loading them separately works fine

var FlashStore=require("FlashStore");
var FlashItem=require("FlashItem");

no Errors and no Warnings...

@Wilberforce any idea why this is not working on my side ?

• #12

Wilberforce in reply to @MaBe

@MaBe
I just tried and I'm getting the same as you - for some reason the FlashStore=require("FlashStore") module is not getting found.

I'll look into it.
• #13

MaBe
Hi @Gordon,

are there some changes made in WebIDE that the snippet shown in #11 fails ?

I downloaded FlashStore.zip and moved files to the modules folder.

console.log contains WebIDE CONSOLE information of last try.

1 Attachment
- console.log
• #14

Gordon

Not that I know of. What's the error? Log all looks ok
• #15

MaBe

good to know

#11 shows the error
• #16

EthraZa

FlashStore has something to do with FlashEEPROM?
• #17

MaBe

Hi @EthtaZa,

this three modules form @Wilberforce are based on Flash Library. It is using free flash areas to write and read simple and complex types of data.
• #18

Gordon
Just to update this, there's now:
- The Storage Module built in to Espruino itself which allows you to save simple 'files' - and load them in a memoryArea-style way that doesn't use up RAM.
- Save to flash and modules as functions in the Web IDE, which automatically save your code direct to flash
These work on all platforms.

Post a reply
- Bold
- Italics
- Link
- Image
- List
- Quote
- code
- Preview
Formatting Help

Don't worry about formatting, just type in the text and we'll take care of making sense of it. We will auto-convert links, and if you put asterisks around words we will make them bold.

Tips:

Create headers by underlining text with ==== or ----

To *italicise* text put one asterisk each side of the word

To **bold** text put two asterisks each side of the word

Embed images by entering:
![](https://www.google.co.uk/images/srpr/logo4w.png)
That's the hard one: exclamation, square brackets and then the URL to the image in brackets.

* Create lists by starting lines with asterisks

1. Create numbered lists by starting lines with a number and a dot

> Quote text by starting lines with >

Mention another user by @username

For syntax highlighting, surround the code block with three backticks:

```
Your code goes here
```
Just like Github, a blank line must precede a code block.

If you upload more than 5 files we will display all attachments as thumbnails.

For a full reference visit the Markdown syntax.

Flash storage Module, any js type , web pages, css, js and images

About