diff --git a/ChangeLog b/ChangeLog index 9d7a4a9b1..f6ce6dac1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -96,6 +96,7 @@ Pixl.js: Fix 30s pause when closing sockets on WIZnet W5100 (fix #1306) Remove HASH/hashlib from all builds as it was confusingly in some and not others. Now use 'crypto' require('crypto').SHA1 is now JS in Espruino Original to cut down on the flash required + Added 'heatshrink' library to expose built-in heatshrink compression to users 1v99 : Increase jslMatch error buffer size to handle "UNFINISHED TEMPLATE LITERAL" string (#1426) nRF5x: Make FlashWrite cope with flash writes > 4k diff --git a/Makefile b/Makefile index 70c2f59b1..ed9db3995 100755 --- a/Makefile +++ b/Makefile @@ -308,7 +308,8 @@ SOURCES += \ libs/compression/heatshrink/heatshrink_encoder.c \ libs/compression/heatshrink/heatshrink_decoder.c \ libs/compression/compress_heatshrink.c - +WRAPPERSOURCES += \ +libs/compression/jswrap_heatshrink.c endif ifndef BOOTLOADER # ------------------------------------------------------------------------------ DON'T USE IN BOOTLOADER diff --git a/libs/compression/compress_heatshrink.c b/libs/compression/compress_heatshrink.c index d1aa55438..d5957a770 100644 --- a/libs/compression/compress_heatshrink.c +++ b/libs/compression/compress_heatshrink.c @@ -21,8 +21,8 @@ #define BUFFERSIZE 128 -/** gets data from array, writes to callback */ -void heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata) { +/** gets data from array, writes to callback if nonzero. Returns total length. */ +uint32_t heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata) { heatshrink_encoder hse; uint8_t outBuf[BUFFERSIZE]; heatshrink_encoder_reset(&hse); @@ -43,8 +43,9 @@ void heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(uns do { pres = heatshrink_encoder_poll(&hse, outBuf, sizeof(outBuf), &count); assert(pres >= 0); - for (i=0;i= 0); + if (data) memcpy(&data[polled], outBuf, count); polled += count; } while (pres == HSER_POLL_MORE); assert(pres == HSER_POLL_EMPTY); @@ -98,4 +102,6 @@ void heatshrink_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsi heatshrink_decoder_finish(&hsd); } } + return (uint32_t)polled; } + diff --git a/libs/compression/compress_heatshrink.h b/libs/compression/compress_heatshrink.h index aaadb4804..537dc38b2 100644 --- a/libs/compression/compress_heatshrink.h +++ b/libs/compression/compress_heatshrink.h @@ -12,8 +12,8 @@ * ---------------------------------------------------------------------------- */ -/** gets data from array, writes to callback */ -void heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata); +/** gets data from array, writes to callback if nonzero. Returns total length. */ +uint32_t heatshrink_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata); -/** gets data from callback, writes it into array */ -void heatshrink_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data); +/** gets data from callback, writes it into array if nonzero. Returns total length */ +uint32_t heatshrink_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data); diff --git a/libs/compression/compress_rle.c b/libs/compression/compress_rle.c index 2bc64e9f8..085e8ef42 100644 --- a/libs/compression/compress_rle.c +++ b/libs/compression/compress_rle.c @@ -14,13 +14,15 @@ #include "compress_rle.h" -// gets data from array, writes to callback -void rle_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata) { +/** gets data from array, writes to callback if nonzero. Returns total length. */ +uint32_t rle_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata) { + uint32_t outputLen = 0; int lastCh = -1; // not a valid char while (dataLen) { unsigned char ch = *(data++); dataLen--; - callback(ch, cbdata); + outputLen++; + if (callback) callback(ch, cbdata); if (ch==lastCh) { int cnt = 0; while (dataLen && lastCh==*data && cnt<255) { @@ -28,25 +30,31 @@ void rle_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned c dataLen--; cnt++; } - callback((unsigned char)cnt, cbdata); + outputLen++; + if (callback) callback((unsigned char)cnt, cbdata); } lastCh = ch; } + return outputLen; } -// gets data from callback, writes it into array -void rle_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data) { +/** gets data from callback, writes it into array if nonzero. Returns total length */ +uint32_t rle_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data) { + uint32_t outputLen = 0; int lastCh = -256; // not a valid char while (true) { int ch = callback(cbdata); - if (ch<0) return; - *(data++) = (unsigned char)ch; + if (ch<0) return outputLen; + if (data) data[outputLen] = (unsigned char)ch; + outputLen++; if (ch==lastCh) { int cnt = callback(cbdata); while (cnt-->0) { - *(data++) = (unsigned char)ch; + if (data) data[outputLen] = (unsigned char)ch; + outputLen++; } } lastCh = ch; } + return outputLen; } diff --git a/libs/compression/compress_rle.h b/libs/compression/compress_rle.h index 65838a2d6..0cf092627 100644 --- a/libs/compression/compress_rle.h +++ b/libs/compression/compress_rle.h @@ -14,8 +14,8 @@ #include "jsutils.h" -/** gets data from array, writes to callback */ -void rle_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata); +/** gets data from array, writes to callback if nonzero. Returns total length. */ +uint32_t rle_encode(unsigned char *data, size_t dataLen, void (*callback)(unsigned char ch, uint32_t *cbdata), uint32_t *cbdata); -/** gets data from callback, writes it into array */ -void rle_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data); +/** gets data from callback, writes it into array if nonzero. Returns total length */ +uint32_t rle_decode(int (*callback)(uint32_t *cbdata), uint32_t *cbdata, unsigned char *data); diff --git a/libs/crypto/jswrap_crypto.c b/libs/crypto/jswrap_crypto.c index 484e99b67..5a6b94412 100644 --- a/libs/crypto/jswrap_crypto.c +++ b/libs/crypto/jswrap_crypto.c @@ -21,7 +21,9 @@ #ifdef USE_AES #include "mbedtls/include/mbedtls/aes.h" #endif +#ifndef USE_SHA1_JS #include "mbedtls/include/mbedtls/sha1.h" +#endif #ifdef USE_SHA256 #include "mbedtls/include/mbedtls/sha256.h" #endif @@ -142,7 +144,7 @@ JsVar *jswrap_crypto_SHAx(JsVar *message, int shaNum) { #ifdef USE_SHA1_JS if (shaNum==1) { // (c) 2016 Rhys Williams, @jumjum. https://github.com/espruino/EspruinoDocs/blob/master/modules/crypto.js - return jspExecuteJSFunction("(function(b){function n(a){for(d=3;0<=d;d--)g.push(a>>8*d&255)}var d,a;b=E.toString(b)+'\\x80';var v=new Int32Array([1518500249,1859775393,2400959708,3395469782]);var k=Math.ceil((b.length/4+2)/16);var g=Array(k);b=E.toUint8Array(b);for(d=0;da;a++){var c=f+(a<<2);e[a]=b[c]<<24|b[c+1]<<16|b[c+2]<<8|b[c+3]}g[d]=e}g[k-1][14]=8*(b.length-1)/Math.pow(2,32);g[k-1][14]=Math.floor(g[k-1][14]);g[k-1][15]=8*(b.length-1)&4294967295;b=1732584193;var p=4023233417;var q=2562383102;var r=271733878;var t=3285377520;var l=new Int32Array(80);for(d=0;da;a++)l[a]=g[d][a];for(a=16;80>a;a++)f=l[a-3]^l[a-8]^l[a-14]^l[a-16],l[a]=f<<1|f>>>31;f=b;c=p;e=q;var h=r;var u=t;for(a=0;80>a;a++){var m=Math.floor(a/20);var w=f<<5|f>>>27;var x=0===m?c&e^~c&h:1===m?c^e^h:2===m?c&e^c&h^e&h:c^e^h;m=w+x+u+v[m]+l[a]&4294967295;u=h;h=e;e=c<<30|c>>>2;c=f;f=m}b=b+f&4294967295;p=p+c&4294967295;q=q+e&4294967295;r=r+h&4294967295;t=t+u&4294967295}g=[];n(b);n(p);n(q);n(r);n(t);return E.toUint8Array(g)})",0,1,&message); + return jspExecuteJSFunction("(function(b){function n(a){for(d=3;0<=d;d--)g.push(a>>8*d&255)}var d,a;b=E.toString(b)+'\\x80';var v=new Int32Array([1518500249,1859775393,2400959708,3395469782]);var k=Math.ceil((b.length/4+2)/16);var g=Array(k);b=E.toUint8Array(b);for(d=0;da;a++){var c=f+(a<<2);e[a]=b[c]<<24|b[c+1]<<16|b[c+2]<<8|b[c+3]}g[d]=e}g[k-1][14]=8*(b.length-1)/Math.pow(2,32);g[k-1][14]=Math.floor(g[k-1][14]);g[k-1][15]=8*(b.length-1)&4294967295;b=1732584193;var p=4023233417;var q=2562383102;var r=271733878;var t=3285377520;var l=new Int32Array(80);for(d=0;da;a++)l[a]=g[d][a];for(a=16;80>a;a++)f=l[a-3]^l[a-8]^l[a-14]^l[a-16],l[a]=f<<1|f>>>31;f=b;c=p;e=q;var h=r;var u=t;for(a=0;80>a;a++){var m=Math.floor(a/20);var w=f<<5|f>>>27;var x=0===m?c&e^~c&h:1===m?c^e^h:2===m?c&e^c&h^e&h:c^e^h;m=w+x+u+v[m]+l[a]&4294967295;u=h;h=e;e=c<<30|c>>>2;c=f;f=m}b=b+f&4294967295;p=p+c&4294967295;q=q+e&4294967295;r=r+h&4294967295;t=t+u&4294967295}g=[];n(b);n(p);n(q);n(r);n(t);return E.toUint8Array(g).buffer})",0,1,&message); } #endif diff --git a/src/jsflash.c b/src/jsflash.c index 677c83164..41e065b9f 100644 --- a/src/jsflash.c +++ b/src/jsflash.c @@ -592,7 +592,7 @@ JsVar *jsfListFiles() { // Get a hash of the current Git commit, so new builds won't load saved code static uint32_t getBuildHash() { #ifdef GIT_COMMIT - const char *s = STRINGIFY(GIT_COMMIT); + const unsigned char *s = (unsigned char*)STRINGIFY(GIT_COMMIT); uint32_t hash = 0; while (*s) hash = (hash<<1) ^ *(s++); @@ -601,11 +601,6 @@ static uint32_t getBuildHash() { return 0; #endif } -// cbdata = uint32_t -void jsfSaveToFlash_countcb(unsigned char ch, uint32_t *cbdata) { - NOT_USED(ch); - cbdata[0]++; -} typedef struct { uint32_t address; @@ -654,9 +649,8 @@ void jsfSaveToFlash() { // Try and compact, just to ensure we get the maximum amount saved jsfCompact(); jsiConsolePrint("Calculating Size...\n"); - // Work out how much data this'll take - uint32_t compressedSize = 4; /* 4 bytes for build hash */ - COMPRESS(varPtr, varSize, jsfSaveToFlash_countcb, &compressedSize); + // Work out how much data this'll take, plus 4 bytes for build hash + uint32_t compressedSize = 4 + COMPRESS(varPtr, varSize, NULL, NULL); // How much data do we have? uint32_t savedCodeAddr = jsfCreateFile(jsfNameFromString(SAVED_CODE_VARIMAGE), compressedSize, JSFF_COMPRESSED, JSF_START_ADDRESS, 0); if (!savedCodeAddr) { @@ -687,7 +681,7 @@ void jsfSaveToFlash() { uint32_t hash = getBuildHash(); int i; for (i=0;i<4;i++) - jsfSaveToFlash_writecb(((char*)&hash)[i], (uint32_t*)&cbData); + jsfSaveToFlash_writecb(((unsigned char*)&hash)[i], (uint32_t*)&cbData); // write compressed data COMPRESS(varPtr, varSize, jsfSaveToFlash_writecb, (uint32_t*)&cbData); jsfSaveToFlash_finish(&cbData); @@ -713,7 +707,7 @@ void jsfLoadStateFromFlash() { uint32_t hash; int i; for (i=0;i<4;i++) - ((char*)&hash)[i] = jsfLoadFromFlash_readcb((uint32_t*)&cbData); + ((char*)&hash)[i] = (char)jsfLoadFromFlash_readcb((uint32_t*)&cbData); if (hash != getBuildHash()) { jsiConsolePrintf("Not loading saved code from different Espruino firmware.\n"); return; diff --git a/tests/test_heatshrink.js b/tests/test_heatshrink.js new file mode 100644 index 000000000..503789837 --- /dev/null +++ b/tests/test_heatshrink.js @@ -0,0 +1,8 @@ + + +var source = "HelloHelloHelloHelloWorld"; +var compr = require("heatshrink").compress(source) +var decompr = E.toString(require("heatshrink").decompress(compr)); + +result = decompr == source; +