Unverified Commit 172fb276 authored by Terry Ellison's avatar Terry Ellison Committed by GitHub
Browse files

Add compression to LFS images (#2448)

* Merge of LFS compress, optimize against current dev
* Fixes to LFS compress patch
parent 0e103a39
.gdb_history
sdk/ sdk/
cache/ cache/
.ccache/ .ccache/
local/ local/
luac.cross
user_config.h user_config.h
server-ca.crt server-ca.crt
luac.cross
uz_unzip
uz_zip
#ignore Eclipse project files #ignore Eclipse project files
.cproject .cproject
......
...@@ -45,6 +45,7 @@ SUBDIRS= \ ...@@ -45,6 +45,7 @@ SUBDIRS= \
fatfs \ fatfs \
esp-gdbstub \ esp-gdbstub \
pm \ pm \
uzlib \
$(OPT_SEL_MKTARGETS) $(OPT_SEL_MKTARGETS)
endif # } PDIR endif # } PDIR
...@@ -76,6 +77,8 @@ COMPONENTS_eagle.app.v6 = \ ...@@ -76,6 +77,8 @@ COMPONENTS_eagle.app.v6 = \
net/libnodemcu_net.a \ net/libnodemcu_net.a \
mbedtls/libmbedtls.a \ mbedtls/libmbedtls.a \
modules/libmodules.a \ modules/libmodules.a \
smart/smart.a \
uzlib/libuzlib.a \
$(OPT_SEL_COMPONENTS) $(OPT_SEL_COMPONENTS)
......
...@@ -46,6 +46,7 @@ INCLUDES += -I ../spiffs ...@@ -46,6 +46,7 @@ INCLUDES += -I ../spiffs
INCLUDES += -I ../libc INCLUDES += -I ../libc
INCLUDES += -I ../modules INCLUDES += -I ../modules
INCLUDES += -I ../platform INCLUDES += -I ../platform
INCLUDES += -I ../uzlib
PDIR := ../$(PDIR) PDIR := ../$(PDIR)
sinclude $(PDIR)Makefile sinclude $(PDIR)Makefile
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
#include "lflash.h" #include "lflash.h"
#include "platform.h" #include "platform.h"
#include "vfs.h" #include "vfs.h"
#include "uzlib.h"
#include "c_fcntl.h" #include "c_fcntl.h"
#include "c_stdio.h" #include "c_stdio.h"
...@@ -34,15 +35,52 @@ static uint32_t flashAddrPhys; ...@@ -34,15 +35,52 @@ static uint32_t flashAddrPhys;
static uint32_t flashSector; static uint32_t flashSector;
static uint32_t curOffset; static uint32_t curOffset;
#define ALIGN(s) (((s)+sizeof(size_t)-1) & ((size_t) (- (signed) sizeof(size_t)))) #define ALIGN(s) (((s)+sizeof(size_t)-1) & ((size_t) (- (signed) sizeof(size_t))))
#define ALIGN_BITS(s) (((uint32_t)s) & (sizeof(size_t)-1)) #define ALIGN_BITS(s) (((uint32_t)s) & (sizeof(size_t)-1))
#define ALL_SET cast(uint32_t, -1) #define ALL_SET (~0)
#define FLASH_SIZE LUA_FLASH_STORE #define FLASH_SIZE LUA_FLASH_STORE
#define FLASH_PAGE_SIZE INTERNAL_FLASH_SECTOR_SIZE #define FLASH_PAGE_SIZE INTERNAL_FLASH_SECTOR_SIZE
#define FLASH_PAGES (FLASH_SIZE/FLASH_PAGE_SIZE) #define FLASH_PAGES (FLASH_SIZE/FLASH_PAGE_SIZE)
#define READ_BLOCKSIZE 1024
#define WRITE_BLOCKSIZE 2048
#define DICTIONARY_WINDOW 16384
#define WORDSIZE (sizeof(int))
#define BITS_PER_WORD 32
#define WRITE_BLOCKS ((DICTIONARY_WINDOW/WRITE_BLOCKSIZE)+1)
#define WRITE_BLOCK_WORDS (WRITE_BLOCKSIZE/WORDSIZE)
char flash_region_base[FLASH_SIZE] ICACHE_FLASH_RESERVED_ATTR; char flash_region_base[FLASH_SIZE] ICACHE_FLASH_RESERVED_ATTR;
struct INPUT {
int fd;
int len;
uint8_t block[READ_BLOCKSIZE];
uint8_t *inPtr;
int bytesRead;
int left;
void *inflate_state;
} *in;
typedef struct {
uint8_t byte[WRITE_BLOCKSIZE];
} outBlock;
struct OUTPUT {
lua_State *L;
lu_int32 flash_sig;
int len;
outBlock *block[WRITE_BLOCKS];
outBlock buffer;
int ndx;
uint32_t crc;
int (*fullBlkCB) (void);
int flashLen;
int flagsLen;
int flagsNdx;
uint32_t *flags;
const char *error;
} *out;
#ifdef NODE_DEBUG #ifdef NODE_DEBUG
extern void dbg_printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2))); extern void dbg_printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
void dumpStrt(stringtable *tb, const char *type) { void dumpStrt(stringtable *tb, const char *type) {
...@@ -54,11 +92,11 @@ void dumpStrt(stringtable *tb, const char *type) { ...@@ -54,11 +92,11 @@ void dumpStrt(stringtable *tb, const char *type) {
for (i=0; i<tb->size; i++) for (i=0; i<tb->size; i++)
for(o = tb->hash[i], j=0; o; (o=o->gch.next), j++ ) { for(o = tb->hash[i], j=0; o; (o=o->gch.next), j++ ) {
TString *ts =cast(TString *, o); TString *ts =cast(TString *, o);
NODE_DBG("%5d %5d %08x %08x %5d %1s %s\n", NODE_DBG("%5d %5d %08x %08x %5d %1s %s\n",
i, j, (size_t) ts, ts->tsv.hash, ts->tsv.len, i, j, (size_t) ts, ts->tsv.hash, ts->tsv.len,
ts_isreadonly(ts) ? "R" : " ", getstr(ts)); ts_isreadonly(ts) ? "R" : " ", getstr(ts));
} }
} }
LUA_API void dumpStrings(lua_State *L) { LUA_API void dumpStrings(lua_State *L) {
dumpStrt(&G(L)->strt, "RAM"); dumpStrt(&G(L)->strt, "RAM");
...@@ -70,7 +108,7 @@ LUA_API void dumpStrings(lua_State *L) { ...@@ -70,7 +108,7 @@ LUA_API void dumpStrings(lua_State *L) {
/* ===================================================================================== /* =====================================================================================
* The next 4 functions: flashPosition, flashSetPosition, flashBlock and flashErase * The next 4 functions: flashPosition, flashSetPosition, flashBlock and flashErase
* wrap writing to flash. The last two are platform dependent. Also note that any * wrap writing to flash. The last two are platform dependent. Also note that any
* writes are suppressed if the global writeToFlash is false. This is used in * writes are suppressed if the global writeToFlash is false. This is used in
* phase I where the pass is used to size the structures in flash. * phase I where the pass is used to size the structures in flash.
*/ */
static char *flashPosition(void){ static char *flashPosition(void){
...@@ -104,28 +142,37 @@ static void flashErase(uint32_t start, uint32_t end){ ...@@ -104,28 +142,37 @@ static void flashErase(uint32_t start, uint32_t end){
platform_flash_erase_sector( flashSector + i ); platform_flash_erase_sector( flashSector + i );
} }
/* =====================================================================================
* luaN_init(), luaN_reload_reboot() and luaN_index() are exported via lflash.h.
* The first is the startup hook used in lstate.c and the last two are
* implementations of the node.flash API calls.
*/
/* /*
* Hook in lstate.c:f_luaopen() to set up ROstrt and ROpvmain if needed * Hook in lstate.c:f_luaopen() to set up ROstrt and ROpvmain if needed
*/ */
LUAI_FUNC void luaN_init (lua_State *L) { LUAI_FUNC void luaN_init (lua_State *L) {
// luaL_dbgbreak(); curOffset = 0;
curOffset = 0; flashAddr = flash_region_base;
flashAddr = flash_region_base; flashAddrPhys = platform_flash_mapped2phys((uint32_t)flashAddr);
flashAddrPhys = platform_flash_mapped2phys((uint32_t)flashAddr); flashSector = platform_flash_get_sector_of_address(flashAddrPhys);
flashSector = platform_flash_get_sector_of_address(flashAddrPhys);
FlashHeader *fh = cast(FlashHeader *, flashAddr); FlashHeader *fh = cast(FlashHeader *, flashAddr);
/* /*
* For the LFS to be valid, its signature has to be correct for this build variant, * For the LFS to be valid, its signature has to be correct for this build
* thr ROhash and main proto fields must be defined and the main proto address * variant, the ROhash and main proto fields must be defined and the main proto
* be within the LFS address bounds. (This last check is primarily to detect the * address be within the LFS address bounds. (This last check is primarily to
* direct imaging of an absolute LFS with the wrong base address. * detect the direct imaging of an absolute LFS with the wrong base address.
*/ */
if (fh->flash_sig == 0 || fh->flash_sig == ~0 ) {
NODE_ERR("No LFS image loaded\n");
return;
}
if ((fh->flash_sig & (~FLASH_SIG_ABSOLUTE)) != FLASH_SIG ) { if ((fh->flash_sig & (~FLASH_SIG_ABSOLUTE)) != FLASH_SIG ) {
NODE_ERR("Flash sig not correct: %p vs %p\n", NODE_ERR("Flash sig not correct: %p vs %p\n",
fh->flash_sig & (~FLASH_SIG_ABSOLUTE), FLASH_SIG); fh->flash_sig & (~FLASH_SIG_ABSOLUTE), FLASH_SIG);
return; return;
} }
...@@ -135,107 +182,89 @@ LUAI_FUNC void luaN_init (lua_State *L) { ...@@ -135,107 +182,89 @@ LUAI_FUNC void luaN_init (lua_State *L) {
fh->mainProto - cast(FlashAddr, fh), fh->flash_size); fh->mainProto - cast(FlashAddr, fh), fh->flash_size);
return; return;
} }
G(L)->ROstrt.hash = cast(GCObject **, fh->pROhash); G(L)->ROstrt.hash = cast(GCObject **, fh->pROhash);
G(L)->ROstrt.nuse = fh->nROuse ; G(L)->ROstrt.nuse = fh->nROuse ;
G(L)->ROstrt.size = fh->nROsize; G(L)->ROstrt.size = fh->nROsize;
G(L)->ROpvmain = cast(Proto *,fh->mainProto); G(L)->ROpvmain = cast(Proto *,fh->mainProto);
} }
#define BYTE_OFFSET(t,f) cast(size_t, &(cast(t *, NULL)->f)) //extern void software_reset(void);
/* static int loadLFS (lua_State *L);
* Rehook address chain to correct Flash byte addressed within the mapped adress space static int loadLFSgc (lua_State *L);
* Note that on input each 32-bit address field is split into 2×16-bit subfields static int procFirstPass (void);
* - the lu_int16 offset of the target address being referenced
* - the lu_int16 offset of the next address pointer.
*/
static int rebuild_core (int fd, uint32_t size, lu_int32 *buf, int is_absolute) {
int bi; /* byte offset into memory mapped LFS of current buffer */
int wNextOffset = BYTE_OFFSET(FlashHeader,mainProto)/sizeof(lu_int32);
int wj; /* word offset into current input buffer */
for (bi = 0; bi < size; bi += FLASH_PAGE_SIZE) {
int wi = bi / sizeof(lu_int32);
int blen = ((bi + FLASH_PAGE_SIZE) < size) ? FLASH_PAGE_SIZE : size - bi;
int wlen = blen / sizeof(lu_int32);
if (vfs_read(fd, buf , blen) != blen)
return 0;
if (!is_absolute) {
for (wj = 0; wj < wlen; wj++) {
if ((wi + wj) == wNextOffset) { /* this word is the next linked address */
int wTargetOffset = buf[wj]&0xFFFF;
wNextOffset = buf[wj]>>16;
lua_assert(!wNextOffset || (wNextOffset>(wi+wj) && wNextOffset<size/sizeof(lu_int32)));
buf[wj] = cast(lu_int32, flashAddr + wTargetOffset*sizeof(lu_int32));
}
}
}
flashBlock(buf, blen);
}
return size;
}
/* /*
* Library function called by node.flash.load(filename). * Library function called by node.flashreload(filename).
*/ */
LUALIB_API int luaN_reload_reboot (lua_State *L) { LUALIB_API int luaN_reload_reboot (lua_State *L) {
int fd, status, is_absolute; // luaL_dbgbreak();
FlashHeader fh; const char *fn = lua_tostring(L, 1), *msg = "";
int status;
const char *fn = lua_tostring(L, 1); /*
if (!fn || !(fd = vfs_open(fn, "r"))) * Do a protected call of loadLFS.
return 0; *
* - This will normally rewrite the LFS and reboot, with no return.
if (vfs_read(fd, &fh, sizeof(fh)) != sizeof(fh) || * - If an error occurs then it is sent to the UART.
(fh.flash_sig & (~FLASH_SIG_ABSOLUTE)) != FLASH_SIG) * - If this occured in the 1st pass, the previous LFS is unchanged so it is
return 0; * safe to return to the calling Lua.
* - If in the 1st pass, then the ESP is rebooted.
if (vfs_lseek(fd, -1, VFS_SEEK_END) != fh.flash_size-1 || */
vfs_lseek(fd, 0, VFS_SEEK_SET) != 0) status = lua_cpcall(L, &loadLFS, cast(void *,fn));
return 0;
if (!out || out->fullBlkCB == procFirstPass) {
is_absolute = fh.flash_sig & FLASH_SIG_ABSOLUTE; /*
lu_int32 *buffer = luaM_newvector(L, FLASH_PAGE_SIZE / sizeof(lu_int32), lu_int32); * Never entered the 2nd pass, so it is safe to return the error. Note
* that I've gone to some trouble to ensure that all dynamically allocated
/* * working areas have been freed, so that we have no memory leaks.
* This is the point of no return. We attempt to rebuild the flash. If there */
* are any problems them the Flash is going to be corrupt, so the only fallback if (status == LUA_ERRMEM)
* is to erase it and reboot with a clean but blank flash. Otherwise the reboot msg = "Memory allocation error";
* will load the new LFS. else if (out && out->error)
* msg = out->error;
* Note that the Lua state is not passed into the lua core because from this else
* point on, we make no calls on the Lua RTS. msg = "Unknown Error";
*/
flashErase(0,-1); /* We can clean up and return error */
if (rebuild_core(fd, fh.flash_size, buffer, is_absolute) != fh.flash_size) lua_cpcall(L, &loadLFSgc, NULL);
flashErase(0,-1); lua_settop(L, 0);
/* lua_pushstring(L, msg);
* Issue a break 0,0. This will either enter the debugger or force a restart if return 1;
* not installed. Follow this by a H/W timeout is a robust way to insure that }
* other interrupts / callbacks don't fire and reference THE old LFS context.
*/
asm("break 0,0" ::);
while (1) {}
if (status == 0) {
/* Successful LFS rewrite */
msg = "LFS region updated. Restarting.";
} else {
/* We have errored during the second pass so clear the LFS and reboot */
if (status == LUA_ERRMEM)
msg = "Memory allocation error";
else if (out->error)
msg = out->error;
else
msg = "Unknown Error";
flashErase(0,-1);
}
NODE_ERR(msg);
while (1) {} // Force WDT as the ROM software_reset() doesn't seem to work
return 0; return 0;
} }
/* /*
* In the arg is a valid LFS module name then return the LClosure pointing to it. * If the arg is a valid LFS module name then return the LClosure
* Otherwise return: * pointing to it. Otherwise return:
* - The Unix time that the LFS was built * - The Unix time that the LFS was built
* - The base address and length of the LFS * - The base address and length of the LFS
* - An array of the module names in the the LFS * - An array of the module names in the LFS
*/ */
LUAI_FUNC int luaN_index (lua_State *L) { LUAI_FUNC int luaN_index (lua_State *L) {
int i; int i;
int n = lua_gettop(L); int n = lua_gettop(L);
/* Return nil + the LFS base address if the LFS isn't loaded */ /* Return nil + the LFS base address if the LFS isn't loaded */
if(!(G(L)->ROpvmain)) { if(!(G(L)->ROpvmain)) {
lua_settop(L, 0); lua_settop(L, 0);
lua_pushnil(L); lua_pushnil(L);
...@@ -270,5 +299,262 @@ LUAI_FUNC int luaN_index (lua_State *L) { ...@@ -270,5 +299,262 @@ LUAI_FUNC int luaN_index (lua_State *L) {
lua_insert(L, 4); lua_insert(L, 4);
return 5; return 5;
} }
/* =====================================================================================
* The following routines use my uzlib which was based on pfalcon's inflate and
* deflate routines. The standard NodeMCU make also makes two host tools uz_zip
* and uz_unzip which also use these and luac.cross uses the deflate. As discussed
* below, The main action routine loadLFS() calls uzlib_inflate() to do the actual
* stream inflation but uses three supplied CBs to abstract input and output
* stream handling.
*
* ESP8266 RAM limitations and heap fragmentation are a key implementation
* constraint and hence these routines use a number of ~2K buffers (11) as
* working storage.
*
* The inflate is done twice, in order to limit storage use and avoid forward /
* backward reference issues. However this has a major advantage that the LFS
* is scanned with the headers, CRC, etc. validated BEFORE the write to flash
* is started, so the only real chance of failure during the second pass
* write is if a power fail occurs during the pass.
*/
static void flash_error(const char *err) {
if (out)
out->error = err;
if (in && in->inflate_state)
uz_free(in->inflate_state);
lua_pushnil(out->L); /* can't use it on a cpcall anyway */
lua_error(out->L);
}
/*
* uzlib_inflate does a stream inflate on an RFC 1951 encoded data stream.
* It uses three application-specific CBs passed in the call to do the work:
*
* - get_byte() CB to return next byte in input stream
* - put_byte() CB to output byte to output buffer
* - recall_byte() CB to output byte to retrieve a historic byte from
* the output buffer.
*
* Note that put_byte() also triggers secondary CBs to do further processing.
*/
static uint8_t get_byte (void) {
if (--in->left < 0) {
/* Read next input block */
int remaining = in->len - in->bytesRead;
int wanted = remaining >= READ_BLOCKSIZE ? READ_BLOCKSIZE : remaining;
if (vfs_read(in->fd, in->block, wanted) != wanted)
flash_error("read error on LFS image file");
system_soft_wdt_feed();
in->bytesRead += wanted;
in->inPtr = in->block;
in->left = wanted-1;
}
return *in->inPtr++;
}
static void put_byte (uint8_t value) {
int offset = out->ndx % WRITE_BLOCKSIZE; /* counts from 0 */
out->block[0]->byte[offset++] = value;
out->ndx++;
if (offset == WRITE_BLOCKSIZE || out->ndx == out->len) {
if (out->fullBlkCB)
out->fullBlkCB();
/* circular shift the block pointers (redundant on last block, but so what) */
outBlock *nextBlock = out->block[WRITE_BLOCKS - 1];
memmove(out->block+1, out->block, (WRITE_BLOCKS-1)*sizeof(void*));
out->block[0] = nextBlock ;
}
}
static uint8_t recall_byte (uint offset) {
if(offset > DICTIONARY_WINDOW || offset >= out->ndx)
flash_error("invalid dictionary offset on inflate");
/* ndx starts at 1. Need relative to 0 */
uint n = out->ndx - offset;
uint pos = n % WRITE_BLOCKSIZE;
uint blockNo = out->ndx / WRITE_BLOCKSIZE - n / WRITE_BLOCKSIZE;
return out->block[blockNo]->byte[pos];
}
/*
* On the first pass the break index is set to call this process at the end
* of each completed output buffer.
* - On the first call, the Flash Header is checked.
* - On each call the CRC is rolled up for that buffer.
* - Once the flags array is in-buffer this is also captured.
* This logic is slightly complicated by the last buffer is typically short.
*/
int procFirstPass (void) {
int len = (out->ndx % WRITE_BLOCKSIZE) ?
out->ndx % WRITE_BLOCKSIZE : WRITE_BLOCKSIZE;
if (out->ndx <= WRITE_BLOCKSIZE) {
uint32_t fl;
/* Process the flash header and cache the FlashHeader fields we need */
FlashHeader *fh = cast(FlashHeader *, out->block[0]);
out->flashLen = fh->flash_size; /* in bytes */
out->flagsLen = (out->len-fh->flash_size)/WORDSIZE; /* in words */
out->flash_sig = fh->flash_sig;
if ((fh->flash_sig & FLASH_FORMAT_MASK) != FLASH_FORMAT_VERSION)
flash_error("Incorrect LFS header version");
if ((fh->flash_sig & FLASH_SIG_B2_MASK) != FLASH_SIG_B2)
flash_error("Incorrect LFS build type");
if ((fh->flash_sig & ~FLASH_SIG_ABSOLUTE) != FLASH_SIG)
flash_error("incorrect LFS header signature");
if (fh->flash_size > FLASH_SIZE)
flash_error("LFS Image too big for configured LFS region");
if ((fh->flash_size & 0x3) ||
fh->flash_size > FLASH_SIZE ||
out->flagsLen != 1 + (out->flashLen/WORDSIZE - 1) / BITS_PER_WORD)
flash_error("LFS length mismatch");
out->flags = luaM_newvector(out->L, out->flagsLen, uint);
}
/* update running CRC */
out->crc = uzlib_crc32(out->block[0], len, out->crc);
/* copy out any flag vector */
if (out->ndx > out->flashLen) {
int start = out->flashLen - (out->ndx - len);
if (start < 0) start = 0;
memcpy(out->flags + out->flagsNdx, out->block[0]->byte + start, len - start);
out->flagsNdx += (len -start) / WORDSIZE; /* flashLen and len are word aligned */
}
return 1;
}
int procSecondPass (void) {
/*
* The length rules are different for the second pass since this only processes
* upto the flashLen and not the full image. This also works in word units.
* (We've already validated these are word multiples.)
*/
int i, len = (out->ndx > out->flashLen) ?
(out->flashLen % WRITE_BLOCKSIZE) / WORDSIZE :
WRITE_BLOCKSIZE / WORDSIZE;
uint32_t *buf = (uint32_t *) out->buffer.byte, flags;
/*
* Relocate all the addresses tagged in out->flags. This can't be done in
* place because the out->blocks are still in use as dictionary content so
* first copy the block to a working buffer and do the relocation in this.
*/
memcpy(out->buffer.byte, out->block[0]->byte, WRITE_BLOCKSIZE);
for (i=0; i<len; i++,flags>>=1 ) {
if ((i&31)==0)
flags = out->flags[out->flagsNdx++];
if (flags&1)
buf[i] = WORDSIZE*buf[i] + cast(uint32_t, flashAddr);
}
/*
* On first block, set the flash_sig has the in progress bit set and this
* is not cleared until end.
*/
if (out->ndx <= WRITE_BLOCKSIZE)
buf[0] = out->flash_sig | FLASH_SIG_IN_PROGRESS;
flashBlock(buf, len*WORDSIZE);
if (out->ndx >= out->flashLen) {
/* we're done so disable CB and rewrite flash sig to complete flash */
flashSetPosition(0);
flashBlock(&out->flash_sig, WORDSIZE);
out->fullBlkCB = NULL;
}
}
/*
* loadLFS)() is protected called from luaN_reload_reboot so that it can recover
* from out of memory and other thrown errors. loadLFSgc() GCs any resources.
*/
static int loadLFS (lua_State *L) {
const char *fn = cast(const char *, lua_touserdata(L, 1));
int i, n, res;
uint32_t crc;
/* Allocate and zero in and out structures */
in = NULL; out = NULL;
in = luaM_new(L, struct INPUT);
memset(in, 0, sizeof(*in));
out = luaM_new(L, struct OUTPUT);
memset(out, 0, sizeof(*out));
out->L = L;
out->fullBlkCB = procFirstPass;
out->crc = ~0;
/* Open LFS image/ file, read unpacked length from last 4 byte and rewind */
if (!(in->fd = vfs_open(fn, "r")))
flash_error("LFS image file not found");
in->len = vfs_size(in->fd);
if (in->len <= 200 || /* size of an empty luac output */
vfs_lseek(in->fd, in->len-4, VFS_SEEK_SET) != in->len-4 ||
vfs_read(in->fd, &out->len, sizeof(uint)) != sizeof(uint))
flash_error("read error on LFS image file");
vfs_lseek(in->fd, 0, VFS_SEEK_SET);
/* Allocate the out buffers */
for(i = 0; i <= WRITE_BLOCKS; i++)
out->block[i] = luaM_new(L, outBlock);
/* first inflate pass */
if (uzlib_inflate (get_byte, put_byte, recall_byte,
in->len, &crc, &in->inflate_state) < 0)
flash_error("read error on LFS image file");
if (crc != ~out->crc)
flash_error("checksum error on LFS image file");
out->fullBlkCB = procSecondPass;
out->flagsNdx = 0;
out->ndx = 0;
in->bytesRead = in->left = 0;
/*
* Once we have completed the 1st pass then the LFS image has passed the
* basic signature, crc and length checks, so now we can reset the counts
* to do the actual write to flash on the second pass.
*/
vfs_lseek(in->fd, 0, VFS_SEEK_SET);
flashErase(0,(out->flashLen - 1)/FLASH_PAGE_SIZE);
flashSetPosition(0);
if (uzlib_inflate(get_byte, put_byte, recall_byte,
in->len, &crc, &in->inflate_state) != UZLIB_OK)
if (res < 0) {
const char *err[] = {"Data_error during decompression",
"Chksum_error during decompression",
"Dictionary error during decompression"
"Memory_error during decompression"};
flash_error(err[UZLIB_DATA_ERROR - res]);
}
return 0;
}
static int loadLFSgc (lua_State *L) {
int i;
if (out) {
for (i = 0; i < WRITE_BLOCKS; i++)
if (out->block[i])
luaM_free(L, out->block[i]);
if (out->flags)
luaM_freearray(L, out->flags, out->flagsLen, uint32_t);
luaM_free(L, out);
}
if (in) {
if (in->fd)
vfs_close(in->fd);
luaM_free(L, in);
}
return 0;
}
#endif #endif
...@@ -15,7 +15,8 @@ ...@@ -15,7 +15,8 @@
#else #else
# define FLASH_SIG_B1 0x00 # define FLASH_SIG_B1 0x00
#endif #endif
#define FLASH_FORMAT_VERSION (1 << 8)
#define FLASH_FORMAT_MASK 0xF00
#ifdef LUA_PACK_TVALUES #ifdef LUA_PACK_TVALUES
#ifdef LUA_NUMBER_INTEGRAL #ifdef LUA_NUMBER_INTEGRAL
#error "LUA_PACK_TVALUES is only valid for Floating point builds" #error "LUA_PACK_TVALUES is only valid for Floating point builds"
...@@ -24,9 +25,10 @@ ...@@ -24,9 +25,10 @@
#else #else
# define FLASH_SIG_B2 0x00 # define FLASH_SIG_B2 0x00
#endif #endif
# define FLASH_SIG_B2_MASK 0x04
#define FLASH_SIG_ABSOLUTE 0x01 #define FLASH_SIG_ABSOLUTE 0x01
#define FLASH_SIG_IN_PROGRESS 0x08 #define FLASH_SIG_IN_PROGRESS 0x08
#define FLASH_SIG (0xfafaaf50 | FLASH_SIG_B2 | FLASH_SIG_B1) #define FLASH_SIG (0xfafaa050 | FLASH_FORMAT_VERSION |FLASH_SIG_B2 | FLASH_SIG_B1)
typedef lu_int32 FlashAddr; typedef lu_int32 FlashAddr;
typedef struct { typedef struct {
......
...@@ -43,7 +43,7 @@ ...@@ -43,7 +43,7 @@
#define stringmark(s) if (!isLFSobject(&(s)->tsv)) {reset2bits((s)->tsv.marked, WHITE0BIT, WHITE1BIT);} #define stringmark(s) if (!isLFSobject(&(s)->tsv)) {reset2bits((s)->tsv.marked, WHITE0BIT, WHITE1BIT);}
#define isfinalized(u) testbit((u)->marked, FINALIZEDBIT) #define isfinalized(u) testbit(getmarked(u), FINALIZEDBIT)
#define markfinalized(u) l_setbit((u)->marked, FINALIZEDBIT) #define markfinalized(u) l_setbit((u)->marked, FINALIZEDBIT)
...@@ -73,12 +73,12 @@ static void removeentry (Node *n) { ...@@ -73,12 +73,12 @@ static void removeentry (Node *n) {
static void reallymarkobject (global_State *g, GCObject *o) { static void reallymarkobject (global_State *g, GCObject *o) {
/* don't mark LFS Protos (or strings) */ /* don't mark LFS Protos (or strings) */
if (o->gch.tt == LUA_TPROTO && isLFSobject(&(o->gch))) if (gettt(&o->gch) == LUA_TPROTO && isLFSobject(&(o->gch)))
return; return;
lua_assert(iswhite(o) && !isdead(g, o)); lua_assert(iswhite(o) && !isdead(g, o));
white2gray(o); white2gray(o);
switch (o->gch.tt) { switch (gettt(&o->gch)) {
case LUA_TSTRING: { case LUA_TSTRING: {
return; return;
} }
...@@ -295,7 +295,7 @@ static l_mem propagatemark (global_State *g) { ...@@ -295,7 +295,7 @@ static l_mem propagatemark (global_State *g) {
GCObject *o = g->gray; GCObject *o = g->gray;
lua_assert(isgray(o)); lua_assert(isgray(o));
gray2black(o); gray2black(o);
switch (o->gch.tt) { switch (gettt(&o->gch)) {
case LUA_TTABLE: { case LUA_TTABLE: {
Table *h = gco2h(o); Table *h = gco2h(o);
g->gray = h->gclist; g->gray = h->gclist;
...@@ -400,7 +400,7 @@ static void cleartable (GCObject *l) { ...@@ -400,7 +400,7 @@ static void cleartable (GCObject *l) {
static void freeobj (lua_State *L, GCObject *o) { static void freeobj (lua_State *L, GCObject *o) {
switch (o->gch.tt) { switch (gettt(&o->gch)) {
case LUA_TPROTO: case LUA_TPROTO:
lua_assert(!isLFSobject(&(o->gch))); lua_assert(!isLFSobject(&(o->gch)));
luaF_freeproto(L, gco2p(o)); luaF_freeproto(L, gco2p(o));
......
...@@ -102,8 +102,8 @@ ...@@ -102,8 +102,8 @@
#define fixedstack(x) l_setbit((x)->marked, FIXEDSTACKBIT) #define fixedstack(x) l_setbit((x)->marked, FIXEDSTACKBIT)
#define unfixedstack(x) resetbit((x)->marked, FIXEDSTACKBIT) #define unfixedstack(x) resetbit((x)->marked, FIXEDSTACKBIT)
#ifdef LUA_FLASH_STORE #ifdef LUA_FLASH_STORE
#define isLFSobject(x) testbit((x)->marked, LFSBIT) #define isLFSobject(x) testbit(getmarked(x), LFSBIT)
#define stringfix(s) if (!test2bits((s)->tsv.marked, FIXEDBIT, LFSBIT)) {l_setbit((s)->tsv.marked, FIXEDBIT);} #define stringfix(s) if (!test2bits(getmarked(&(s)->tsv), FIXEDBIT, LFSBIT)) {l_setbit((s)->tsv.marked, FIXEDBIT);}
#else #else
#define isLFSobject(x) (0) #define isLFSobject(x) (0)
#define stringfix(s) {l_setbit((s)->tsv.marked, FIXEDBIT);} #define stringfix(s) {l_setbit((s)->tsv.marked, FIXEDBIT);}
......
...@@ -34,20 +34,33 @@ ...@@ -34,20 +34,33 @@
#define LUA_TUPVAL (LAST_TAG+2) #define LUA_TUPVAL (LAST_TAG+2)
#define LUA_TDEADKEY (LAST_TAG+3) #define LUA_TDEADKEY (LAST_TAG+3)
#ifdef __XTENSA__
/*
** force aligned access to critical fields in Flash-based structures
** wo is the offset of aligned word in bytes 0,4,8,..
** bo is the field within the word in bits 0..31
*/
#define GET_BYTE_FN(name,t,wo,bo) \
static inline lu_byte get ## name(void *o) { \
lu_byte res; /* extract named field */ \
asm ("l32i %0, %1, " #wo "; extui %0, %0, " #bo ", 8;" : "=r"(res) : "r"(o) : );\
return res; }
#else
#define GET_BYTE_FN(name,t,wo,bo) \
static inline lu_byte get ## name(void *o) { return ((t *)o)->name; }
#endif
/* /*
** Union of all collectable objects ** Union of all collectable objects
*/ */
typedef union GCObject GCObject; typedef union GCObject GCObject;
/* /*
** Common Header for all collectable objects (in macro form, to be ** Common Header for all collectable objects (in macro form, to be
** included in other objects) ** included in other objects)
*/ */
#define CommonHeader GCObject *next; lu_byte tt; lu_byte marked #define CommonHeader GCObject *next; lu_byte tt; lu_byte marked
/* /*
** Common header in struct form ** Common header in struct form
*/ */
...@@ -55,11 +68,18 @@ typedef struct GCheader { ...@@ -55,11 +68,18 @@ typedef struct GCheader {
CommonHeader; CommonHeader;
} GCheader; } GCheader;
/*
** Word aligned inline access functions for the CommonHeader tt and marked fields.
** Note that these MUST be consistent with the CommonHeader definition above. Arg
** 3 is a word offset (4 bytes in this case) and arg 4 the bit offset in the word.
*/
GET_BYTE_FN(tt,GCheader,4,0)
GET_BYTE_FN(marked,GCheader,4,8)
#if defined(LUA_PACK_VALUE) || defined(ELUA_ENDIAN_BIG) || defined(ELUA_ENDIAN_SMALL) #if defined(LUA_PACK_VALUE) || defined(ELUA_ENDIAN_BIG) || defined(ELUA_ENDIAN_SMALL)
# error "NodeMCU does not support the eLua LUA_PACK_VALUE and ELUA_ENDIAN defines" # error "NodeMCU does not support the eLua LUA_PACK_VALUE and ELUA_ENDIAN defines"
#endif #endif
/* /*
** Union of all Lua values ** Union of all Lua values
*/ */
...@@ -214,7 +234,6 @@ typedef struct lua_TValue { ...@@ -214,7 +234,6 @@ typedef struct lua_TValue {
#define iscollectable(o) (ttype(o) >= LUA_TSTRING) #define iscollectable(o) (ttype(o) >= LUA_TSTRING)
typedef TValue *StkId; /* index to stack elements */ typedef TValue *StkId; /* index to stack elements */
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
#include "lstate.h" #include "lstate.h"
#define sizestring(s) (sizeof(union TString)+(testbit((s)->marked, READONLYBIT) ? sizeof(char **) : ((s)->len+1)*sizeof(char))) #define sizestring(s) (sizeof(union TString)+(testbit(getmarked(s), READONLYBIT) ? sizeof(char **) : ((s)->len+1)*sizeof(char)))
#define sizeudata(u) (sizeof(union Udata)+(u)->len) #define sizeudata(u) (sizeof(union Udata)+(u)->len)
......
# #
# This Make file is called from the core Makefile hierarchy which is a hierarchical # This Make file is called from the core Makefile hierarchy with is a hierarchical
# make which uses parent callbacks to implement inheritance. However if luac_cross # make wwhich uses parent callbacks to implement inheritance. However is luac_cross
# build stands outside this it uses the host toolchain to implement a separate # build stands outside this and uses the host toolchain to implement a separate
# host build of the luac.cross image. # host build of the luac.cross image.
# #
.NOTPARALLEL: .NOTPARALLEL:
CCFLAGS:= -I.. -I../../include -I../../../include -I ../../libc CCFLAGS:= -I.. -I../../include -I../../libc -I../../uzlib
LDFLAGS:= -L$(SDK_DIR)/lib -L$(SDK_DIR)/ld -lm -ldl -Wl,-Map=mapfile LDFLAGS:= -L$(SDK_DIR)/lib -L$(SDK_DIR)/ld -lm -ldl -Wl,-Map=mapfile
CCFLAGS += -Wall CCFLAGS += -Wall
...@@ -31,13 +31,13 @@ LUASRC := lapi.c lauxlib.c lbaselib.c lcode.c ldblib.c ldebug.c ...@@ -31,13 +31,13 @@ LUASRC := lapi.c lauxlib.c lbaselib.c lcode.c ldblib.c ldebug.c
lrotable.c lstate.c lstring.c lstrlib.c ltable.c ltablib.c \ lrotable.c lstate.c lstring.c lstrlib.c ltable.c ltablib.c \
ltm.c lundump.c lvm.c lzio.c ltm.c lundump.c lvm.c lzio.c
LIBCSRC := c_stdlib.c LIBCSRC := c_stdlib.c
UZSRC := uzlib_deflate.c crc32.c
# #
# This relies on the files being unique on the vpath # This relies on the files being unique on the vpath
# #
SRC := $(LUACSRC) $(LUASRC) $(LIBCSRC) SRC := $(LUACSRC) $(LUASRC) $(LIBCSRC) $(UZSRC)
vpath %.c .:..:../../libc vpath %.c .:..:../../libc:../../uzlib
ODIR := .output/$(TARGET)/$(FLAVOR)/obj ODIR := .output/$(TARGET)/$(FLAVOR)/obj
...@@ -47,16 +47,11 @@ DEPS := $(SRC:%.c=$(ODIR)/%.d) ...@@ -47,16 +47,11 @@ DEPS := $(SRC:%.c=$(ODIR)/%.d)
CFLAGS = $(CCFLAGS) $(DEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES) CFLAGS = $(CCFLAGS) $(DEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES)
DFLAGS = $(CCFLAGS) $(DDEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES) DFLAGS = $(CCFLAGS) $(DDEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES)
CC := $(WRAPCC) gcc CC := gcc
ECHO := echo ECHO := echo
BUILD_TYPE := $(shell $(CC) $(EXTRA_CCFLAGS) -E -dM - <../../../app/include/user_config.h | grep LUA_NUMBER_INTEGRAL | wc -l)
ifeq ($(BUILD_TYPE),0)
IMAGE := ../../../luac.cross IMAGE := ../../../luac.cross
else
IMAGE := ../../../luac.cross.int
endif
.PHONY: test clean all .PHONY: test clean all
...@@ -70,7 +65,6 @@ test : ...@@ -70,7 +65,6 @@ test :
@echo SRC: $(SRC) @echo SRC: $(SRC)
@echo OBJS: $(OBJS) @echo OBJS: $(OBJS)
@echo DEPS: $(DEPS) @echo DEPS: $(DEPS)
@echo IMAGE: $(IMAGE)
clean : clean :
$(RM) -r $(ODIR) $(RM) -r $(ODIR)
......
/* /***--
** lflashimg.c ** lflashimg.c
** Dump a compiled Proto hiearchy to a RO (FLash) image file ** Dump a compiled Proto hiearchy to a RO (FLash) image file
** See Copyright Notice in lua.h ** See Copyright Notice in lua.h
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#undef LUA_FLASH_STORE #undef LUA_FLASH_STORE
#define LUA_FLASH_STORE #define LUA_FLASH_STORE
#include "lflash.h" #include "lflash.h"
#include "uzlib.h"
//#define LOCAL_DEBUG //#define LOCAL_DEBUG
...@@ -46,18 +47,18 @@ typedef unsigned int uint; ...@@ -46,18 +47,18 @@ typedef unsigned int uint;
* independent image format, which permits the on-device image loader to load the LFS * independent image format, which permits the on-device image loader to load the LFS
* image at an appropriate base within the flash address space. As all objects in the * image at an appropriate base within the flash address space. As all objects in the
* LFS can be treated as multiples of 4-byte words, also all address fields are both * LFS can be treated as multiples of 4-byte words, also all address fields are both
* word aligned, and any address references within the LFS are also word-aligned, * word aligned, and any address references within the LFS are also word-aligned.
* such addresses are stored in a special format, where each PI address is two
* 16-bit unsigned offsets:
* *
* Bits 0-15 is the offset into the LFS that this address refers to * This version adds gzip compression of the generated LFS image for more efficient
* Bits 16-31 is the offset linking to the PIC next address. * over-the-air (OTA) transfer, so the method of tagging address words has been
* replaced by a scheme which achieves better compression: an additional bitmap
* has been added to the image, with each bit corresponding to a word in the image
* and set if the corresponding work is an address. The addresses are stored as
* signed relative word offsets.
* *
* Hence the LFS can be up to 256Kb in length and the flash loader can use the forward * The unloader is documented in lflash.c Note that his relocation process is
* links to chain down PI address from the mainProto address at offet 3 to all image * skipped for absolute addressed images (which are identified by the
* addresses during load and convert them to the corresponding correct absolute memory * FLASH_SIG_ABSOLUTE bit setting in the flash signature).
* addresses. This reloation process is skipped for absolute addressed images (which
* are identified by the FLASH_SIG_ABSOLUTE bit setting in the flash signature.
* *
* The flash image has a standard header detailed in lflash.h * The flash image has a standard header detailed in lflash.h
* *
...@@ -66,7 +67,7 @@ typedef unsigned int uint; ...@@ -66,7 +67,7 @@ typedef unsigned int uint;
* and int may not have the same size. Hence addresses with the must be declared as * and int may not have the same size. Hence addresses with the must be declared as
* the FlashAddr type rather than typed C pointers and must be accessed through macros. * the FlashAddr type rather than typed C pointers and must be accessed through macros.
* *
* ALso note that image built with a given LUA_PACK_TVALUES / LUA_NUNBER_INTEGRAL * Also note that image built with a given LUA_PACK_TVALUES / LUA_NUNBER_INTEGRAL
* combination must be loaded into a corresponding firmware build. Hence these * combination must be loaded into a corresponding firmware build. Hence these
* configuration options are also included in the FLash Signature. * configuration options are also included in the FLash Signature.
* *
...@@ -96,8 +97,19 @@ typedef struct flashts { /* This is the fixed 32-bit equivalent of TString ...@@ -96,8 +97,19 @@ typedef struct flashts { /* This is the fixed 32-bit equivalent of TString
#endif #endif
static uint curOffset = 0; static uint curOffset = 0;
static uint flashImage[LUA_MAX_FLASH_SIZE];
static unsigned char flashAddrTag[LUA_MAX_FLASH_SIZE/WORDSIZE]; /*
* The flashAddrTag is a bit array, one bit per flashImage word denoting
* whether the corresponding word is a relative address. The defines
* are access methods for this bit array.
*/
static uint flashImage[LUA_MAX_FLASH_SIZE + LUA_MAX_FLASH_SIZE/32];
static uint *flashAddrTag = flashImage + LUA_MAX_FLASH_SIZE;
#define _TW(v) (v)>>5
#define _TB(v) (1<<((v)&0x1F))
#define setFlashAddrTag(v) flashAddrTag[_TW(v)] |= _TB(v)
#define getFlashAddrTag(v) ((flashAddrTag[_TW(v)]&_TB(v)) != 0)
#define fatal luac_fatal #define fatal luac_fatal
extern void __attribute__((noreturn)) luac_fatal(const char* message); extern void __attribute__((noreturn)) luac_fatal(const char* message);
...@@ -115,7 +127,7 @@ static void *flashAlloc(lua_State* L, size_t n) { ...@@ -115,7 +127,7 @@ static void *flashAlloc(lua_State* L, size_t n) {
void *p = (void *)(flashImage + curOffset); void *p = (void *)(flashImage + curOffset);
curOffset += ALIGN(n)>>WORDSHIFT; curOffset += ALIGN(n)>>WORDSHIFT;
if (curOffset > LUA_MAX_FLASH_SIZE) { if (curOffset > LUA_MAX_FLASH_SIZE) {
fatal("Out of Flash memmory"); fatal("Out of Flash memory");
} }
return p; return p;
} }
...@@ -128,8 +140,8 @@ static void *flashAlloc(lua_State* L, size_t n) { ...@@ -128,8 +140,8 @@ static void *flashAlloc(lua_State* L, size_t n) {
#define toFlashAddr(l, pd, s) _toFlashAddr(l, &(pd), s) #define toFlashAddr(l, pd, s) _toFlashAddr(l, &(pd), s)
static void _toFlashAddr(lua_State* L, FlashAddr *a, void *p) { static void _toFlashAddr(lua_State* L, FlashAddr *a, void *p) {
uint doffset = cast(char *, a) - cast(char *,flashImage); uint doffset = cast(char *, a) - cast(char *,flashImage);
lua_assert(!(doffset & (WORDSIZE-1))); lua_assert(!(doffset & (WORDSIZE-1))); // check word aligned
doffset >>= WORDSHIFT; doffset >>= WORDSHIFT; // and convert to a word offset
lua_assert(doffset <= curOffset); lua_assert(doffset <= curOffset);
if (p) { if (p) {
uint poffset = cast(char *, p) - cast(char *,flashImage); uint poffset = cast(char *, p) - cast(char *,flashImage);
...@@ -137,10 +149,8 @@ static void _toFlashAddr(lua_State* L, FlashAddr *a, void *p) { ...@@ -137,10 +149,8 @@ static void _toFlashAddr(lua_State* L, FlashAddr *a, void *p) {
poffset >>= WORDSHIFT; poffset >>= WORDSHIFT;
lua_assert(poffset <= curOffset); lua_assert(poffset <= curOffset);
flashImage[doffset] = poffset; // Set the pointer to the offset flashImage[doffset] = poffset; // Set the pointer to the offset
flashAddrTag[doffset] = 1; // And tag as an address setFlashAddrTag(doffset); // And tag as an address
} else { // Special case for NULL pointer } /* else leave clear */ // Special case for NULL pointer
flashImage[doffset] = 0;
}
} }
/* /*
...@@ -231,7 +241,7 @@ static void createROstrt(lua_State *L, FlashHeader *fh) { ...@@ -231,7 +241,7 @@ static void createROstrt(lua_State *L, FlashHeader *fh) {
fts->marked = bitmask(LFSBIT); // LFS string with no Whitebits set fts->marked = bitmask(LFSBIT); // LFS string with no Whitebits set
fts->hash = hash; // add hash fts->hash = hash; // add hash
fts->len = len; // and length fts->len = len; // and length
memcpy(flashAlloc(L, ALIGN(len+1)), p, ALIGN(len+1)); // copy string memcpy(flashAlloc(L, len+1), p, len+1); // copy string
// include the trailing null char // include the trailing null char
lua_pop(L, 1); // Junk the value lua_pop(L, 1); // Junk the value
lua_pushvalue(L, -1); // Dup the key as rawset dumps its copy lua_pushvalue(L, -1); // Dup the key as rawset dumps its copy
...@@ -308,6 +318,9 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) { ...@@ -308,6 +318,9 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) {
case 'I': case 'I':
*d++ = *s++; *d++ = *s++;
break; break;
case 'H':
*d++ = (*s++) & 0;
break;
case 'S': case 'S':
newts = resolveTString(L, *cast(TString **, s)); newts = resolveTString(L, *cast(TString **, s));
toFlashAddr(L, *d, newts); toFlashAddr(L, *d, newts);
...@@ -318,11 +331,15 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) { ...@@ -318,11 +331,15 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) {
/* This code has to work for both Integer and Float build variants */ /* This code has to work for both Integer and Float build variants */
memset(d, 0, TARGET_TV_SIZE); memset(d, 0, TARGET_TV_SIZE);
TValue *sv = cast(TValue *, s); TValue *sv = cast(TValue *, s);
/* The value is 0, 4 or 8 bytes depending on type */
if (ttisstring(sv)) { if (ttisstring(sv)) {
toFlashAddr(L, *d, resolveTString(L, rawtsvalue(sv))); toFlashAddr(L, *d, resolveTString(L, rawtsvalue(sv)));
} else { /* non-collectable types all of size lua_Number */ } else if (ttisnumber(sv)) {
lua_assert(!iscollectable(sv));
*cast(lua_Number*,d) = *cast(lua_Number*,s); *cast(lua_Number*,d) = *cast(lua_Number*,s);
} else if (!ttisnil(sv)){
/* all other types are 4 byte */
lua_assert(!iscollectable(sv));
*cast(uint *,d) = *cast(uint *,s);
} }
*cast(int *,cast(lua_Number*,d)+1) = ttype(sv); *cast(int *,cast(lua_Number*,d)+1) = ttype(sv);
s += FLASH_WORDS(TValue); s += FLASH_WORDS(TValue);
...@@ -338,9 +355,9 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) { ...@@ -338,9 +355,9 @@ static void *flashCopy(lua_State* L, int n, const char *fmt, void *src) {
/* The debug optimised version has a different Proto layout */ /* The debug optimised version has a different Proto layout */
#ifdef LUA_OPTIMIZE_DEBUG #ifdef LUA_OPTIMIZE_DEBUG
#define PROTO_COPY_MASK "AIAAAAAASIIIIIIIAI" #define PROTO_COPY_MASK "AHAAAAAASIIIIIIIAI"
#else #else
#define PROTO_COPY_MASK "AIAAAAAASIIIIIIIIAI" #define PROTO_COPY_MASK "AHAAAAAASIIIIIIIIAI"
#endif #endif
/* /*
...@@ -378,44 +395,52 @@ static void *functionToFlash(lua_State* L, const Proto* orig) { ...@@ -378,44 +395,52 @@ static void *functionToFlash(lua_State* L, const Proto* orig) {
return cast(void *, flashCopy(L, 1, PROTO_COPY_MASK, &f)); return cast(void *, flashCopy(L, 1, PROTO_COPY_MASK, &f));
} }
/*
* Scan through the tagged addresses. This operates in one of two modes.
* - If address is non-zero then the offset is converted back into an absolute
* mapped flash address using the specified address base.
*
* - If the address is zero then form a form linked chain with the upper 16 bits
* the link to the last offset. As the scan is backwards, this 'last' address
* becomes forward reference for the on-chip LFS loader.
*/
void linkAddresses(lu_int32 address){
int i, last = 0;
for (i = curOffset-1 ; i >= 0; i--) {
if (flashAddrTag[i]) {
lua_assert(flashImage[i]<curOffset);
if (address) {
flashImage[i] = 4*flashImage[i] + address;
} else {
flashImage[i] |= last<<16;
last = i;
}
}
}
}
uint dumpToFlashImage (lua_State* L, const Proto *main, lua_Writer w, uint dumpToFlashImage (lua_State* L, const Proto *main, lua_Writer w,
void* data, int strip, lu_int32 address) { void* data, int strip,
lu_int32 address, lu_int32 maxSize) {
// parameter strip is ignored for now // parameter strip is ignored for now
lua_newtable(L);
FlashHeader *fh = cast(FlashHeader *, flashAlloc(L, sizeof(FlashHeader))); FlashHeader *fh = cast(FlashHeader *, flashAlloc(L, sizeof(FlashHeader)));
int i, status;
lua_newtable(L);
scanProtoStrings(L, main); scanProtoStrings(L, main);
createROstrt(L, fh); createROstrt(L, fh);
toFlashAddr(L, fh->mainProto, functionToFlash(L, main)); toFlashAddr(L, fh->mainProto, functionToFlash(L, main));
fh->flash_sig = FLASH_SIG + (address ? FLASH_SIG_ABSOLUTE : 0); fh->flash_sig = FLASH_SIG + (address ? FLASH_SIG_ABSOLUTE : 0);
fh->flash_size = curOffset*WORDSIZE; fh->flash_size = curOffset*WORDSIZE;
linkAddresses(address); if (fh->flash_size>maxSize) {
lua_unlock(L); fatal ("The image is too large for specfied LFS size");
int status = w(L, flashImage, curOffset * sizeof(uint), data); }
if (address) { /* in absolute mode convert addresses to mapped address */
for (i = 0 ; i < curOffset; i++)
if (getFlashAddrTag(i))
flashImage[i] = 4*flashImage[i] + address;
lua_unlock(L);
status = w(L, flashImage, fh->flash_size, data);
} else { /* compressed PI mode */
/*
* In image mode, shift the relocation bitmap down directly above
* the used flashimage. This consolidated array is then gzipped.
*/
uint oLen;
uint8_t *oBuf;
int bmLen = sizeof(uint)*((curOffset+31)/32); /* 32 flags to a word */
memmove(flashImage+curOffset, flashAddrTag, bmLen);
status = uzlib_compress (&oBuf, &oLen,
(const uint8_t *)flashImage, bmLen+fh->flash_size);
if (status != UZLIB_OK) {
luac_fatal("Out of memory during image compression");
}
lua_unlock(L);
#if 0
status = w(L, flashImage, bmLen+fh->flash_size, data);
#else
status = w(L, oBuf, oLen, data);
free(oBuf);
#endif
}
lua_lock(L); lua_lock(L);
return status; return status;
} }
...@@ -35,6 +35,7 @@ static int dumping=1; /* dump bytecodes? */ ...@@ -35,6 +35,7 @@ static int dumping=1; /* dump bytecodes? */
static int stripping=0; /* strip debug information? */ static int stripping=0; /* strip debug information? */
static int flash=0; /* output flash image */ static int flash=0; /* output flash image */
static lu_int32 address=0; /* output flash image at absolute location */ static lu_int32 address=0; /* output flash image at absolute location */
static lu_int32 maxSize=0x40000; /* maximuum uncompressed image size */
static int lookup=0; /* output lookup-style master combination header */ static int lookup=0; /* output lookup-style master combination header */
static char Output[]={ OUTPUT }; /* default output file name */ static char Output[]={ OUTPUT }; /* default output file name */
static const char* output=Output; /* actual output file name */ static const char* output=Output; /* actual output file name */
...@@ -72,6 +73,7 @@ static void usage(const char* message) ...@@ -72,6 +73,7 @@ static void usage(const char* message)
" -f output a flash image file\n" " -f output a flash image file\n"
" -a addr generate an absolute, rather than position independent flash image file\n" " -a addr generate an absolute, rather than position independent flash image file\n"
" -i generate lookup combination master (default with option -f)\n" " -i generate lookup combination master (default with option -f)\n"
" -m size maximum LFS image in bytes\n"
" -p parse only\n" " -p parse only\n"
" -s strip debug information\n" " -s strip debug information\n"
" -v show version information\n" " -v show version information\n"
...@@ -123,6 +125,13 @@ static int doargs(int argc, char* argv[]) ...@@ -123,6 +125,13 @@ static int doargs(int argc, char* argv[])
lookup = 1; lookup = 1;
else if (IS("-l")) /* list */ else if (IS("-l")) /* list */
++listing; ++listing;
else if (IS("-m")) /* specify a maximum image size */
{
flash=lookup=1;
maxSize=strtol(argv[++i],NULL,0);
if (maxSize & 0xFFF)
usage(LUA_QL("-e") " maximum size must be a multiple of 4,096");
}
else if (IS("-o")) /* output file */ else if (IS("-o")) /* output file */
{ {
output=argv[++i]; output=argv[++i];
...@@ -264,7 +273,8 @@ struct Smain { ...@@ -264,7 +273,8 @@ struct Smain {
}; };
extern uint dumpToFlashImage (lua_State* L,const Proto *main, lua_Writer w, extern uint dumpToFlashImage (lua_State* L,const Proto *main, lua_Writer w,
void* data, int strip, lu_int32 address); void* data, int strip,
lu_int32 address, lu_int32 maxSize);
static int pmain(lua_State* L) static int pmain(lua_State* L)
{ {
...@@ -302,7 +312,7 @@ static int pmain(lua_State* L) ...@@ -302,7 +312,7 @@ static int pmain(lua_State* L)
lua_lock(L); lua_lock(L);
if (flash) if (flash)
{ {
result=dumpToFlashImage(L,f,writer, D, stripping, address); result=dumpToFlashImage(L,f,writer, D, stripping, address, maxSize);
} else } else
{ {
result=luaU_dump_crosscompile(L,f,writer,D,stripping,target); result=luaU_dump_crosscompile(L,f,writer,D,stripping,target);
......
...@@ -33,77 +33,70 @@ ...@@ -33,77 +33,70 @@
#include "user_exceptions.h" #include "user_exceptions.h"
#define LOAD_MASK 0x00f00fu
#define L8UI_MATCH 0x000002u
#define L16UI_MATCH 0x001002u
#define L16SI_MATCH 0x009002u
static exception_handler_fn load_store_handler; static exception_handler_fn load_store_handler;
void load_non_32_wide_handler (struct exception_frame *ef, uint32_t cause) void load_non_32_wide_handler (struct exception_frame *ef, uint32_t cause)
{ {
/* If this is not EXCCAUSE_LOAD_STORE_ERROR you're doing it wrong! */ uint32_t val, insn;
(void)cause; (void)cause; /* If this is not EXCCAUSE_LOAD_STORE_ERROR you're doing it wrong! */
uint32_t epc1 = ef->epc;
uint32_t excvaddr;
uint32_t insn;
asm ( asm (
"rsr %0, EXCVADDR;" /* read out the faulting address */ /*
"movi a4, ~3;" /* prepare a mask for the EPC */ * Move the aligned content of the exception addr to val
"and a4, a4, %2;" /* apply mask for 32bit aligned base */ */
"l32i a5, a4, 0;" /* load part 1 */ "rsr a6, EXCVADDR;" /* read out the faulting address */
"l32i a6, a4, 4;" /* load part 2 */ "movi a5, ~3;" /* prepare a mask for the EPC */
"ssa8l %2;" /* set up shift register for src op */ "and a5, a5, a6;" /* apply mask for 32bit aligned base */
"src %1, a6, a5;" /* right shift to get faulting instruction */ "l32i a5, a5, 0;" /* load aligned value */
:"=r"(excvaddr), "=r"(insn) "ssa8l a6;" /* set up shift register for value */
:"r"(epc1) "srl %[val], a5;" /* shift left to align value */
:"a4", "a5", "a6" /* we are done with a6 = EXCVADDR */
); /*
* Move the aligned instruction to insn
*/
"movi a5, ~3;" /* prepare a mask for the insn */
"and a6, a5, %[epc];" /* apply mask for 32bit aligned base */
"l32i a5, a6, 0;" /* load part 1 */
"l32i a6, a6, 4;" /* load part 2 */
"ssa8l %[epc];" /* set up shift register for src op */
"src %[op], a6, a5;" /* right shift to get faulting instruction */
:[val]"=r"(val), [op]"=r"(insn)
:[epc]"r"(ef->epc)
:"a5", "a6"
);
uint32_t valmask = 0; /* These instructions have the format 0xADSBII where AB = opcode and D = dest reg */
uint32_t what = insn & LOAD_MASK; uint32_t regno = (insn>>4)&0x0f; /* pick out nibble D*/
uint32_t opcode = (uint8_t) (((insn>>12)<<4)|(insn&0xf)); /* and nibbles AB */
#define L8UI 0x02u
#define L16UI 0x12u
#define L16SI 0x92u
if (what == L8UI_MATCH) if (opcode == L8UI) { /* L8UI */
valmask = 0xffu; val = (uint8_t) val;
else if (what == L16UI_MATCH || what == L16SI_MATCH) } else {
valmask = 0xffffu; val = (uint16_t) val; /* assume L16SI or L16UI */
else if (opcode == L16SI) {
{ val = (unsigned)((int)((sint16_t)val)); /* force signed 16->32 bit */
die: } else if (opcode != L16UI) {
/* Turns out we couldn't fix this, so try and chain to the handler /*
* that was set. (This is typically a remote GDB break). If none * Anything other than L8UI, L16SI or L16UI then chain to the next handler
* then trigger a system break instead and hang if the break doesn't * if set (typically a remote GDB break). Otherwise execute the default action
* get handled. This is effectively what would happen if the default * which is to trigger a system break and hang if the break doesn't get handled
* handler was installed. */ */
if (load_store_handler) { if (load_store_handler) {
load_store_handler(ef, cause); load_store_handler(NULL, 0 /* ef , cause */);
return; return;
} else {
asm ("break 1, 1");
while (1) {}
}
} }
asm ("break 1, 1"); }
while (1) {} ef->a_reg[regno ? regno-1: regno] = val; /* carry out the load */
} ef->epc += 3; /* resume at following instruction */
/* Load, shift and mask down to correct size */
uint32_t val = (*(uint32_t *)(excvaddr & ~0x3));
val >>= (excvaddr & 0x3) * 8;
val &= valmask;
/* Sign-extend for L16SI, if applicable */
if (what == L16SI_MATCH && (val & 0x8000))
val |= 0xffff0000;
int regno = (insn & 0x0000f0u) >> 4;
if (regno == 1)
goto die; /* we can't support loading into a1, just die */
else if (regno != 0)
--regno; /* account for skipped a1 in exception_frame */
ef->a_reg[regno] = val; /* carry out the load */
ef->epc += 3; /* resume at following instruction */
} }
/** /**
* The SDK's user_main function installs a debugging handler regardless * The SDK's user_main function installs a debugging handler regardless
* of whether there's a proper handler installed for EXCCAUSE_LOAD_STORE_ERROR, * of whether there's a proper handler installed for EXCCAUSE_LOAD_STORE_ERROR,
......
#############################################################
# Required variables for each makefile
# Discard this section from all parent makefiles
# Expected variables (with automatic defaults):
# CSRCS (all "C" files in the dir)
# SUBDIRS (all subdirs with a Makefile)
# GEN_LIBS - list of libs to be generated ()
# GEN_IMAGES - list of images to be generated ()
# COMPONENTS_xxx - a list of libs/objs in the form
# subdir/lib to be extracted and rolled up into
# a generated lib/image xxx.a ()
#
ifndef PDIR
GEN_LIBS = libuzlib.a
SUBDIRS = host
endif
#############################################################
# Configuration i.e. compile options etc.
# Target specific stuff (defines etc.) goes in here!
# Generally values applying to a tree are captured in the
# makefile at its root level - these are then overridden
# for a subtree within the makefile rooted therein
#
#DEFINES +=
#############################################################
# Recursion Magic - Don't touch this!!
#
# Each subtree potentially has an include directory
# corresponding to the common APIs applicable to modules
# rooted at that subtree. Accordingly, the INCLUDE PATH
# of a module can only contain the include directories up
# its parent path, and not its siblings
#
# Required for each makefile to inherit from the parent
#
INCLUDES := $(INCLUDES) -I $(PDIR)include
INCLUDES += -I ./
INCLUDES += -I ../libc
PDIR := ../$(PDIR)
sinclude $(PDIR)Makefile
uzlib - Deflate/Zlib-compatible LZ77 compression library
======================================================
This is a heavily modified and cut down version of Paul Sokolovsky's
uzlib library. This library has exported routines which
- Can compress data to a Deflate-compatible bitstream, albeit with lower
compression ratio than the Zlib Deflate algorithm as a static Deflate Huffman
tree encoding is used for bitstream). Note that since this compression is
in RAM and requires ~4 bytes per byte of the input record, should only be
called for compressing small records on the ESP8266.
- Can decompress any valid Deflate, Zlib, and Gzip (further called just
"Deflate") bitstream less than 16Kb, and any arbitrary length stream
compressed by the uzlib compressor.
uzlib aims for minimal code size and runtime memory requirements, and thus
is suitable for embedded systems and IoT devices such as the ESP8266.
uzlib is based on:
- tinf library by Joergen Ibsen (Deflate decompression)
- Deflate Static Huffman tree routines by Simon Tatham
- LZ77 compressor by Paul Sokolovsky provided my initial inspiration, but
I ended up rewriting this following RFC 1951 to get improved compression
performance.
The above 16Kb limitation arises from the RFC 1951 use of a 32Kb dictionary,
which is impractical on a chipset with only ~40 Kb RAM avialable to
applications.
The relevant copyright statements are provided in the source files which
use this code.
uzlib library is licensed under Zlib license.
/*
* CRC32 checksum
*
* Copyright (c) 1998-2003 by Joergen Ibsen / Jibz
* All Rights Reserved
*
* http://www.ibsensoftware.com/
*
* This software is provided 'as-is', without any express
* or implied warranty. In no event will the authors be
* held liable for any damages arising from the use of
* this software.
*
* Permission is granted to anyone to use this software
* for any purpose, including commercial applications,
* and to alter it and redistribute it freely, subject to
* the following restrictions:
*
* 1. The origin of this software must not be
* misrepresented; you must not claim that you
* wrote the original software. If you use this
* software in a product, an acknowledgment in
* the product documentation would be appreciated
* but is not required.
*
* 2. Altered source versions must be plainly marked
* as such, and must not be misrepresented as
* being the original software.
*
* 3. This notice may not be removed or altered from
* any source distribution.
*/
/*
* CRC32 algorithm taken from the zlib source, which is
* Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler
*/
#include <stdint.h>
static const unsigned int tinf_crc32tab[16] = {
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190,
0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320, 0xf00f9344,
0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278,
0xbdbdf21c
};
/* crc is previous value for incremental computation, 0xffffffff initially */
uint32_t uzlib_crc32(const void *data, unsigned int length, uint32_t crc)
{
const unsigned char *buf = (const unsigned char *)data;
unsigned int i;
for (i = 0; i < length; ++i)
{
crc ^= buf[i];
crc = tinf_crc32tab[crc & 0x0f] ^ (crc >> 4);
crc = tinf_crc32tab[crc & 0x0f] ^ (crc >> 4);
}
// return value suitable for passing in next time, for final value invert it
return crc/* ^ 0xffffffff*/;
}
#
# This Make file is called from the core Makefile hierarchy with is a hierarchical
# make wwhich uses parent callbacks to implement inheritance. However is luac_cross
# build stands outside this and uses the host toolchain to implement a separate
# host build of the luac.cross image.
#
.NOTPARALLEL:
CCFLAGS:= -I..
LDFLAGS:= -L$(SDK_DIR)/lib -L$(SDK_DIR)/ld -lm -ldl -Wl,-Map=mapfile
CCFLAGS += -Wall
#DEFINES +=
TARGET = host
ifeq ($(FLAVOR),debug)
CCFLAGS += -O0 -g
TARGET_LDFLAGS += -O0 -g
DEFINES += -DDEBUG_COUNTS
else
FLAVOR = release
CCFLAGS += -O2
TARGET_LDFLAGS += -O2
endif
#
# This relies on the files being unique on the vpath
#
SRC := uz_unzip.c uz_zip.c crc32.c uzlib_inflate.c uzlib_deflate.c
vpath %.c .:..
ODIR := .output/$(TARGET)/$(FLAVOR)/obj
CFLAGS = $(CCFLAGS) $(DEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES)
DFLAGS = $(CCFLAGS) $(DDEFINES) $(EXTRA_CCFLAGS) $(STD_CFLAGS) $(INCLUDES)
ROOT = ../../..
CC := gcc
ECHO := echo
IMAGES := $(ROOT)/uz_zip $(ROOT)/uz_unzip
.PHONY: test clean all
all: $(IMAGES)
$(ROOT)/uz_zip : $(ODIR)/uz_zip.o $(ODIR)/crc32.o $(ODIR)/uzlib_deflate.o
$(CC) $^ -o $@ $(LDFLAGS)
$(ROOT)/uz_unzip : $(ODIR)/uz_unzip.o $(ODIR)/crc32.o $(ODIR)/uzlib_inflate.o
$(CC) $^ -o $@ $(LDFLAGS)
test :
@echo CC: $(CC)
@echo SRC: $(SRC)
@echo DEPS: $(DEPS)
clean :
$(RM) -r $(ODIR)
$(RM) $(IMAGES)
$(ODIR)/%.o: %.c
@mkdir -p $(ODIR);
$(CC) $(CFLAGS) -o $@ -c $<
/************************************************************************
* NodeMCU unzip wrapper code for uzlib_inflate
*
* Note that whilst it would be more straightforward to implement a
* simple in memory approach, this utility adopts the same streaming
* callback architecture as app/lua/lflash.c to enable this code to be
* tested in a pure host development environment
*/
#include <string.h>
#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include "uzlib.h"
/* Test wrapper */
#define DICTIONARY_WINDOW 16384
#define READ_BLOCKSIZE 2048
#define WRITE_BLOCKSIZE 2048
#define WRITE_BLOCKS ((DICTIONARY_WINDOW/WRITE_BLOCKSIZE)+1)
#define FREE(v) if (v) uz_free(v)
typedef uint8_t uchar;
typedef uint16_t ushort;
typedef uint32_t uint;
struct INPUT {
FILE *fin;
int len;
uint8_t block[READ_BLOCKSIZE];
uint8_t *inPtr;
int bytesRead;
int left;
} *in;
typedef struct {
uint8_t byte[WRITE_BLOCKSIZE];
} outBlock;
struct OUTPUT {
FILE *fout;
outBlock *block[WRITE_BLOCKS];
int ndx;
int written;
int len;
uint32_t crc;
int breakNdx;
int (*fullBlkCB) (void);
} *out;
/*
* uzlib_inflate does a stream inflate on an RFC 1951 encoded data stream.
* It uses three application-specific CBs passed in the call to do the work:
*
* - get_byte() CB to return next byte in input stream
* - put_byte() CB to output byte to output buffer
* - recall_byte() CB to output byte to retrieve a historic byte from
* the output buffer.
*
* Note that put_byte() also triggers secondary CBs to do further processing.
*/
uint8_t get_byte (void) {
if (--in->left < 0) {
/* Read next input block */
int remaining = in->len - in->bytesRead;
int wanted = remaining >= READ_BLOCKSIZE ? READ_BLOCKSIZE : remaining;
if (fread(in->block, 1, wanted, in->fin) != wanted)
UZLIB_THROW(UZLIB_DATA_ERROR);
in->bytesRead += wanted;
in->inPtr = in->block;
in->left = wanted-1;
}
return *in->inPtr++;
}
void put_byte (uint8_t value) {
int offset = out->ndx % WRITE_BLOCKSIZE; /* counts from 0 */
out->block[0]->byte[offset++] = value;
out->ndx++;
if (offset == WRITE_BLOCKSIZE || out->ndx == out->len) {
if (out->fullBlkCB)
out->fullBlkCB();
/* circular shift the block pointers (redundant on last block, but so what) */
outBlock *nextBlock = out->block[WRITE_BLOCKS - 1];
memmove(out->block+1, out->block, (WRITE_BLOCKS-1)*sizeof(void*));
out->block[0] = nextBlock;
}
}
uint8_t recall_byte (uint offset) {
if(offset > DICTIONARY_WINDOW || offset >= out->ndx)
UZLIB_THROW(UZLIB_DICT_ERROR);
/* ndx starts at 1. Need relative to 0 */
uint n = out->ndx - offset;
uint pos = n % WRITE_BLOCKSIZE;
uint blockNo = out->ndx / WRITE_BLOCKSIZE - n / WRITE_BLOCKSIZE;
return out->block[blockNo]->byte[pos];
}
int processOutRec (void) {
int len = (out->ndx % WRITE_BLOCKSIZE) ? out->ndx % WRITE_BLOCKSIZE :
WRITE_BLOCKSIZE;
if (fwrite(out->block[0], 1, len, out->fout) != len)
UZLIB_THROW(UZLIB_DATA_ERROR);
out->crc = uzlib_crc32(out->block[0], len, out->crc);
out->written += len;
if (out->written == out->len) {
fclose(out->fout);
out->fullBlkCB = NULL;
}
return 1;
}
int main(int argc, char *argv[]) {
assert (argc==3);
const char *inFile = argv[1], *outFile = argv[2];
int i, n, res;
uint crc;
void *cxt_not_used;
assert(sizeof(unsigned int) == 4);
/* allocate and zero the in and out Blocks */
assert(in = uz_malloc(sizeof(*in)));
assert(out = uz_malloc(sizeof(*out)));
memset(in, 0, sizeof(*in));
memset(out, 0, sizeof(*out));
/* open input files and probe end to read the expanded length */
assert((in->fin = fopen(inFile, "rb")));
fseek(in->fin, -4, SEEK_END);
assert(fread((uchar*)&(out->len), 1, 4, in->fin) == 4);
in->len = ftell(in->fin);
fseek(in->fin, 0, SEEK_SET);
assert((out->fout = fopen(outFile, "wb")));
printf ("Inflating in=%s out=%s\n", inFile, outFile);
/* Allocate the out buffers (number depends on the unpacked length) */
n = (out->len > DICTIONARY_WINDOW) ? WRITE_BLOCKS :
1 + (out->len-1) / WRITE_BLOCKSIZE;
for(i = WRITE_BLOCKS - n + 1; i <= WRITE_BLOCKS; i++)
assert(out->block[i % WRITE_BLOCKS] = uz_malloc(sizeof(outBlock)));
out->breakNdx = (out->len < WRITE_BLOCKSIZE) ? out->len : WRITE_BLOCKSIZE;
out->fullBlkCB = processOutRec;
out->crc = ~0;
/* Call inflate to do the business */
res = uzlib_inflate (get_byte, put_byte, recall_byte, in->len, &crc, &cxt_not_used);
if (res > 0 && crc != ~out->crc)
res = UZLIB_CHKSUM_ERROR;
for (i = 0; i < WRITE_BLOCKS; i++)
FREE(out->block[i]);
fclose(in->fin);
FREE(in);
FREE(out);
if (res < 0)
printf("Error during decompression: %d\n", res);
return (res != 0) ? 1: 0;
}
/************************************************************************
* NodeMCU zip wrapper code for uzlib_compress
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <unistd.h>
#include "uzlib.h"
#define fwriterec(r) fwrite(&(r), sizeof(r), 1, fout);
#define BAD_FILE (-1)
int main (int argc, char **argv) {
const char *in = argv[1], *out = argv[2];
if (argc!=3)
return 1;
printf ("Compressing in=%s out=%s\n", in, out);
FILE *fin, *fout;
int status = -1;
uint32_t iLen, oLen;
uint8_t *iBuf, *oBuf;
if (!(fin = fopen(in, "rb")) || fseek(fin, 0, SEEK_END) ||
(iLen = ftell(fin)) <= 0 || fseek(fin, 0, SEEK_SET))
return 1;
if ((fout = fopen(out, "wb")) == NULL ||
(iBuf = (uint8_t *) uz_malloc(iLen)) == NULL ||
fread(iBuf, 1, iLen, fin) != iLen)
return 1;
if (uzlib_compress (&oBuf, &oLen, iBuf, iLen) == UZLIB_OK &&
oLen == fwrite(oBuf, oLen, 1, fout))
status = UZLIB_OK;
uz_free(iBuf);
if (oBuf) uz_free(oBuf);
fclose(fin);
fclose(fout);
if (status == UZLIB_OK)
unlink(out);
return (status == UZLIB_OK) ? 1: 0;
}
/*
* uzlib - tiny deflate/inflate library (deflate, gzip, zlib)
*
* Copyright (c) 2003 by Joergen Ibsen / Jibz
* All Rights Reserved
* http://www.ibsensoftware.com/
*
* Copyright (c) 2014-2016 by Paul Sokolovsky
*/
#ifndef UZLIB_INFLATE_H
#define UZLIB_INFLATE_H
#include <setjmp.h>
#if defined(__XTENSA__)
#include "c_stdint.h"
#include "mem.h"
#define UZLIB_THROW(v) longjmp(unwindAddr, (v))
#define UZLIB_SETJMP setjmp
#define uz_malloc os_malloc
#define uz_free os_free
#else /* POSIX */
#include <stdint.h>
#include <stdlib.h>
extern int dbg_break(void);
#define UZLIB_THROW(v) {dbg_break();_longjmp(unwindAddr, (v));}
#define UZLIB_SETJMP _setjmp
#define uz_malloc malloc
#define uz_free free
#endif
extern jmp_buf unwindAddr;
/* ok status, more data produced */
#define UZLIB_OK 0
/* end of compressed stream reached */
#define UZLIB_DONE 1
#define UZLIB_DATA_ERROR (-3)
#define UZLIB_CHKSUM_ERROR (-4)
#define UZLIB_DICT_ERROR (-5)
#define UZLIB_MEMORY_ERROR (-6)
/* checksum types */
#define UZLIB_CHKSUM_NONE 0
#define UZLIB_CHKSUM_ADLER 1
#define UZLIB_CHKSUM_CRC 2
/* Gzip header codes */
#define UZLIB_FTEXT 1
#define UZLIB_FHCRC 2
#define UZLIB_FEXTRA 4
#define UZLIB_FNAME 8
#define UZLIB_FCOMMENT 16
/* Compression API */
typedef struct uzlib_data UZLIB_DATA;
int uzlib_inflate (uint8_t (*)(void), void (*)(uint8_t),
uint8_t (*)(uint32_t), uint32_t len, uint32_t *crc, void **state);
int uzlib_compress (uint8_t **dest, uint32_t *destLen,
const uint8_t *src, uint32_t srcLen);
/* Checksum API */
/* crc is previous value for incremental computation, 0xffffffff initially */
uint32_t uzlib_crc32(const void *data, uint32_t length, uint32_t crc);
#endif /* UZLIB_INFLATE_H */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment