Commit a2ba49e3 authored by Johny Mattsson's avatar Johny Mattsson
Browse files

Switch to IDF-provided VFS and standard `io` module.

The IDF-provided VFS resolves several issues:

 - The IDF components having a different view of the (virtual) file system
   compared to the Lua environment.

 - RTOS task/thread safety. Our legacy VFS was only ever safe to use
   from the LVM thread, which limited its usability. Upgrading it
   would have effectively required a reimplementation of the IDF VFS,
   which would have been a bigger task with larger on-going maintenance
   issues.

 - We're no longer needing to maintain our own SPIFFS component.

 - We're no longer needing to maintain our own FATFS component.

 - The legacy of the 8266's lack of standard C interface to the file system
   is no longer holding us back, meaning that we can use the standard
   Lua `io` module rather than the cobbled-together swiss army knife
   also known as the file module.

Of course, the downside is that we'll either have to declare a backwards
breakage in regard to the file module, or provide a Lua shim for the old
functions, where applicable.

Also included is some necessary integer type fixups in unrelated code,
which apparently had depended on some non-standard types in either the
SPIFFS or FATFS headers.

A memory leak issue in the sdmmc module was also found and fixed while
said module got switched over to the Espressif VFS.

Module documentation has been updated to match the new reality (and I
discovered in some places it wasn't even matching the old reality).
parent 32f66e3d
......@@ -2,6 +2,6 @@
nvs, data, nvs, 0x9000, 0x6000
phy_init, data, phy, 0xf000, 0x1000
factory, app, factory, 0x10000, 0x180000
# Type 0xC2 => NodeMCU. SubTypes: 0x00 = SPIFFS, 0x01 = LFS
lfs, 0xC2, 0x01, , 0x10000
nodemcuspiffs, 0xC2, 0x00, , 0x70000
# Type 0xC2 => NodeMCU. SubTypes: 0x01 = LFS
lfs, 0xC2, 0x01, , 0x10000
storage, data, spiffs, , 0x70000
......@@ -146,7 +146,7 @@ uint32_t platform_s_flash_write( const void *from, uint32_t toaddr, uint32_t siz
return 0;
memcpy(apbuf, from, size);
}
r = flash_write(toaddr, apbuf?(uint32 *)apbuf:(uint32 *)from, size);
r = flash_write(toaddr, apbuf?(uint32_t *)apbuf:(uint32_t *)from, size);
if(apbuf)
free(apbuf);
if(ESP_OK == r)
......@@ -174,18 +174,18 @@ uint32_t platform_s_flash_read( void *to, uint32_t fromaddr, uint32_t size )
if( ((uint32_t)to) & blkmask )
{
uint32_t size2=size-INTERNAL_FLASH_READ_UNIT_SIZE;
uint32* to2=(uint32*)((((uint32_t)to)&(~blkmask))+INTERNAL_FLASH_READ_UNIT_SIZE);
uint32_t* to2=(uint32_t*)((((uint32_t)to)&(~blkmask))+INTERNAL_FLASH_READ_UNIT_SIZE);
r = flash_read(fromaddr, to2, size2);
if(ESP_OK == r)
{
memmove(to,to2,size2);
char back[ INTERNAL_FLASH_READ_UNIT_SIZE ] __attribute__ ((aligned(INTERNAL_FLASH_READ_UNIT_SIZE)));
r=flash_read(fromaddr+size2,(uint32*)back,INTERNAL_FLASH_READ_UNIT_SIZE);
r=flash_read(fromaddr+size2,(uint32_t*)back,INTERNAL_FLASH_READ_UNIT_SIZE);
memcpy((uint8_t*)to+size2,back,INTERNAL_FLASH_READ_UNIT_SIZE);
}
}
else
r = flash_read(fromaddr, (uint32 *)to, size);
r = flash_read(fromaddr, (uint32_t *)to, size);
if(ESP_OK == r)
return size;
......
#include <stdlib.h>
#include <stdio.h>
#include "vfs.h"
#include "platform.h"
#define LDRV_TRAVERSAL 0
// ---------------------------------------------------------------------------
// RTC system interface
//
static int32_t (*rtc_cb)( vfs_time *tm ) = NULL;
// called by operating system
void vfs_register_rtc_cb( int32_t (*cb)( vfs_time *tm ) )
{
// allow NULL pointer to unregister callback function
rtc_cb = cb;
}
// called by file system drivers
int32_t vfs_get_rtc( vfs_time *tm )
{
if (rtc_cb) {
return rtc_cb( tm );
}
return VFS_RES_ERR;
}
#if LDRV_TRAVERSAL
static int dir_level = 1;
#endif
static const char *normalize_path( const char *path )
{
#if ! LDRV_TRAVERSAL
return path;
#else
const char *temp = path;
size_t len;
while ((len = strlen( temp )) >= 2) {
if (temp[0] == '.' && temp[1] == '.') {
--dir_level;
if (len >= 4 && dir_level > 0) {
// prepare next step
temp = &(temp[4]);
} else {
// we're at top, the remainder is expected be an absolute path
temp = &(temp[3]);
}
} else {
break;
}
}
if (dir_level > 0) {
// no traversal on root level
return path;
} else {
// path traverses via root
return temp;
}
#endif
}
// ---------------------------------------------------------------------------
// file system functions
//
vfs_vol *vfs_mount( const char *name, int num )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->mount( outname, num );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
vfs_vol *r = fs_fns->mount( outname, num );
free( outname );
return r;
}
#endif
return NULL;
}
int vfs_open( const char *name, const char *mode )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return (int)fs_fns->open( outname, mode );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
int r = (int)fs_fns->open( outname, mode );
free( outname );
return r;
}
#endif
return 0;
}
vfs_dir *vfs_opendir( const char *name )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->opendir( outname );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
vfs_dir *r = fs_fns->opendir( outname );
free( outname );
return r;
}
#endif
return NULL;
}
int32_t vfs_stat( const char *name, struct vfs_stat *buf )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->stat( outname, buf );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
int32_t r = fs_fns->stat( outname, buf );
free( outname );
return r;
}
#endif
return VFS_RES_ERR;
}
int32_t vfs_remove( const char *name )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->remove( outname );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
int32_t r = fs_fns->remove( outname );
free( outname );
return r;
}
#endif
return VFS_RES_ERR;
}
int32_t vfs_rename( const char *oldname, const char *newname )
{
vfs_fs_fns *fs_fns;
const char *normoldname = normalize_path( oldname );
const char *normnewname = normalize_path( newname );
char *oldoutname, *newoutname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if (myspiffs_realm( normoldname, &oldoutname, false )) {
if ((fs_fns = myspiffs_realm( normnewname, &newoutname, false ))) {
return fs_fns->rename( oldoutname, newoutname );
}
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if (myfatfs_realm( normoldname, &oldoutname, false )) {
if ((fs_fns = myfatfs_realm( normnewname, &newoutname, false ))) {
int32_t r = fs_fns->rename( oldoutname, newoutname );
free( oldoutname );
free( newoutname );
return r;
}
free( oldoutname );
}
#endif
return -1;
}
int32_t vfs_mkdir( const char *name )
{
vfs_fs_fns *fs_fns;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
// not supported
(void)fs_fns;
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
const char *normname = normalize_path( name );
char *outname;
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
int32_t r = fs_fns->mkdir( outname );
free( outname );
return r;
}
#endif
return VFS_RES_ERR;
}
int32_t vfs_fsinfo( const char *name, uint32_t *total, uint32_t *used )
{
vfs_fs_fns *fs_fns;
char *outname;
if (!name) name = ""; // current drive
const char *normname = normalize_path( name );
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->fsinfo( total, used );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
free( outname );
return fs_fns->fsinfo( total, used );
}
#endif
return VFS_RES_ERR;
}
int32_t vfs_fscfg( const char *name, uint32_t *phys_addr, uint32_t *phys_size)
{
vfs_fs_fns *fs_fns;
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( "/FLASH", &outname, false ))) {
return fs_fns->fscfg( phys_addr, phys_size );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
// not supported
#endif
// Error
return VFS_RES_ERR;
}
int32_t vfs_format( void )
{
vfs_fs_fns *fs_fns;
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( "/FLASH", &outname, false ))) {
return fs_fns->format();
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
// not supported
#endif
// Error
return 0;
}
int32_t vfs_chdir( const char *path )
{
vfs_fs_fns *fs_fns;
const char *normpath = normalize_path( path );
char *outname;
int ok = VFS_RES_ERR;
#if LDRV_TRAVERSAL
const char *level;
// track dir level
if (normpath[0] == '/') {
dir_level = 0;
level = &(normpath[1]);
} else {
level = normpath;
}
while (strlen( level ) > 0) {
dir_level++;
if (level = strchr( level, '/' )) {
level++;
} else {
break;
}
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normpath, &outname, true ))) {
// our SPIFFS integration doesn't support directories
if (strlen( outname ) == 0) {
ok = VFS_RES_OK;
}
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normpath, &outname, true ))) {
if (strchr( outname, ':' )) {
// need to set FatFS' default drive
fs_fns->chdrive( outname );
// and force chdir to root in case path points only to root
fs_fns->chdir( "/" );
}
if (fs_fns->chdir( outname ) == VFS_RES_OK) {
ok = VFS_RES_OK;
}
free( outname );
}
#endif
return ok == VFS_RES_OK ? VFS_RES_OK : VFS_RES_ERR;
}
int32_t vfs_errno( const char *name )
{
vfs_fs_fns *fs_fns;
const char *normname = normalize_path( name );
char *outname;
if (!name) name = ""; // current drive
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
return fs_fns->ferrno( );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
int32_t r = fs_fns->ferrno( );
free( outname );
return r;
}
#endif
return VFS_RES_ERR;
}
int32_t vfs_ferrno( int fd )
{
vfs_file *f = (vfs_file *)fd;
if (f) {
return f->fns->ferrno ? f->fns->ferrno( f ) : 0;
} else {
vfs_fs_fns *fs_fns;
const char *name = ""; // current drive
char *outname;
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( name, &outname, false ))) {
return fs_fns->ferrno( );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( name, &outname, false ))) {
int32_t r = fs_fns->ferrno( );
free( outname );
return r;
}
#endif
}
return 0;
}
void vfs_clearerr( const char *name )
{
vfs_fs_fns *fs_fns;
char *outname;
if (!name) name = ""; // current drive
const char *normname = normalize_path( name );
#ifdef CONFIG_NODEMCU_BUILD_SPIFFS
if ((fs_fns = myspiffs_realm( normname, &outname, false ))) {
fs_fns->clearerr( );
}
#endif
#ifdef CONFIG_NODEMCU_BUILD_FATFS
if ((fs_fns = myfatfs_realm( normname, &outname, false ))) {
fs_fns->clearerr( );
free( outname );
}
#endif
}
const char *vfs_basename( const char *path )
{
const char *basename;
// deduce basename (incl. extension) for length check
if ((basename = strrchr( path, '/' ))) {
basename++;
} else if ((basename = strrchr( path, ':' ))) {
basename++;
} else {
basename = path;
}
return basename;
}
// ---------------------------------------------------------------------------
// supplementary functions
//
int vfs_getc( int fd )
{
unsigned char c = 0xFF;
if(!vfs_eof( fd )) {
if (1 != vfs_read( fd, &c, 1 )) {
NODE_DBG("getc errno %i\n", vfs_ferrno( fd ));
return VFS_EOF;
} else {
return (int)c;
}
}
return VFS_EOF;
}
int vfs_ungetc( int c, int fd )
{
return vfs_lseek( fd, -1, VFS_SEEK_CUR );
}
idf_component_register(
SRCS "spiffs.c" "spiffs_cache.c" "spiffs_check.c" "spiffs_gc.c"
"spiffs_hydrogen.c" "spiffs_nucleus.c"
INCLUDE_DIRS "."
REQUIRES "spi_flash"
PRIV_REQUIRES "platform"
)
target_compile_options(${COMPONENT_LIB} PRIVATE -Wno-error=pointer-sign )
The MIT License (MIT)
Copyright (c) 2013-2016 Peter Andersson (pelleplutt1976<at>gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
* QUICK AND DIRTY INTEGRATION EXAMPLE
So, assume you're running a Cortex-M3 board with a 2 MB SPI flash on it. The
SPI flash has 64kB blocks. Your project is built using gnumake, and now you
want to try things out.
First, you simply copy the files in src/ to your own source folder. Exclude
all files in test folder. Then you point out these files in your make script
for compilation.
Also copy the spiffs_config.h over from the src/default/ folder.
Try building. This fails, nagging about inclusions and u32_t and whatnot. Open
the spiffs_config.h and delete the bad inclusions. Also, add following
typedefs:
typedef signed int s32_t;
typedef unsigned int u32_t;
typedef signed short s16_t;
typedef unsigned short u16_t;
typedef signed char s8_t;
typedef unsigned char u8_t;
Now it should build. Over to the mounting business. Assume you already
implemented the read, write and erase functions to your SPI flash:
void my_spi_read(int addr, int size, char *buf)
void my_spi_write(int addr, int size, char *buf)
void my_spi_erase(int addr, int size)
In your main.c or similar, include the spiffs.h and do that spiffs struct:
#include <spiffs.h>
static spiffs fs;
Also, toss up some of the needed buffers:
#define LOG_PAGE_SIZE 256
static u8_t spiffs_work_buf[LOG_PAGE_SIZE*2];
static u8_t spiffs_fds[32*4];
static u8_t spiffs_cache_buf[(LOG_PAGE_SIZE+32)*4];
Now, write the my_spiffs_mount function:
void my_spiffs_mount() {
spiffs_config cfg;
cfg.phys_size = 2*1024*1024; // use all spi flash
cfg.phys_addr = 0; // start spiffs at start of spi flash
cfg.phys_erase_block = 65536; // according to datasheet
cfg.log_block_size = 65536; // let us not complicate things
cfg.log_page_size = LOG_PAGE_SIZE; // as we said
cfg.hal_read_f = my_spi_read;
cfg.hal_write_f = my_spi_write;
cfg.hal_erase_f = my_spi_erase;
int res = SPIFFS_mount(&fs,
&cfg,
spiffs_work_buf,
spiffs_fds,
sizeof(spiffs_fds),
spiffs_cache_buf,
sizeof(spiffs_cache_buf),
0);
printf("mount res: %i\n", res);
}
Now, build warns about the my_spi_read, write and erase functions. Wrong
signatures, so go wrap them:
static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) {
my_spi_read(addr, size, dst);
return SPIFFS_OK;
}
static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) {
my_spi_write(addr, size, dst);
return SPIFFS_OK;
}
static s32_t my_spiffs_erase(u32_t addr, u32_t size) {
my_spi_erase(addr, size);
return SPIFFS_OK;
}
Redirect the config in my_spiffs_mount to the wrappers instead:
cfg.hal_read_f = my_spiffs_read;
cfg.hal_write_f = my_spiffs_write;
cfg.hal_erase_f = my_spiffs_erase;
Ok, now you should be able to build and run. However, you get this output:
mount res: -1
but you wanted
mount res: 0
This is probably due to you having experimented with your SPI flash, so it
contains rubbish from spiffs's point of view. Do a mass erase and run again.
If all is ok now, you're good to go. Try creating a file and read it back:
static void test_spiffs() {
char buf[12];
// Surely, I've mounted spiffs before entering here
spiffs_file fd = SPIFFS_open(&fs, "my_file", SPIFFS_CREAT | SPIFFS_TRUNC | SPIFFS_RDWR, 0);
if (SPIFFS_write(&fs, fd, (u8_t *)"Hello world", 12) < 0) printf("errno %i\n", SPIFFS_errno(&fs));
SPIFFS_close(&fs, fd);
fd = SPIFFS_open(&fs, "my_file", SPIFFS_RDWR, 0);
if (SPIFFS_read(&fs, fd, (u8_t *)buf, 12) < 0) printf("errno %i\n", SPIFFS_errno(&fs));
SPIFFS_close(&fs, fd);
printf("--> %s <--\n", buf);
}
Compile, run, cross fingers hard, and you'll get the output:
--> Hello world <--
Got errors? Check spiffs.h for error definitions to get a clue what went voodoo.
* THINGS TO CHECK
When you alter the spiffs_config values, make sure you also check the typedefs
in spiffs_config.h:
- spiffs_block_ix
- spiffs_page_ix
- spiffs_obj_id
- spiffs_span_ix
The sizes of these typedefs must not underflow, else spiffs might end up in
eternal loops. Each typedef is commented what check for.
Also, if you alter the code or just want to verify your configuration, you can
run
> make test
in the spiffs folder. This will run all testcases using the configuration in
default/spiffs_config.h and test/params_test.h. The tests are written for linux
but should run under cygwin also.
* INTEGRATING SPIFFS
In order to integrate spiffs to your embedded target, you will basically need:
- A SPI flash device which your processor can communicate with
- An implementation for reading, writing and erasing the flash
- Memory (flash or ram) for the code
- Memory (ram) for the stack
Other stuff may be needed, threaded systems might need mutexes and so on.
** Logical structure
First and foremost, one must decide how to divide up the SPI flash for spiffs.
Having the datasheet for the actual SPI flash in hand will help. Spiffs can be
defined to use all or only parts of the SPI flash.
If following seems arcane, read the "HOW TO CONFIG" chapter first.
- Decide the logical size of blocks. This must be a multiple of the biggest
physical SPI flash block size. To go safe, use the physical block size -
which in many cases is 65536 bytes.
- Decide the logical size of pages. This must be a 2nd logarithm part of the
logical block size. To go safe, use 256 bytes to start with.
- Decide how much of the SPI flash memory to be used for spiffs. This must be
on logical block boundary. If unsafe, use 1 megabyte to start with.
- Decide where on the SPI flash memory the spiffs area should start. This must
be on physical block/sector boundary. If unsafe, use address 0.
** SPI flash API
The target must provide three functions to spiffs:
- s32_t (*spiffs_read)(u32_t addr, u32_t size, u8_t *dst)
- s32_t (*spiffs_write)(u32_t addr, u32_t size, u8_t *src)
- s32_t (*spiffs_erase)(u32_t addr, u32_t size)
These functions define the only communication between the SPI flash and the
spiffs stack.
On success these must return 0 (or SPIFFS_OK). Anything else will be considered
an error.
The size for read and write requests will never exceed the logical page size,
but it may be less.
The address and size on erase requests will always be on physical block size
boundaries.
** Mount specification
In spiffs.h, there is a SPIFFS_mount function defined, used to mount spiffs on
the SPI flash.
s32_t SPIFFS_mount(
spiffs *fs,
spiffs_config *config,
u8_t *work,
u8_t *fd_space,
u32_t fd_space_size,
void *cache,
u32_t cache_size,
spiffs_check_callback check_cb_f)
- fs Points to a spiffs struct. This may be totally uninitialized.
- config Points to a spiffs_config struct. This struct must be
initialized when mounting. See below.
- work A ram memory buffer being double the size of the logical page
size. This buffer is used excessively by the spiffs stack. If
logical page size is 256, this buffer must be 512 bytes.
- fd_space A ram memory buffer used for file descriptors.
- fd_space_size The size of the file descriptor buffer. A file descriptor
normally is around 32 bytes depending on the build config -
the bigger the buffer, the more file descriptors are
available.
- cache A ram memory buffer used for cache. Ignored if cache is
disabled in build config.
- cache_size The size of the cache buffer. Ignored if cache is disabled in
build config. One cache page will be slightly larger than the
logical page size. The more ram, the more cache pages, the
quicker the system.
- check_cb_f Callback function for monitoring spiffs consistency checks and
mending operations. May be null.
The config struct must be initialized prior to mounting. One must always
define the SPI flash access functions:
spiffs_config.hal_read_f - pointing to the function reading the SPI flash
spiffs_config.hal_write_f - pointing to the function writing the SPI flash
spiffs_config.hal_erase_f - pointing to the function erasing the SPI flash
Depending on the build config - if SPIFFS_SINGLETON is set to zero - following
parameters must be defined:
spiffs_config.phys_size - the physical number of bytes accounted for
spiffs on the SPI flash
spiffs_config.phys_addr - the physical starting address on the SPI flash
spiffs_config.phys_erase_block - the physical size of the largest block/sector
on the SPI flash found within the spiffs
usage address space
spiffs_config.log_block_size - the logical size of a spiffs block
spiffs_config.log_page_size - the logical size of a spiffs page
If SPIFFS_SINGLETON is set to one, above parameters must be set ny defines in
the config header file, spiffs_config.h.
** Build config
makefile: The files needed to be compiled to your target resides in files.mk to
be included in your makefile, either by cut and paste or by inclusion.
Types: spiffs uses the types u8_t, s8_t, u16_t, s16_t, u32_t, s32_t; these must
be typedeffed.
spiffs_config.h: you also need to define a spiffs_config.h header. Example of
this is found in the default/ directory.
** RAM
Spiffs needs ram. It needs a working buffer being double the size of the
logical page size. It also needs at least one file descriptor. If cache is
enabled (highly recommended), it will also need a bunch of cache pages.
Say you have a logical page size of 256 bytes. You want to be able to have four
files open simultaneously, and you can give spiffs four cache pages. This
roughly sums up to:
256*2 (work buffer) +
32*4 (file descriptors) +
(256+32)*4 (cache pages) + 40 (cache metadata)
i.e. 1832 bytes.
This is apart from call stack usage.
To get the exact amount of bytes needed on your specific target, enable
SPIFFS_BUFFER_HELP in spiffs_config.h, rebuild and call:
SPIFFS_buffer_bytes_for_filedescs
SPIFFS_buffer_bytes_for_cache
Having these figures you can disable SPIFFS_BUFFER_HELP again to save flash.
* HOW TO CONFIG
TODO
\ No newline at end of file
* USING SPIFFS
TODO
* SPIFFS DESIGN
Spiffs is inspired by YAFFS. However, YAFFS is designed for NAND flashes, and
for bigger targets with much more ram. Nevertheless, many wise thoughts have
been borrowed from YAFFS when writing spiffs. Kudos!
The main complication writing spiffs was that it cannot be assumed the target
has a heap. Spiffs must go along only with the work ram buffer given to it.
This forces extra implementation on many areas of spiffs.
** SPI flash devices using NOR technology
Below is a small description of how SPI flashes work internally. This is to
give an understanding of the design choices made in spiffs.
SPI flash devices are physically divided in blocks. On some SPI flash devices,
blocks are further divided into sectors. Datasheets sometimes name blocks as
sectors and vice versa.
Common memory capacaties for SPI flashes are 512kB up to 8MB of data, where
blocks may be 64kB. Sectors can be e.g. 4kB, if supported. Many SPI flashes
have uniform block sizes, whereas others have non-uniform - the latter meaning
that e.g. the first 16 blocks are 4kB big, and the rest are 64kB.
The entire memory is linear and can be read and written in random access.
Erasing can only be done block- or sectorwise; or by mass erase.
SPI flashes can normally be erased from 100.000 up to 1.000.000 cycles before
they fail.
A clean SPI flash from factory have all bits in entire memory set to one. A
mass erase will reset the device to this state. Block or sector erasing will
put the all bits in the area given by the sector or block to ones. Writing to a
NOR flash pulls ones to zeroes. Writing 0xFF to an address is simply a no-op.
Writing 0b10101010 to a flash address holding 0b00001111 will yield 0b00001010.
This way of "write by nand" is used considerably in spiffs.
Common characteristics of NOR flashes are quick reads, but slow writes.
And finally, unlike NAND flashes, NOR flashes seem to not need any error
correction. They always write correctly I gather.
** Spiffs logical structure
Some terminology before proceeding. Physical blocks/sectors means sizes stated
in the datasheet. Logical blocks and pages is something the integrator choose.
** Blocks and pages
Spiffs is allocated to a part or all of the memory of the SPI flash device.
This area is divided into logical blocks, which in turn are divided into
logical pages. The boundary of a logical block must coincide with one or more
physical blocks. The sizes for logical blocks and logical pages always remain
the same, they are uniform.
Example: non-uniform flash mapped to spiffs with 128kB logical blocks
PHYSICAL FLASH BLOCKS SPIFFS LOGICAL BLOCKS: 128kB
+-----------------------+ - - - +-----------------------+
| Block 1 : 16kB | | Block 1 : 128kB |
+-----------------------+ | |
| Block 2 : 16kB | | |
+-----------------------+ | |
| Block 3 : 16kB | | |
+-----------------------+ | |
| Block 4 : 16kB | | |
+-----------------------+ | |
| Block 5 : 64kB | | |
+-----------------------+ - - - +-----------------------+
| Block 6 : 64kB | | Block 2 : 128kB |
+-----------------------+ | |
| Block 7 : 64kB | | |
+-----------------------+ - - - +-----------------------+
| Block 8 : 64kB | | Block 3 : 128kB |
+-----------------------+ | |
| Block 9 : 64kB | | |
+-----------------------+ - - - +-----------------------+
| ... | | ... |
A logical block is divided further into a number of logical pages. A page
defines the smallest data holding element known to spiffs. Hence, if a file
is created being one byte big, it will occupy one page for index and one page
for data - it will occupy 2 x size of a logical page on flash.
So it seems it is good to select a small page size.
Each page has a metadata header being normally 5 to 9 bytes. This said, a very
small page size will make metadata occupy a lot of the memory on the flash. A
page size of 64 bytes will waste 8-14% on metadata, while 256 bytes 2-4%.
So it seems it is good to select a big page size.
Also, spiffs uses a ram buffer being two times the page size. This ram buffer
is used for loading and manipulating pages, but it is also used for algorithms
to find free file ids, scanning the file system, etc. Having too small a page
size means less work buffer for spiffs, ending up in more reads operations and
eventually gives a slower file system.
Choosing the page size for the system involves many factors:
- How big is the logical block size
- What is the normal size of most files
- How much ram can be spent
- How much data (vs metadata) must be crammed into the file system
- How fast must spiffs be
- Other things impossible to find out
So, chosing the Optimal Page Size (tm) seems tricky, to say the least. Don't
fret - there is no optimal page size. This varies from how the target will use
spiffs. Use the golden rule:
~~~ Logical Page Size = Logical Block Size / 256 ~~~
This is a good starting point. The final page size can then be derived through
heuristical experimenting for us non-analytical minds.
** Objects, indices and look-ups
A file, or an object as called in spiffs, is identified by an object id.
Another YAFFS rip-off. This object id is a part of the page header. So, all
pages know to which object/file they belong - not counting the free pages.
An object is made up of two types of pages: object index pages and data pages.
Data pages contain the data written by user. Index pages contain metadata about
the object, more specifically what data pages are part of the object.
The page header also includes something called a span index. Let's say a file
is written covering three data pages. The first data page will then have span
index 0, the second span index 1, and the last data page will have span index
2. Simple as that.
Finally, each page header contain flags, telling if the page is used,
deleted, finalized, holds index or data, and more.
Object indices also have span indices, where an object index with span index 0
is referred to as the object index header. This page does not only contain
references to data pages, but also extra info such as object name, object size
in bytes, flags for file or directory, etc.
If one were to create a file covering three data pages, named e.g.
"spandex-joke.txt", given object id 12, it could look like this:
PAGE 0 <things to be unveiled soon>
PAGE 1 page header: [obj_id:12 span_ix:0 flags:USED|DATA]
<first data page of joke>
PAGE 2 page header: [obj_id:12 span_ix:1 flags:USED|DATA]
<second data page of joke>
PAGE 3 page header: [obj_id:545 span_ix:13 flags:USED|DATA]
<some data belonging to object 545, probably not very amusing>
PAGE 4 page header: [obj_id:12 span_ix:2 flags:USED|DATA]
<third data page of joke>
PAGE 5 page header: [obj_id:12 span_ix:0 flags:USED|INDEX]
obj ix header: [name:spandex-joke.txt size:600 bytes flags:FILE]
obj ix: [1 2 4]
Looking in detail at page 5, the object index header page, the object index
array refers to each data page in order, as mentioned before. The index of the
object index array correlates with the data page span index.
entry ix: 0 1 2
obj ix: [1 2 4]
| | |
PAGE 1, DATA, SPAN_IX 0 --------/ | |
PAGE 2, DATA, SPAN_IX 1 --------/ |
PAGE 4, DATA, SPAN_IX 2 --------/
Things to be unveiled in page 0 - well.. Spiffs is designed for systems low on
ram. We cannot keep a dynamic list on the whereabouts of each object index
header so we can find a file fast. There might not even be a heap! But, we do
not want to scan all page headers on the flash to find the object index header.
The first page(s) of each block contains the so called object look-up. These
are not normal pages, they do not have a header. Instead, they are arrays
pointing out what object-id the rest of all pages in the block belongs to.
By this look-up, only the first page(s) in each block must to scanned to find
the actual page which contains the object index header of the desired object.
The object lookup is redundant metadata. The assumption is that it presents
less overhead reading a full page of data to memory from each block and search
that, instead of reading a small amount of data from each page (i.e. the page
header) in all blocks. Each read operation from SPI flash normally contains
extra data as the read command itself and the flash address. Also, depending on
the underlying implementation, other criterions may need to be passed for each
read transaction, like mutexes and such.
The veiled example unveiled would look like this, with some extra pages:
PAGE 0 [ 12 12 545 12 12 34 34 4 0 0 0 0 ...]
PAGE 1 page header: [obj_id:12 span_ix:0 flags:USED|DATA] ...
PAGE 2 page header: [obj_id:12 span_ix:1 flags:USED|DATA] ...
PAGE 3 page header: [obj_id:545 span_ix:13 flags:USED|DATA] ...
PAGE 4 page header: [obj_id:12 span_ix:2 flags:USED|DATA] ...
PAGE 5 page header: [obj_id:12 span_ix:0 flags:USED|INDEX] ...
PAGE 6 page header: [obj_id:34 span_ix:0 flags:USED|DATA] ...
PAGE 7 page header: [obj_id:34 span_ix:1 flags:USED|DATA] ...
PAGE 8 page header: [obj_id:4 span_ix:1 flags:USED|INDEX] ...
PAGE 9 page header: [obj_id:23 span_ix:0 flags:DELETED|INDEX] ...
PAGE 10 page header: [obj_id:23 span_ix:0 flags:DELETED|DATA] ...
PAGE 11 page header: [obj_id:23 span_ix:1 flags:DELETED|DATA] ...
PAGE 12 page header: [obj_id:23 span_ix:2 flags:DELETED|DATA] ...
...
Ok, so why are page 9 to 12 marked as 0 when they belong to object id 23? These
pages are deleted, so this is marked both in page header flags and in the look
up. This is an example where spiffs uses NOR flashes "nand-way" of writing.
As a matter of fact, there are two object id's which are special:
obj id 0 (all bits zeroes) - indicates a deleted page in object look up
obj id 0xff.. (all bits ones) - indicates a free page in object look up
Actually, the object id's have another quirk: if the most significant bit is
set, this indicates an object index page. If the most significant bit is zero,
this indicates a data page. So to be fully correct, page 0 in above example
would look like this:
PAGE 0 [ 12 12 545 12 *12 34 34 *4 0 0 0 0 ...]
where the asterisk means the msb of the object id is set.
This is another way to speed up the searches when looking for object indices.
By looking on the object id's msb in the object lookup, it is also possible
to find out whether the page is an object index page or a data page.
* When mending lost pages, also see if they fit into length specified in object index header
SPIFFS2 thoughts
* Instead of exact object id:s in the object lookup tables, use a hash of span index and object id.
Eg. object id xor:ed with bit-reversed span index.
This should decrease number of actual pages that needs to be visited when looking thru the obj lut.
* Logical number of each block. When moving stuff in a garbage collected page, the free
page is assigned the same number as the garbage collected. Thus, object index pages do not have to
be rewritten.
* Steal one page, use as a bit parity page. When starting an fs modification operation, write one bit
as zero. When ending, write another bit as zero. On mount, if number of zeroes in page is uneven, a
check is automatically run.
\ No newline at end of file
#ifndef _NODEMCU_SPIFFS_H
#define _NODEMCU_SPIFFS_H
#ifndef NODEMCU_SPIFFS_NO_INCLUDE
#include <stdint.h>
#include <stddef.h>
#include <stdio.h>
#endif
// Turn off stats
#define SPIFFS_CACHE_STATS 0
#define SPIFFS_GC_STATS 0
// Needs to align stuff
#define SPIFFS_ALIGNED_OBJECT_INDEX_TABLES 1
// Enable magic so we can find the file system (but not yet)
#define SPIFFS_USE_MAGIC 1
#define SPIFFS_USE_MAGIC_LENGTH 1
// Reduce the chance of returning disk full
#define SPIFFS_GC_MAX_RUNS 256
#endif
foreach(def
"-Du32_t=uint32_t"
"-Du16_t=uint16_t"
"-Du8_t=uint8_t"
"-Ds32_t=int32_t"
"-Ds16_t=int16_t"
"-Duint32=uint32_t"
"-Duint16=uint16_t"
"-Duint8=uint8_t"
"-Dsint32=int32_t"
"-Dsint16=int16_t"
"-Dsint8=int8_t"
)
idf_build_set_property(COMPILE_DEFINITIONS ${def} APPEND)
endforeach()
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment