Commit b09cac05 authored by Philip Gladstone's avatar Philip Gladstone Committed by Johny Mattsson
Browse files

Add support for streaming JSON encoder/decoder (#1755)

Replaces the problematic cjson module.
parent b6ef1ffe
...@@ -38,7 +38,6 @@ SUBDIRS= \ ...@@ -38,7 +38,6 @@ SUBDIRS= \
smart \ smart \
modules \ modules \
spiffs \ spiffs \
cjson \
crypto \ crypto \
dhtlib \ dhtlib \
tsl2561 \ tsl2561 \
...@@ -87,7 +86,6 @@ COMPONENTS_eagle.app.v6 = \ ...@@ -87,7 +86,6 @@ COMPONENTS_eagle.app.v6 = \
smart/smart.a \ smart/smart.a \
spiffs/spiffs.a \ spiffs/spiffs.a \
fatfs/libfatfs.a \ fatfs/libfatfs.a \
cjson/libcjson.a \
crypto/libcrypto.a \ crypto/libcrypto.a \
dhtlib/libdhtlib.a \ dhtlib/libdhtlib.a \
tsl2561/tsl2561lib.a \ tsl2561/tsl2561lib.a \
......
# If Lua is installed in a non-standard location, please set the LUA_DIR
# environment variable to point to prefix for the install. Eg:
# Unix: export LUA_DIR=/home/user/pkg
# Windows: set LUA_DIR=c:\lua51
project(lua-cjson C)
cmake_minimum_required(VERSION 2.6)
option(USE_INTERNAL_FPCONV "Use internal strtod() / g_fmt() code for performance")
option(MULTIPLE_THREADS "Support multi-threaded apps with internal fpconv - recommended" ON)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING
"Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
find_package(Lua51 REQUIRED)
include_directories(${LUA_INCLUDE_DIR})
if(NOT USE_INTERNAL_FPCONV)
# Use libc number conversion routines (strtod(), sprintf())
set(FPCONV_SOURCES fpconv.c)
else()
# Use internal number conversion routines
add_definitions(-DUSE_INTERNAL_FPCONV)
set(FPCONV_SOURCES g_fmt.c dtoa.c)
include(TestBigEndian)
TEST_BIG_ENDIAN(IEEE_BIG_ENDIAN)
if(IEEE_BIG_ENDIAN)
add_definitions(-DIEEE_BIG_ENDIAN)
endif()
if(MULTIPLE_THREADS)
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
find_package(Threads REQUIRED)
if(NOT CMAKE_USE_PTHREADS_INIT)
message(FATAL_ERROR
"Pthreads not found - required by MULTIPLE_THREADS option")
endif()
add_definitions(-DMULTIPLE_THREADS)
endif()
endif()
# Handle platforms missing isinf() macro (Eg, some Solaris systems).
include(CheckSymbolExists)
CHECK_SYMBOL_EXISTS(isinf math.h HAVE_ISINF)
if(NOT HAVE_ISINF)
add_definitions(-DUSE_INTERNAL_ISINF)
endif()
set(_MODULE_LINK "${CMAKE_THREAD_LIBS_INIT}")
get_filename_component(_lua_lib_dir ${LUA_LIBRARY} PATH)
if(APPLE)
set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS
"${CMAKE_SHARED_MODULE_CREATE_C_FLAGS} -undefined dynamic_lookup")
endif()
if(WIN32)
# Win32 modules need to be linked to the Lua library.
set(_MODULE_LINK ${LUA_LIBRARY} ${_MODULE_LINK})
set(_lua_module_dir "${_lua_lib_dir}")
# Windows sprintf()/strtod() handle NaN/inf differently. Not supported.
add_definitions(-DDISABLE_INVALID_NUMBERS)
else()
set(_lua_module_dir "${_lua_lib_dir}/lua/5.1")
endif()
add_library(cjson MODULE lua_cjson.c strbuf.c ${FPCONV_SOURCES})
set_target_properties(cjson PROPERTIES PREFIX "")
target_link_libraries(cjson ${_MODULE_LINK})
install(TARGETS cjson DESTINATION "${_lua_module_dir}")
# vi:ai et sw=4 ts=4:
Copyright (c) 2010-2012 Mark Pulford <mark@kyne.com.au>
2015 Zeroday Hong <zeroday@nodemcu.com> nodemcu.com
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#############################################################
# Required variables for each makefile
# Discard this section from all parent makefiles
# Expected variables (with automatic defaults):
# CSRCS (all "C" files in the dir)
# SUBDIRS (all subdirs with a Makefile)
# GEN_LIBS - list of libs to be generated ()
# GEN_IMAGES - list of images to be generated ()
# COMPONENTS_xxx - a list of libs/objs in the form
# subdir/lib to be extracted and rolled up into
# a generated lib/image xxx.a ()
#
ifndef PDIR
GEN_LIBS = libcjson.a
endif
#############################################################
# Configuration i.e. compile options etc.
# Target specific stuff (defines etc.) goes in here!
# Generally values applying to a tree are captured in the
# makefile at its root level - these are then overridden
# for a subtree within the makefile rooted therein
#
#DEFINES +=
#############################################################
# Recursion Magic - Don't touch this!!
#
# Each subtree potentially has an include directory
# corresponding to the common APIs applicable to modules
# rooted at that subtree. Accordingly, the INCLUDE PATH
# of a module can only contain the include directories up
# its parent path, and not its siblings
#
# Required for each makefile to inherit from the parent
#
INCLUDES := $(INCLUDES) -I $(PDIR)include
INCLUDES += -I ./
INCLUDES += -I ../libc
PDIR := ../$(PDIR)
sinclude $(PDIR)Makefile
The following people have helped with bug reports, testing and/or
suggestions:
- Louis-Philippe Perron (@loopole)
- Ondřej Jirman
- Steve Donovan <steve.j.donovan@gmail.com>
- Zhang "agentzh" Yichun <agentzh@gmail.com>
Thanks!
#include "cjson_mem.h"
#include "../lua/lauxlib.h"
#include <c_stdlib.h>
static const char errfmt[] = "cjson %salloc: out of mem (%d bytes)";
void *cjson_mem_malloc (uint32_t sz)
{
void *p = (void*)c_malloc (sz);
lua_State *L = lua_getstate();
if (!p)
luaL_error (L, errfmt, "m", sz);
return p;
}
void *cjson_mem_realloc (void *o, uint32_t sz)
{
void *p = (void*)c_realloc (o, sz);
lua_State *L = lua_getstate();
if (!p)
luaL_error (L, errfmt, "re", sz);
return p;
}
#ifndef _CJSON_MEM_H_
#define _CJSON_MEM_H_
#include "../lua/lua.h"
void *cjson_mem_malloc (uint32_t sz);
void *cjson_mem_realloc (void *p, uint32_t sz);
#endif
parser:
- call parse_value
- next_token
? <EOF> nop.
parse_value:
- next_token
? <OBJ_BEGIN> call parse_object.
? <ARR_BEGIN> call parse_array.
? <STRING> push. return.
? <BOOLEAN> push. return.
? <NULL> push. return.
? <NUMBER> push. return.
parse_object:
- push table
- next_token
? <STRING> push.
- next_token
? <COLON> nop.
- call parse_value
- set table
- next_token
? <OBJ_END> return.
? <COMMA> loop parse_object.
parse_array:
- push table
- call parse_value
- table append
- next_token
? <COMMA> loop parse_array.
? ] return.
next_token:
- check next character
? { return <OBJ_BEGIN>
? } return <OBJ_END>
? [ return <ARR_BEGIN>
? ] return <ARR_END>
? , return <COMMA>
? : return <COLON>
? [-0-9] gobble number. return <NUMBER>
? " gobble string. return <STRING>
? [ \t\n] eat whitespace.
? n Check "null". return <NULL> or <UNKNOWN>
? t Check "true". return <BOOLEAN> or <UNKNOWN>
? f Check "false". return <BOOLEAN> or <UNKNOWN>
? . return <UNKNOWN>
? \0 return <END>
This diff is collapsed.
#ifndef _DTOA_CONFIG_H
#define _DTOA_CONFIG_H
#if 0
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
/* Ensure dtoa.c does not USE_LOCALE. Lua CJSON must not use locale
* aware conversion routines. */
#undef USE_LOCALE
/* dtoa.c should not touch errno, Lua CJSON does not use it, and it
* may not be threadsafe */
#define NO_ERRNO
#define Long int32_t
#define ULong uint32_t
#define Llong int64_t
#define ULLong uint64_t
#ifdef IEEE_BIG_ENDIAN
#define IEEE_MC68k
#else
#define IEEE_8087
#endif
#define MALLOC(n) xmalloc(n)
static void *xmalloc(size_t size)
{
void *p;
p = malloc(size);
if (!p) {
fprintf(stderr, "Out of memory");
abort();
}
return p;
}
#ifdef MULTIPLE_THREADS
/* Enable locking to support multi-threaded applications */
#include <pthread.h>
static pthread_mutex_t private_dtoa_lock[2] = {
PTHREAD_MUTEX_INITIALIZER,
PTHREAD_MUTEX_INITIALIZER
};
#define ACQUIRE_DTOA_LOCK(n) do { \
int r = pthread_mutex_lock(&private_dtoa_lock[n]); \
if (r) { \
fprintf(stderr, "pthread_mutex_lock failed with %d\n", r); \
abort(); \
} \
} while (0)
#define FREE_DTOA_LOCK(n) do { \
int r = pthread_mutex_unlock(&private_dtoa_lock[n]); \
if (r) { \
fprintf(stderr, "pthread_mutex_unlock failed with %d\n", r);\
abort(); \
} \
} while (0)
#endif /* MULTIPLE_THREADS */
#endif
#endif /* _DTOA_CONFIG_H */
/* vi:ai et sw=4 ts=4:
*/
/* fpconv - Floating point conversion routines
*
* Copyright (c) 2011-2012 Mark Pulford <mark@kyne.com.au>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
/* JSON uses a '.' decimal separator. strtod() / sprintf() under C libraries
* with locale support will break when the decimal separator is a comma.
*
* fpconv_* will around these issues with a translation buffer if required.
*/
#include "c_stdio.h"
#include "c_stdlib.h"
// #include <assert.h>
#include "c_string.h"
#include "fpconv.h"
#if 0
/* Lua CJSON assumes the locale is the same for all threads within a
* process and doesn't change after initialisation.
*
* This avoids the need for per thread storage or expensive checks
* for call. */
static char locale_decimal_point = '.';
/* In theory multibyte decimal_points are possible, but
* Lua CJSON only supports UTF-8 and known locales only have
* single byte decimal points ([.,]).
*
* localconv() may not be thread safe (=>crash), and nl_langinfo() is
* not supported on some platforms. Use sprintf() instead - if the
* locale does change, at least Lua CJSON won't crash. */
static void fpconv_update_locale()
{
char buf[8];
c_sprintf(buf, "%g", 0.5);
/* Failing this test might imply the platform has a buggy dtoa
* implementation or wide characters */
if (buf[0] != '0' || buf[2] != '5' || buf[3] != 0) {
NODE_ERR("Error: wide characters found or printf() bug.");
return;
}
locale_decimal_point = buf[1];
}
/* Check for a valid number character: [-+0-9a-yA-Y.]
* Eg: -0.6e+5, infinity, 0xF0.F0pF0
*
* Used to find the probable end of a number. It doesn't matter if
* invalid characters are counted - strtod() will find the valid
* number if it exists. The risk is that slightly more memory might
* be allocated before a parse error occurs. */
static inline int valid_number_character(char ch)
{
char lower_ch;
if ('0' <= ch && ch <= '9')
return 1;
if (ch == '-' || ch == '+' || ch == '.')
return 1;
/* Hex digits, exponent (e), base (p), "infinity",.. */
lower_ch = ch | 0x20;
if ('a' <= lower_ch && lower_ch <= 'y')
return 1;
return 0;
}
/* Calculate the size of the buffer required for a strtod locale
* conversion. */
static int strtod_buffer_size(const char *s)
{
const char *p = s;
while (valid_number_character(*p))
p++;
return p - s;
}
/* Similar to strtod(), but must be passed the current locale's decimal point
* character. Guaranteed to be called at the start of any valid number in a string */
double fpconv_strtod(const char *nptr, char **endptr)
{
char localbuf[FPCONV_G_FMT_BUFSIZE];
char *buf, *endbuf, *dp;
int buflen;
double value;
/* System strtod() is fine when decimal point is '.' */
if (locale_decimal_point == '.')
return c_strtod(nptr, endptr);
buflen = strtod_buffer_size(nptr);
if (!buflen) {
/* No valid characters found, standard strtod() return */
*endptr = (char *)nptr;
return 0;
}
/* Duplicate number into buffer */
if (buflen >= FPCONV_G_FMT_BUFSIZE) {
/* Handle unusually large numbers */
buf = c_malloc(buflen + 1);
if (!buf) {
NODE_ERR("not enough memory\n");
return;
}
} else {
/* This is the common case.. */
buf = localbuf;
}
c_memcpy(buf, nptr, buflen);
buf[buflen] = 0;
/* Update decimal point character if found */
dp = c_strchr(buf, '.');
if (dp)
*dp = locale_decimal_point;
value = c_strtod(buf, &endbuf);
*endptr = (char *)&nptr[endbuf - buf];
if (buflen >= FPCONV_G_FMT_BUFSIZE)
c_free(buf);
return value;
}
/* "fmt" must point to a buffer of at least 6 characters */
static void set_number_format(char *fmt, int precision)
{
int d1, d2, i;
if(!(1 <= precision && precision <= 14)) return;
/* Create printf format (%.14g) from precision */
d1 = precision / 10;
d2 = precision % 10;
fmt[0] = '%';
fmt[1] = '.';
i = 2;
if (d1) {
fmt[i++] = '0' + d1;
}
fmt[i++] = '0' + d2;
fmt[i++] = 'g';
fmt[i] = 0;
}
/* Assumes there is always at least 32 characters available in the target buffer */
int fpconv_g_fmt(char *str, double num, int precision)
{
char buf[FPCONV_G_FMT_BUFSIZE];
char fmt[6];
int len;
char *b;
set_number_format(fmt, precision);
/* Pass through when decimal point character is dot. */
if (locale_decimal_point == '.'){
c_sprintf(str, fmt, num);
return c_strlen(str);
}
/* snprintf() to a buffer then translate for other decimal point characters */
c_sprintf(buf, fmt, num);
len = c_strlen(buf);
/* Copy into target location. Translate decimal point if required */
b = buf;
do {
*str++ = (*b == locale_decimal_point ? '.' : *b);
} while(*b++);
return len;
}
void fpconv_init()
{
fpconv_update_locale();
}
#endif
/* vi:ai et sw=4 ts=4:
*/
/* Lua CJSON floating point conversion routines */
/* Buffer required to store the largest string representation of a double.
*
* Longest double printed with %.14g is 21 characters long:
* -1.7976931348623e+308 */
# define FPCONV_G_FMT_BUFSIZE 32
#ifdef USE_INTERNAL_FPCONV
static inline void fpconv_init()
{
/* Do nothing - not required */
}
#else
extern inline void fpconv_init();
#endif
extern int fpconv_g_fmt(char*, double, int);
extern double fpconv_strtod(const char*, char**);
/* vi:ai et sw=4 ts=4:
*/
This diff is collapsed.
This diff is collapsed.
#!/usr/bin/env lua
-- usage: json2lua.lua [json_file]
--
-- Eg:
-- echo '[ "testing" ]' | ./json2lua.lua
-- ./json2lua.lua test.json
local json = require "cjson"
local util = require "cjson.util"
local json_text = util.file_load(arg[1])
local t = json.decode(json_text)
print(util.serialise_value(t))
#!/usr/bin/env lua
-- usage: lua2json.lua [lua_file]
--
-- Eg:
-- echo '{ "testing" }' | ./lua2json.lua
-- ./lua2json.lua test.lua
local json = require "cjson"
local util = require "cjson.util"
local env = {
json = { null = json.null },
null = json.null
}
local t = util.run_script("data = " .. util.file_load(arg[1]), env)
print(json.encode(t.data))
-- vi:ai et sw=4 ts=4:
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment