Commit 1f72ec7d authored by flowly's avatar flowly Committed by GitHub
Browse files

Merge pull request #1 from antirez/unstable

update to upstream
parents dfc98dcc f917e0da
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "fmacros.h"
#include <string.h>
#include <stdlib.h>
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include <assert.h>
#include <errno.h>
#include <ctype.h>
#include "read.h"
#include "sds.h"
static void __redisReaderSetError(redisReader *r, int type, const char *str) {
size_t len;
if (r->reply != NULL && r->fn && r->fn->freeObject) {
r->fn->freeObject(r->reply);
r->reply = NULL;
}
/* Clear input buffer on errors. */
if (r->buf != NULL) {
sdsfree(r->buf);
r->buf = NULL;
r->pos = r->len = 0;
}
/* Reset task stack. */
r->ridx = -1;
/* Set error. */
r->err = type;
len = strlen(str);
len = len < (sizeof(r->errstr)-1) ? len : (sizeof(r->errstr)-1);
memcpy(r->errstr,str,len);
r->errstr[len] = '\0';
}
static size_t chrtos(char *buf, size_t size, char byte) {
size_t len = 0;
switch(byte) {
case '\\':
case '"':
len = snprintf(buf,size,"\"\\%c\"",byte);
break;
case '\n': len = snprintf(buf,size,"\"\\n\""); break;
case '\r': len = snprintf(buf,size,"\"\\r\""); break;
case '\t': len = snprintf(buf,size,"\"\\t\""); break;
case '\a': len = snprintf(buf,size,"\"\\a\""); break;
case '\b': len = snprintf(buf,size,"\"\\b\""); break;
default:
if (isprint(byte))
len = snprintf(buf,size,"\"%c\"",byte);
else
len = snprintf(buf,size,"\"\\x%02x\"",(unsigned char)byte);
break;
}
return len;
}
static void __redisReaderSetErrorProtocolByte(redisReader *r, char byte) {
char cbuf[8], sbuf[128];
chrtos(cbuf,sizeof(cbuf),byte);
snprintf(sbuf,sizeof(sbuf),
"Protocol error, got %s as reply type byte", cbuf);
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,sbuf);
}
static void __redisReaderSetErrorOOM(redisReader *r) {
__redisReaderSetError(r,REDIS_ERR_OOM,"Out of memory");
}
static char *readBytes(redisReader *r, unsigned int bytes) {
char *p;
if (r->len-r->pos >= bytes) {
p = r->buf+r->pos;
r->pos += bytes;
return p;
}
return NULL;
}
/* Find pointer to \r\n. */
static char *seekNewline(char *s, size_t len) {
int pos = 0;
int _len = len-1;
/* Position should be < len-1 because the character at "pos" should be
* followed by a \n. Note that strchr cannot be used because it doesn't
* allow to search a limited length and the buffer that is being searched
* might not have a trailing NULL character. */
while (pos < _len) {
while(pos < _len && s[pos] != '\r') pos++;
if (pos==_len) {
/* Not found. */
return NULL;
} else {
if (s[pos+1] == '\n') {
/* Found. */
return s+pos;
} else {
/* Continue searching. */
pos++;
}
}
}
return NULL;
}
/* Read a long long value starting at *s, under the assumption that it will be
* terminated by \r\n. Ambiguously returns -1 for unexpected input. */
static long long readLongLong(char *s) {
long long v = 0;
int dec, mult = 1;
char c;
if (*s == '-') {
mult = -1;
s++;
} else if (*s == '+') {
mult = 1;
s++;
}
while ((c = *(s++)) != '\r') {
dec = c - '0';
if (dec >= 0 && dec < 10) {
v *= 10;
v += dec;
} else {
/* Should not happen... */
return -1;
}
}
return mult*v;
}
static char *readLine(redisReader *r, int *_len) {
char *p, *s;
int len;
p = r->buf+r->pos;
s = seekNewline(p,(r->len-r->pos));
if (s != NULL) {
len = s-(r->buf+r->pos);
r->pos += len+2; /* skip \r\n */
if (_len) *_len = len;
return p;
}
return NULL;
}
static void moveToNextTask(redisReader *r) {
redisReadTask *cur, *prv;
while (r->ridx >= 0) {
/* Return a.s.a.p. when the stack is now empty. */
if (r->ridx == 0) {
r->ridx--;
return;
}
cur = &(r->rstack[r->ridx]);
prv = &(r->rstack[r->ridx-1]);
assert(prv->type == REDIS_REPLY_ARRAY);
if (cur->idx == prv->elements-1) {
r->ridx--;
} else {
/* Reset the type because the next item can be anything */
assert(cur->idx < prv->elements);
cur->type = -1;
cur->elements = -1;
cur->idx++;
return;
}
}
}
static int processLineItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
int len;
if ((p = readLine(r,&len)) != NULL) {
if (cur->type == REDIS_REPLY_INTEGER) {
if (r->fn && r->fn->createInteger)
obj = r->fn->createInteger(cur,readLongLong(p));
else
obj = (void*)REDIS_REPLY_INTEGER;
} else {
/* Type will be error or status. */
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,p,len);
else
obj = (void*)(size_t)(cur->type);
}
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
return REDIS_ERR;
}
static int processBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj = NULL;
char *p, *s;
long len;
unsigned long bytelen;
int success = 0;
p = r->buf+r->pos;
s = seekNewline(p,r->len-r->pos);
if (s != NULL) {
p = r->buf+r->pos;
bytelen = s-(r->buf+r->pos)+2; /* include \r\n */
len = readLongLong(p);
if (len < 0) {
/* The nil object can always be created. */
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
success = 1;
} else {
/* Only continue when the buffer contains the entire bulk item. */
bytelen += len+2; /* include \r\n */
if (r->pos+bytelen <= r->len) {
if (r->fn && r->fn->createString)
obj = r->fn->createString(cur,s+2,len);
else
obj = (void*)REDIS_REPLY_STRING;
success = 1;
}
}
/* Proceed when obj was created. */
if (success) {
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->pos += bytelen;
/* Set reply if this is the root object. */
if (r->ridx == 0) r->reply = obj;
moveToNextTask(r);
return REDIS_OK;
}
}
return REDIS_ERR;
}
static int processMultiBulkItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
void *obj;
char *p;
long elements;
int root = 0;
/* Set error for nested multi bulks with depth > 7 */
if (r->ridx == 8) {
__redisReaderSetError(r,REDIS_ERR_PROTOCOL,
"No support for nested multi bulk replies with depth > 7");
return REDIS_ERR;
}
if ((p = readLine(r,NULL)) != NULL) {
elements = readLongLong(p);
root = (r->ridx == 0);
if (elements == -1) {
if (r->fn && r->fn->createNil)
obj = r->fn->createNil(cur);
else
obj = (void*)REDIS_REPLY_NIL;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
moveToNextTask(r);
} else {
if (r->fn && r->fn->createArray)
obj = r->fn->createArray(cur,elements);
else
obj = (void*)REDIS_REPLY_ARRAY;
if (obj == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
/* Modify task stack when there are more than 0 elements. */
if (elements > 0) {
cur->elements = elements;
cur->obj = obj;
r->ridx++;
r->rstack[r->ridx].type = -1;
r->rstack[r->ridx].elements = -1;
r->rstack[r->ridx].idx = 0;
r->rstack[r->ridx].obj = NULL;
r->rstack[r->ridx].parent = cur;
r->rstack[r->ridx].privdata = r->privdata;
} else {
moveToNextTask(r);
}
}
/* Set reply if this is the root object. */
if (root) r->reply = obj;
return REDIS_OK;
}
return REDIS_ERR;
}
static int processItem(redisReader *r) {
redisReadTask *cur = &(r->rstack[r->ridx]);
char *p;
/* check if we need to read type */
if (cur->type < 0) {
if ((p = readBytes(r,1)) != NULL) {
switch (p[0]) {
case '-':
cur->type = REDIS_REPLY_ERROR;
break;
case '+':
cur->type = REDIS_REPLY_STATUS;
break;
case ':':
cur->type = REDIS_REPLY_INTEGER;
break;
case '$':
cur->type = REDIS_REPLY_STRING;
break;
case '*':
cur->type = REDIS_REPLY_ARRAY;
break;
default:
__redisReaderSetErrorProtocolByte(r,*p);
return REDIS_ERR;
}
} else {
/* could not consume 1 byte */
return REDIS_ERR;
}
}
/* process typed item */
switch(cur->type) {
case REDIS_REPLY_ERROR:
case REDIS_REPLY_STATUS:
case REDIS_REPLY_INTEGER:
return processLineItem(r);
case REDIS_REPLY_STRING:
return processBulkItem(r);
case REDIS_REPLY_ARRAY:
return processMultiBulkItem(r);
default:
assert(NULL);
return REDIS_ERR; /* Avoid warning. */
}
}
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn) {
redisReader *r;
r = calloc(sizeof(redisReader),1);
if (r == NULL)
return NULL;
r->err = 0;
r->errstr[0] = '\0';
r->fn = fn;
r->buf = sdsempty();
r->maxbuf = REDIS_READER_MAX_BUF;
if (r->buf == NULL) {
free(r);
return NULL;
}
r->ridx = -1;
return r;
}
void redisReaderFree(redisReader *r) {
if (r->reply != NULL && r->fn && r->fn->freeObject)
r->fn->freeObject(r->reply);
if (r->buf != NULL)
sdsfree(r->buf);
free(r);
}
int redisReaderFeed(redisReader *r, const char *buf, size_t len) {
sds newbuf;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* Copy the provided buffer. */
if (buf != NULL && len >= 1) {
/* Destroy internal buffer when it is empty and is quite large. */
if (r->len == 0 && r->maxbuf != 0 && sdsavail(r->buf) > r->maxbuf) {
sdsfree(r->buf);
r->buf = sdsempty();
r->pos = 0;
/* r->buf should not be NULL since we just free'd a larger one. */
assert(r->buf != NULL);
}
newbuf = sdscatlen(r->buf,buf,len);
if (newbuf == NULL) {
__redisReaderSetErrorOOM(r);
return REDIS_ERR;
}
r->buf = newbuf;
r->len = sdslen(r->buf);
}
return REDIS_OK;
}
int redisReaderGetReply(redisReader *r, void **reply) {
/* Default target pointer to NULL. */
if (reply != NULL)
*reply = NULL;
/* Return early when this reader is in an erroneous state. */
if (r->err)
return REDIS_ERR;
/* When the buffer is empty, there will never be a reply. */
if (r->len == 0)
return REDIS_OK;
/* Set first item to process when the stack is empty. */
if (r->ridx == -1) {
r->rstack[0].type = -1;
r->rstack[0].elements = -1;
r->rstack[0].idx = -1;
r->rstack[0].obj = NULL;
r->rstack[0].parent = NULL;
r->rstack[0].privdata = r->privdata;
r->ridx = 0;
}
/* Process items in reply. */
while (r->ridx >= 0)
if (processItem(r) != REDIS_OK)
break;
/* Return ASAP when an error occurred. */
if (r->err)
return REDIS_ERR;
/* Discard part of the buffer when we've consumed at least 1k, to avoid
* doing unnecessary calls to memmove() in sds.c. */
if (r->pos >= 1024) {
sdsrange(r->buf,r->pos,-1);
r->pos = 0;
r->len = sdslen(r->buf);
}
/* Emit a reply when there is one. */
if (r->ridx == -1) {
if (reply != NULL)
*reply = r->reply;
r->reply = NULL;
}
return REDIS_OK;
}
/*
* Copyright (c) 2009-2011, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2010-2011, Pieter Noordhuis <pcnoordhuis at gmail dot com>
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __HIREDIS_READ_H
#define __HIREDIS_READ_H
#include <stdio.h> /* for size_t */
#define REDIS_ERR -1
#define REDIS_OK 0
/* When an error occurs, the err flag in a context is set to hold the type of
* error that occurred. REDIS_ERR_IO means there was an I/O error and you
* should use the "errno" variable to find out what is wrong.
* For other values, the "errstr" field will hold a description. */
#define REDIS_ERR_IO 1 /* Error in read or write */
#define REDIS_ERR_EOF 3 /* End of file */
#define REDIS_ERR_PROTOCOL 4 /* Protocol error */
#define REDIS_ERR_OOM 5 /* Out of memory */
#define REDIS_ERR_OTHER 2 /* Everything else... */
#define REDIS_REPLY_STRING 1
#define REDIS_REPLY_ARRAY 2
#define REDIS_REPLY_INTEGER 3
#define REDIS_REPLY_NIL 4
#define REDIS_REPLY_STATUS 5
#define REDIS_REPLY_ERROR 6
#define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */
#ifdef __cplusplus
extern "C" {
#endif
typedef struct redisReadTask {
int type;
int elements; /* number of elements in multibulk container */
int idx; /* index in parent (array) object */
void *obj; /* holds user-generated value for a read task */
struct redisReadTask *parent; /* parent task */
void *privdata; /* user-settable arbitrary field */
} redisReadTask;
typedef struct redisReplyObjectFunctions {
void *(*createString)(const redisReadTask*, char*, size_t);
void *(*createArray)(const redisReadTask*, int);
void *(*createInteger)(const redisReadTask*, long long);
void *(*createNil)(const redisReadTask*);
void (*freeObject)(void*);
} redisReplyObjectFunctions;
typedef struct redisReader {
int err; /* Error flags, 0 when there is no error */
char errstr[128]; /* String representation of error when applicable */
char *buf; /* Read buffer */
size_t pos; /* Buffer cursor */
size_t len; /* Buffer length */
size_t maxbuf; /* Max length of unused buffer */
redisReadTask rstack[9];
int ridx; /* Index of current read task */
void *reply; /* Temporary reply pointer */
redisReplyObjectFunctions *fn;
void *privdata;
} redisReader;
/* Public API for the protocol parser. */
redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn);
void redisReaderFree(redisReader *r);
int redisReaderFeed(redisReader *r, const char *buf, size_t len);
int redisReaderGetReply(redisReader *r, void **reply);
#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p))
#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply)
#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr)
#ifdef __cplusplus
}
#endif
#endif
/* SDSLib, A C dynamic strings library /* SDSLib 2.0 -- A C dynamic strings library
* *
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com> * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2015, Oran Agra
* Copyright (c) 2015, Redis Labs, Inc
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
...@@ -32,164 +34,523 @@ ...@@ -32,164 +34,523 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <ctype.h> #include <ctype.h>
#include <assert.h>
#include "sds.h" #include "sds.h"
#include "sdsalloc.h"
static inline int sdsHdrSize(char type) {
switch(type&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return sizeof(struct sdshdr5);
case SDS_TYPE_8:
return sizeof(struct sdshdr8);
case SDS_TYPE_16:
return sizeof(struct sdshdr16);
case SDS_TYPE_32:
return sizeof(struct sdshdr32);
case SDS_TYPE_64:
return sizeof(struct sdshdr64);
}
return 0;
}
#ifdef SDS_ABORT_ON_OOM static inline char sdsReqType(size_t string_size) {
static void sdsOomAbort(void) { if (string_size < 32)
fprintf(stderr,"SDS: Out Of Memory (SDS_ABORT_ON_OOM defined)\n"); return SDS_TYPE_5;
abort(); if (string_size < 0xff)
return SDS_TYPE_8;
if (string_size < 0xffff)
return SDS_TYPE_16;
if (string_size < 0xffffffff)
return SDS_TYPE_32;
return SDS_TYPE_64;
} }
#endif
/* Create a new sds string with the content specified by the 'init' pointer
* and 'initlen'.
* If NULL is used for 'init' the string is initialized with zero bytes.
*
* The string is always null-termined (all the sds strings are, always) so
* even if you create an sds string with:
*
* mystring = sdsnewlen("abc",3);
*
* You can print the string with printf() as there is an implicit \0 at the
* end of the string. However the string is binary safe and can contain
* \0 characters in the middle, as the length is stored in the sds header. */
sds sdsnewlen(const void *init, size_t initlen) { sds sdsnewlen(const void *init, size_t initlen) {
struct sdshdr *sh; void *sh;
sds s;
sh = malloc(sizeof(struct sdshdr)+initlen+1); char type = sdsReqType(initlen);
#ifdef SDS_ABORT_ON_OOM /* Empty strings are usually created in order to append. Use type 8
if (sh == NULL) sdsOomAbort(); * since type 5 is not good at this. */
#else if (type == SDS_TYPE_5 && initlen == 0) type = SDS_TYPE_8;
int hdrlen = sdsHdrSize(type);
unsigned char *fp; /* flags pointer. */
sh = s_malloc(hdrlen+initlen+1);
if (sh == NULL) return NULL; if (sh == NULL) return NULL;
#endif if (!init)
sh->len = initlen; memset(sh, 0, hdrlen+initlen+1);
sh->free = 0; s = (char*)sh+hdrlen;
if (initlen) { fp = ((unsigned char*)s)-1;
if (init) memcpy(sh->buf, init, initlen); switch(type) {
else memset(sh->buf,0,initlen); case SDS_TYPE_5: {
*fp = type | (initlen << SDS_TYPE_BITS);
break;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
sh->len = initlen;
sh->alloc = initlen;
*fp = type;
break;
}
} }
sh->buf[initlen] = '\0'; if (initlen && init)
return (char*)sh->buf; memcpy(s, init, initlen);
s[initlen] = '\0';
return s;
} }
/* Create an empty (zero length) sds string. Even in this case the string
* always has an implicit null term. */
sds sdsempty(void) { sds sdsempty(void) {
return sdsnewlen("",0); return sdsnewlen("",0);
} }
/* Create a new sds string starting from a null terminated C string. */
sds sdsnew(const char *init) { sds sdsnew(const char *init) {
size_t initlen = (init == NULL) ? 0 : strlen(init); size_t initlen = (init == NULL) ? 0 : strlen(init);
return sdsnewlen(init, initlen); return sdsnewlen(init, initlen);
} }
/* Duplicate an sds string. */
sds sdsdup(const sds s) { sds sdsdup(const sds s) {
return sdsnewlen(s, sdslen(s)); return sdsnewlen(s, sdslen(s));
} }
/* Free an sds string. No operation is performed if 's' is NULL. */
void sdsfree(sds s) { void sdsfree(sds s) {
if (s == NULL) return; if (s == NULL) return;
free(s-sizeof(struct sdshdr)); s_free((char*)s-sdsHdrSize(s[-1]));
} }
/* Set the sds string length to the length as obtained with strlen(), so
* considering as content only up to the first null term character.
*
* This function is useful when the sds string is hacked manually in some
* way, like in the following example:
*
* s = sdsnew("foobar");
* s[2] = '\0';
* sdsupdatelen(s);
* printf("%d\n", sdslen(s));
*
* The output will be "2", but if we comment out the call to sdsupdatelen()
* the output will be "6" as the string was modified but the logical length
* remains 6 bytes. */
void sdsupdatelen(sds s) { void sdsupdatelen(sds s) {
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
int reallen = strlen(s); int reallen = strlen(s);
sh->free += (sh->len-reallen); sdssetlen(s, reallen);
sh->len = reallen;
} }
static sds sdsMakeRoomFor(sds s, size_t addlen) { /* Modify an sds string in-place to make it empty (zero length).
struct sdshdr *sh, *newsh; * However all the existing buffer is not discarded but set as free space
size_t free = sdsavail(s); * so that next append operations will not require allocations up to the
* number of bytes previously available. */
void sdsclear(sds s) {
sdssetlen(s, 0);
s[0] = '\0';
}
/* Enlarge the free space at the end of the sds string so that the caller
* is sure that after calling this function can overwrite up to addlen
* bytes after the end of the string, plus one more byte for nul term.
*
* Note: this does not change the *length* of the sds string as returned
* by sdslen(), but only the free buffer space we have. */
sds sdsMakeRoomFor(sds s, size_t addlen) {
void *sh, *newsh;
size_t avail = sdsavail(s);
size_t len, newlen; size_t len, newlen;
char type, oldtype = s[-1] & SDS_TYPE_MASK;
int hdrlen;
/* Return ASAP if there is enough space left. */
if (avail >= addlen) return s;
if (free >= addlen) return s;
len = sdslen(s); len = sdslen(s);
sh = (void*) (s-(sizeof(struct sdshdr))); sh = (char*)s-sdsHdrSize(oldtype);
newlen = (len+addlen)*2; newlen = (len+addlen);
newsh = realloc(sh, sizeof(struct sdshdr)+newlen+1); if (newlen < SDS_MAX_PREALLOC)
#ifdef SDS_ABORT_ON_OOM newlen *= 2;
if (newsh == NULL) sdsOomAbort(); else
#else newlen += SDS_MAX_PREALLOC;
if (newsh == NULL) return NULL;
#endif type = sdsReqType(newlen);
/* Don't use type 5: the user is appending to the string and type 5 is
* not able to remember empty space, so sdsMakeRoomFor() must be called
* at every appending operation. */
if (type == SDS_TYPE_5) type = SDS_TYPE_8;
hdrlen = sdsHdrSize(type);
if (oldtype==type) {
newsh = s_realloc(sh, hdrlen+newlen+1);
if (newsh == NULL) return NULL;
s = (char*)newsh+hdrlen;
} else {
/* Since the header size changes, need to move the string forward,
* and can't use realloc */
newsh = s_malloc(hdrlen+newlen+1);
if (newsh == NULL) return NULL;
memcpy((char*)newsh+hdrlen, s, len+1);
s_free(sh);
s = (char*)newsh+hdrlen;
s[-1] = type;
sdssetlen(s, len);
}
sdssetalloc(s, newlen);
return s;
}
newsh->free = newlen - len; /* Reallocate the sds string so that it has no free space at the end. The
return newsh->buf; * contained string remains not altered, but next concatenation operations
* will require a reallocation.
*
* After the call, the passed sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds sdsRemoveFreeSpace(sds s) {
void *sh, *newsh;
char type, oldtype = s[-1] & SDS_TYPE_MASK;
int hdrlen;
size_t len = sdslen(s);
sh = (char*)s-sdsHdrSize(oldtype);
type = sdsReqType(len);
hdrlen = sdsHdrSize(type);
if (oldtype==type) {
newsh = s_realloc(sh, hdrlen+len+1);
if (newsh == NULL) return NULL;
s = (char*)newsh+hdrlen;
} else {
newsh = s_malloc(hdrlen+len+1);
if (newsh == NULL) return NULL;
memcpy((char*)newsh+hdrlen, s, len+1);
s_free(sh);
s = (char*)newsh+hdrlen;
s[-1] = type;
sdssetlen(s, len);
}
sdssetalloc(s, len);
return s;
}
/* Return the total size of the allocation of the specifed sds string,
* including:
* 1) The sds header before the pointer.
* 2) The string.
* 3) The free buffer at the end if any.
* 4) The implicit null term.
*/
size_t sdsAllocSize(sds s) {
size_t alloc = sdsalloc(s);
return sdsHdrSize(s[-1])+alloc+1;
}
/* Return the pointer of the actual SDS allocation (normally SDS strings
* are referenced by the start of the string buffer). */
void *sdsAllocPtr(sds s) {
return (void*) (s-sdsHdrSize(s[-1]));
}
/* Increment the sds length and decrements the left free space at the
* end of the string according to 'incr'. Also set the null term
* in the new end of the string.
*
* This function is used in order to fix the string length after the
* user calls sdsMakeRoomFor(), writes something after the end of
* the current string, and finally needs to set the new length.
*
* Note: it is possible to use a negative increment in order to
* right-trim the string.
*
* Usage example:
*
* Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the
* following schema, to cat bytes coming from the kernel to the end of an
* sds string without copying into an intermediate buffer:
*
* oldlen = sdslen(s);
* s = sdsMakeRoomFor(s, BUFFER_SIZE);
* nread = read(fd, s+oldlen, BUFFER_SIZE);
* ... check for nread <= 0 and handle it ...
* sdsIncrLen(s, nread);
*/
void sdsIncrLen(sds s, int incr) {
unsigned char flags = s[-1];
size_t len;
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5: {
unsigned char *fp = ((unsigned char*)s)-1;
unsigned char oldlen = SDS_TYPE_5_LEN(flags);
assert((incr > 0 && oldlen+incr < 32) || (incr < 0 && oldlen >= (unsigned int)(-incr)));
*fp = SDS_TYPE_5 | ((oldlen+incr) << SDS_TYPE_BITS);
len = oldlen+incr;
break;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8,s);
assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
len = (sh->len += incr);
break;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
assert((incr >= 0 && sh->alloc-sh->len >= incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
len = (sh->len += incr);
break;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
assert((incr >= 0 && sh->alloc-sh->len >= (unsigned int)incr) || (incr < 0 && sh->len >= (unsigned int)(-incr)));
len = (sh->len += incr);
break;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
assert((incr >= 0 && sh->alloc-sh->len >= (uint64_t)incr) || (incr < 0 && sh->len >= (uint64_t)(-incr)));
len = (sh->len += incr);
break;
}
default: len = 0; /* Just to avoid compilation warnings. */
}
s[len] = '\0';
} }
/* Grow the sds to have the specified length. Bytes that were not part of /* Grow the sds to have the specified length. Bytes that were not part of
* the original length of the sds will be set to zero. */ * the original length of the sds will be set to zero.
*
* if the specified length is smaller than the current length, no operation
* is performed. */
sds sdsgrowzero(sds s, size_t len) { sds sdsgrowzero(sds s, size_t len) {
struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); size_t curlen = sdslen(s);
size_t totlen, curlen = sh->len;
if (len <= curlen) return s; if (len <= curlen) return s;
s = sdsMakeRoomFor(s,len-curlen); s = sdsMakeRoomFor(s,len-curlen);
if (s == NULL) return NULL; if (s == NULL) return NULL;
/* Make sure added region doesn't contain garbage */ /* Make sure added region doesn't contain garbage */
sh = (void*)(s-(sizeof(struct sdshdr)));
memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */ memset(s+curlen,0,(len-curlen+1)); /* also set trailing \0 byte */
totlen = sh->len+sh->free; sdssetlen(s, len);
sh->len = len;
sh->free = totlen-sh->len;
return s; return s;
} }
/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the
* end of the specified sds string 's'.
*
* After the call, the passed sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds sdscatlen(sds s, const void *t, size_t len) { sds sdscatlen(sds s, const void *t, size_t len) {
struct sdshdr *sh;
size_t curlen = sdslen(s); size_t curlen = sdslen(s);
s = sdsMakeRoomFor(s,len); s = sdsMakeRoomFor(s,len);
if (s == NULL) return NULL; if (s == NULL) return NULL;
sh = (void*) (s-(sizeof(struct sdshdr)));
memcpy(s+curlen, t, len); memcpy(s+curlen, t, len);
sh->len = curlen+len; sdssetlen(s, curlen+len);
sh->free = sh->free-len;
s[curlen+len] = '\0'; s[curlen+len] = '\0';
return s; return s;
} }
/* Append the specified null termianted C string to the sds string 's'.
*
* After the call, the passed sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds sdscat(sds s, const char *t) { sds sdscat(sds s, const char *t) {
return sdscatlen(s, t, strlen(t)); return sdscatlen(s, t, strlen(t));
} }
sds sdscpylen(sds s, char *t, size_t len) { /* Append the specified sds 't' to the existing sds 's'.
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); *
size_t totlen = sh->free+sh->len; * After the call, the modified sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds sdscatsds(sds s, const sds t) {
return sdscatlen(s, t, sdslen(t));
}
if (totlen < len) { /* Destructively modify the sds string 's' to hold the specified binary
s = sdsMakeRoomFor(s,len-sh->len); * safe string pointed by 't' of length 'len' bytes. */
sds sdscpylen(sds s, const char *t, size_t len) {
if (sdsalloc(s) < len) {
s = sdsMakeRoomFor(s,len-sdslen(s));
if (s == NULL) return NULL; if (s == NULL) return NULL;
sh = (void*) (s-(sizeof(struct sdshdr)));
totlen = sh->free+sh->len;
} }
memcpy(s, t, len); memcpy(s, t, len);
s[len] = '\0'; s[len] = '\0';
sh->len = len; sdssetlen(s, len);
sh->free = totlen-len;
return s; return s;
} }
sds sdscpy(sds s, char *t) { /* Like sdscpylen() but 't' must be a null-termined string so that the length
* of the string is obtained with strlen(). */
sds sdscpy(sds s, const char *t) {
return sdscpylen(s, t, strlen(t)); return sdscpylen(s, t, strlen(t));
} }
/* Helper for sdscatlonglong() doing the actual number -> string
* conversion. 's' must point to a string with room for at least
* SDS_LLSTR_SIZE bytes.
*
* The function returns the length of the null-terminated string
* representation stored at 's'. */
#define SDS_LLSTR_SIZE 21
int sdsll2str(char *s, long long value) {
char *p, aux;
unsigned long long v;
size_t l;
/* Generate the string representation, this method produces
* an reversed string. */
v = (value < 0) ? -value : value;
p = s;
do {
*p++ = '0'+(v%10);
v /= 10;
} while(v);
if (value < 0) *p++ = '-';
/* Compute length and add null term. */
l = p-s;
*p = '\0';
/* Reverse the string. */
p--;
while(s < p) {
aux = *s;
*s = *p;
*p = aux;
s++;
p--;
}
return l;
}
/* Identical sdsll2str(), but for unsigned long long type. */
int sdsull2str(char *s, unsigned long long v) {
char *p, aux;
size_t l;
/* Generate the string representation, this method produces
* an reversed string. */
p = s;
do {
*p++ = '0'+(v%10);
v /= 10;
} while(v);
/* Compute length and add null term. */
l = p-s;
*p = '\0';
/* Reverse the string. */
p--;
while(s < p) {
aux = *s;
*s = *p;
*p = aux;
s++;
p--;
}
return l;
}
/* Create an sds string from a long long value. It is much faster than:
*
* sdscatprintf(sdsempty(),"%lld\n", value);
*/
sds sdsfromlonglong(long long value) {
char buf[SDS_LLSTR_SIZE];
int len = sdsll2str(buf,value);
return sdsnewlen(buf,len);
}
/* Like sdscatprintf() but gets va_list instead of being variadic. */
sds sdscatvprintf(sds s, const char *fmt, va_list ap) { sds sdscatvprintf(sds s, const char *fmt, va_list ap) {
va_list cpy; va_list cpy;
char *buf, *t; char staticbuf[1024], *buf = staticbuf, *t;
size_t buflen = 16; size_t buflen = strlen(fmt)*2;
while(1) { /* We try to start using a static buffer for speed.
buf = malloc(buflen); * If not possible we revert to heap allocation. */
#ifdef SDS_ABORT_ON_OOM if (buflen > sizeof(staticbuf)) {
if (buf == NULL) sdsOomAbort(); buf = s_malloc(buflen);
#else
if (buf == NULL) return NULL; if (buf == NULL) return NULL;
#endif } else {
buflen = sizeof(staticbuf);
}
/* Try with buffers two times bigger every time we fail to
* fit the string in the current buffer size. */
while(1) {
buf[buflen-2] = '\0'; buf[buflen-2] = '\0';
va_copy(cpy,ap); va_copy(cpy,ap);
vsnprintf(buf, buflen, fmt, cpy); vsnprintf(buf, buflen, fmt, cpy);
va_end(cpy);
if (buf[buflen-2] != '\0') { if (buf[buflen-2] != '\0') {
free(buf); if (buf != staticbuf) s_free(buf);
buflen *= 2; buflen *= 2;
buf = s_malloc(buflen);
if (buf == NULL) return NULL;
continue; continue;
} }
break; break;
} }
/* Finally concat the obtained string to the SDS string and return it. */
t = sdscat(s, buf); t = sdscat(s, buf);
free(buf); if (buf != staticbuf) s_free(buf);
return t; return t;
} }
/* Append to the sds string 's' a string obtained using printf-alike format
* specifier.
*
* After the call, the modified sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call.
*
* Example:
*
* s = sdsnew("Sum is: ");
* s = sdscatprintf(s,"%d+%d = %d",a,b,a+b).
*
* Often you need to create a string from scratch with the printf-alike
* format. When this is the need, just use sdsempty() as the target string:
*
* s = sdscatprintf(sdsempty(), "... your format ...", args);
*/
sds sdscatprintf(sds s, const char *fmt, ...) { sds sdscatprintf(sds s, const char *fmt, ...) {
va_list ap; va_list ap;
char *t; char *t;
...@@ -199,28 +560,159 @@ sds sdscatprintf(sds s, const char *fmt, ...) { ...@@ -199,28 +560,159 @@ sds sdscatprintf(sds s, const char *fmt, ...) {
return t; return t;
} }
/* This function is similar to sdscatprintf, but much faster as it does
* not rely on sprintf() family functions implemented by the libc that
* are often very slow. Moreover directly handling the sds string as
* new data is concatenated provides a performance improvement.
*
* However this function only handles an incompatible subset of printf-alike
* format specifiers:
*
* %s - C String
* %S - SDS string
* %i - signed int
* %I - 64 bit signed integer (long long, int64_t)
* %u - unsigned int
* %U - 64 bit unsigned integer (unsigned long long, uint64_t)
* %% - Verbatim "%" character.
*/
sds sdscatfmt(sds s, char const *fmt, ...) {
const char *f = fmt;
int i;
va_list ap;
va_start(ap,fmt);
i = sdslen(s); /* Position of the next byte to write to dest str. */
while(*f) {
char next, *str;
size_t l;
long long num;
unsigned long long unum;
/* Make sure there is always space for at least 1 char. */
if (sdsavail(s)==0) {
s = sdsMakeRoomFor(s,1);
}
switch(*f) {
case '%':
next = *(f+1);
f++;
switch(next) {
case 's':
case 'S':
str = va_arg(ap,char*);
l = (next == 's') ? strlen(str) : sdslen(str);
if (sdsavail(s) < l) {
s = sdsMakeRoomFor(s,l);
}
memcpy(s+i,str,l);
sdsinclen(s,l);
i += l;
break;
case 'i':
case 'I':
if (next == 'i')
num = va_arg(ap,int);
else
num = va_arg(ap,long long);
{
char buf[SDS_LLSTR_SIZE];
l = sdsll2str(buf,num);
if (sdsavail(s) < l) {
s = sdsMakeRoomFor(s,l);
}
memcpy(s+i,buf,l);
sdsinclen(s,l);
i += l;
}
break;
case 'u':
case 'U':
if (next == 'u')
unum = va_arg(ap,unsigned int);
else
unum = va_arg(ap,unsigned long long);
{
char buf[SDS_LLSTR_SIZE];
l = sdsull2str(buf,unum);
if (sdsavail(s) < l) {
s = sdsMakeRoomFor(s,l);
}
memcpy(s+i,buf,l);
sdsinclen(s,l);
i += l;
}
break;
default: /* Handle %% and generally %<unknown>. */
s[i++] = next;
sdsinclen(s,1);
break;
}
break;
default:
s[i++] = *f;
sdsinclen(s,1);
break;
}
f++;
}
va_end(ap);
/* Add null-term */
s[i] = '\0';
return s;
}
/* Remove the part of the string from left and from right composed just of
* contiguous characters found in 'cset', that is a null terminted C string.
*
* After the call, the modified sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call.
*
* Example:
*
* s = sdsnew("AA...AA.a.aa.aHelloWorld :::");
* s = sdstrim(s,"Aa. :");
* printf("%s\n", s);
*
* Output will be just "Hello World".
*/
sds sdstrim(sds s, const char *cset) { sds sdstrim(sds s, const char *cset) {
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr)));
char *start, *end, *sp, *ep; char *start, *end, *sp, *ep;
size_t len; size_t len;
sp = start = s; sp = start = s;
ep = end = s+sdslen(s)-1; ep = end = s+sdslen(s)-1;
while(sp <= end && strchr(cset, *sp)) sp++; while(sp <= end && strchr(cset, *sp)) sp++;
while(ep > start && strchr(cset, *ep)) ep--; while(ep > sp && strchr(cset, *ep)) ep--;
len = (sp > ep) ? 0 : ((ep-sp)+1); len = (sp > ep) ? 0 : ((ep-sp)+1);
if (sh->buf != sp) memmove(sh->buf, sp, len); if (s != sp) memmove(s, sp, len);
sh->buf[len] = '\0'; s[len] = '\0';
sh->free = sh->free+(sh->len-len); sdssetlen(s,len);
sh->len = len;
return s; return s;
} }
sds sdsrange(sds s, int start, int end) { /* Turn the string into a smaller (or equal) string containing only the
struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); * substring specified by the 'start' and 'end' indexes.
*
* start and end can be negative, where -1 means the last character of the
* string, -2 the penultimate character, and so forth.
*
* The interval is inclusive, so the start and end characters will be part
* of the resulting string.
*
* The string is modified in-place.
*
* Example:
*
* s = sdsnew("Hello World");
* sdsrange(s,1,-1); => "ello World"
*/
void sdsrange(sds s, int start, int end) {
size_t newlen, len = sdslen(s); size_t newlen, len = sdslen(s);
if (len == 0) return s; if (len == 0) return;
if (start < 0) { if (start < 0) {
start = len+start; start = len+start;
if (start < 0) start = 0; if (start < 0) start = 0;
...@@ -240,26 +732,37 @@ sds sdsrange(sds s, int start, int end) { ...@@ -240,26 +732,37 @@ sds sdsrange(sds s, int start, int end) {
} else { } else {
start = 0; start = 0;
} }
if (start && newlen) memmove(sh->buf, sh->buf+start, newlen); if (start && newlen) memmove(s, s+start, newlen);
sh->buf[newlen] = 0; s[newlen] = 0;
sh->free = sh->free+(sh->len-newlen); sdssetlen(s,newlen);
sh->len = newlen;
return s;
} }
/* Apply tolower() to every character of the sds string 's'. */
void sdstolower(sds s) { void sdstolower(sds s) {
int len = sdslen(s), j; int len = sdslen(s), j;
for (j = 0; j < len; j++) s[j] = tolower(s[j]); for (j = 0; j < len; j++) s[j] = tolower(s[j]);
} }
/* Apply toupper() to every character of the sds string 's'. */
void sdstoupper(sds s) { void sdstoupper(sds s) {
int len = sdslen(s), j; int len = sdslen(s), j;
for (j = 0; j < len; j++) s[j] = toupper(s[j]); for (j = 0; j < len; j++) s[j] = toupper(s[j]);
} }
int sdscmp(sds s1, sds s2) { /* Compare two sds strings s1 and s2 with memcmp().
*
* Return value:
*
* positive if s1 > s2.
* negative if s1 < s2.
* 0 if s1 and s2 are exactly the same binary string.
*
* If two strings share exactly the same prefix, but one of the two has
* additional characters, the longer string is considered to be greater than
* the smaller one. */
int sdscmp(const sds s1, const sds s2) {
size_t l1, l2, minlen; size_t l1, l2, minlen;
int cmp; int cmp;
...@@ -287,14 +790,15 @@ int sdscmp(sds s1, sds s2) { ...@@ -287,14 +790,15 @@ int sdscmp(sds s1, sds s2) {
* requires length arguments. sdssplit() is just the * requires length arguments. sdssplit() is just the
* same function but for zero-terminated strings. * same function but for zero-terminated strings.
*/ */
sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) {
int elements = 0, slots = 5, start = 0, j; int elements = 0, slots = 5, start = 0, j;
sds *tokens;
if (seplen < 1 || len < 0) return NULL;
tokens = s_malloc(sizeof(sds)*slots);
if (tokens == NULL) return NULL;
sds *tokens = malloc(sizeof(sds)*slots);
#ifdef SDS_ABORT_ON_OOM
if (tokens == NULL) sdsOomAbort();
#endif
if (seplen < 1 || len < 0 || tokens == NULL) return NULL;
if (len == 0) { if (len == 0) {
*count = 0; *count = 0;
return tokens; return tokens;
...@@ -305,26 +809,14 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { ...@@ -305,26 +809,14 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) {
sds *newtokens; sds *newtokens;
slots *= 2; slots *= 2;
newtokens = realloc(tokens,sizeof(sds)*slots); newtokens = s_realloc(tokens,sizeof(sds)*slots);
if (newtokens == NULL) { if (newtokens == NULL) goto cleanup;
#ifdef SDS_ABORT_ON_OOM
sdsOomAbort();
#else
goto cleanup;
#endif
}
tokens = newtokens; tokens = newtokens;
} }
/* search the separator */ /* search the separator */
if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) {
tokens[elements] = sdsnewlen(s+start,j-start); tokens[elements] = sdsnewlen(s+start,j-start);
if (tokens[elements] == NULL) { if (tokens[elements] == NULL) goto cleanup;
#ifdef SDS_ABORT_ON_OOM
sdsOomAbort();
#else
goto cleanup;
#endif
}
elements++; elements++;
start = j+seplen; start = j+seplen;
j = j+seplen-1; /* skip the separator */ j = j+seplen-1; /* skip the separator */
...@@ -332,54 +824,37 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { ...@@ -332,54 +824,37 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) {
} }
/* Add the final element. We are sure there is room in the tokens array. */ /* Add the final element. We are sure there is room in the tokens array. */
tokens[elements] = sdsnewlen(s+start,len-start); tokens[elements] = sdsnewlen(s+start,len-start);
if (tokens[elements] == NULL) { if (tokens[elements] == NULL) goto cleanup;
#ifdef SDS_ABORT_ON_OOM
sdsOomAbort();
#else
goto cleanup;
#endif
}
elements++; elements++;
*count = elements; *count = elements;
return tokens; return tokens;
#ifndef SDS_ABORT_ON_OOM
cleanup: cleanup:
{ {
int i; int i;
for (i = 0; i < elements; i++) sdsfree(tokens[i]); for (i = 0; i < elements; i++) sdsfree(tokens[i]);
free(tokens); s_free(tokens);
*count = 0;
return NULL; return NULL;
} }
#endif
} }
/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */
void sdsfreesplitres(sds *tokens, int count) { void sdsfreesplitres(sds *tokens, int count) {
if (!tokens) return; if (!tokens) return;
while(count--) while(count--)
sdsfree(tokens[count]); sdsfree(tokens[count]);
free(tokens); s_free(tokens);
}
sds sdsfromlonglong(long long value) {
char buf[32], *p;
unsigned long long v;
v = (value < 0) ? -value : value;
p = buf+31; /* point to the last character */
do {
*p-- = '0'+(v%10);
v /= 10;
} while(v);
if (value < 0) *p-- = '-';
p++;
return sdsnewlen(p,32-(p-buf));
} }
sds sdscatrepr(sds s, char *p, size_t len) { /* Append to the sds string "s" an escaped string representation where
* all the non-printable characters (tested with isprint()) are turned into
* escapes in the form "\n\r\a...." or "\x<hex-number>".
*
* After the call, the modified sds string is no longer valid and all the
* references must be substituted with the new pointer returned by the call. */
sds sdscatrepr(sds s, const char *p, size_t len) {
s = sdscatlen(s,"\"",1); s = sdscatlen(s,"\"",1);
if (s == NULL) return NULL;
while(len--) { while(len--) {
switch(*p) { switch(*p) {
case '\\': case '\\':
...@@ -399,27 +874,64 @@ sds sdscatrepr(sds s, char *p, size_t len) { ...@@ -399,27 +874,64 @@ sds sdscatrepr(sds s, char *p, size_t len) {
break; break;
} }
p++; p++;
if (s == NULL) return NULL;
} }
return sdscatlen(s,"\"",1); return sdscatlen(s,"\"",1);
} }
/* Helper function for sdssplitargs() that returns non zero if 'c'
* is a valid hex digit. */
int is_hex_digit(char c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') ||
(c >= 'A' && c <= 'F');
}
/* Helper function for sdssplitargs() that converts a hex digit into an
* integer from 0 to 15 */
int hex_digit_to_int(char c) {
switch(c) {
case '0': return 0;
case '1': return 1;
case '2': return 2;
case '3': return 3;
case '4': return 4;
case '5': return 5;
case '6': return 6;
case '7': return 7;
case '8': return 8;
case '9': return 9;
case 'a': case 'A': return 10;
case 'b': case 'B': return 11;
case 'c': case 'C': return 12;
case 'd': case 'D': return 13;
case 'e': case 'E': return 14;
case 'f': case 'F': return 15;
default: return 0;
}
}
/* Split a line into arguments, where every argument can be in the /* Split a line into arguments, where every argument can be in the
* following programming-language REPL-alike form: * following programming-language REPL-alike form:
* *
* foo bar "newline are supported\n" and "\xff\x00otherstuff" * foo bar "newline are supported\n" and "\xff\x00otherstuff"
* *
* The number of arguments is stored into *argc, and an array * The number of arguments is stored into *argc, and an array
* of sds is returned. The caller should sdsfree() all the returned * of sds is returned.
* strings and finally free() the array itself. *
* The caller should free the resulting array of sds strings with
* sdsfreesplitres().
* *
* Note that sdscatrepr() is able to convert back a string into * Note that sdscatrepr() is able to convert back a string into
* a quoted string in the same format sdssplitargs() is able to parse. * a quoted string in the same format sdssplitargs() is able to parse.
*
* The function returns the allocated tokens on success, even when the
* input string is empty, or NULL if the input contains unbalanced
* quotes or closed quotes followed by non space characters
* as in: "foo"bar or "foo'
*/ */
sds *sdssplitargs(char *line, int *argc) { sds *sdssplitargs(const char *line, int *argc) {
char *p = line; const char *p = line;
char *current = NULL; char *current = NULL;
char **vector = NULL, **_vector = NULL; char **vector = NULL;
*argc = 0; *argc = 0;
while(1) { while(1) {
...@@ -427,17 +939,24 @@ sds *sdssplitargs(char *line, int *argc) { ...@@ -427,17 +939,24 @@ sds *sdssplitargs(char *line, int *argc) {
while(*p && isspace(*p)) p++; while(*p && isspace(*p)) p++;
if (*p) { if (*p) {
/* get a token */ /* get a token */
int inq=0; /* set to 1 if we are in "quotes" */ int inq=0; /* set to 1 if we are in "quotes" */
int insq=0; /* set to 1 if we are in 'single quotes' */
int done=0; int done=0;
if (current == NULL) { if (current == NULL) current = sdsempty();
current = sdsempty();
if (current == NULL) goto err;
}
while(!done) { while(!done) {
if (inq) { if (inq) {
if (*p == '\\' && *(p+1)) { if (*p == '\\' && *(p+1) == 'x' &&
is_hex_digit(*(p+2)) &&
is_hex_digit(*(p+3)))
{
unsigned char byte;
byte = (hex_digit_to_int(*(p+2))*16)+
hex_digit_to_int(*(p+3));
current = sdscatlen(current,(char*)&byte,1);
p += 3;
} else if (*p == '\\' && *(p+1)) {
char c; char c;
p++; p++;
...@@ -451,7 +970,23 @@ sds *sdssplitargs(char *line, int *argc) { ...@@ -451,7 +970,23 @@ sds *sdssplitargs(char *line, int *argc) {
} }
current = sdscatlen(current,&c,1); current = sdscatlen(current,&c,1);
} else if (*p == '"') { } else if (*p == '"') {
/* closing quote must be followed by a space */ /* closing quote must be followed by a space or
* nothing at all. */
if (*(p+1) && !isspace(*(p+1))) goto err;
done=1;
} else if (!*p) {
/* unterminated quotes */
goto err;
} else {
current = sdscatlen(current,p,1);
}
} else if (insq) {
if (*p == '\\' && *(p+1) == '\'') {
p++;
current = sdscatlen(current,"'",1);
} else if (*p == '\'') {
/* closing quote must be followed by a space or
* nothing at all. */
if (*(p+1) && !isspace(*(p+1))) goto err; if (*(p+1) && !isspace(*(p+1))) goto err;
done=1; done=1;
} else if (!*p) { } else if (!*p) {
...@@ -472,23 +1007,24 @@ sds *sdssplitargs(char *line, int *argc) { ...@@ -472,23 +1007,24 @@ sds *sdssplitargs(char *line, int *argc) {
case '"': case '"':
inq=1; inq=1;
break; break;
case '\'':
insq=1;
break;
default: default:
current = sdscatlen(current,p,1); current = sdscatlen(current,p,1);
break; break;
} }
} }
if (*p) p++; if (*p) p++;
if (current == NULL) goto err;
} }
/* add the token to the vector */ /* add the token to the vector */
_vector = realloc(vector,((*argc)+1)*sizeof(char*)); vector = s_realloc(vector,((*argc)+1)*sizeof(char*));
if (_vector == NULL) goto err;
vector = _vector;
vector[*argc] = current; vector[*argc] = current;
(*argc)++; (*argc)++;
current = NULL; current = NULL;
} else { } else {
/* Even on empty input string return something not NULL. */
if (vector == NULL) vector = s_malloc(sizeof(void*));
return vector; return vector;
} }
} }
...@@ -496,29 +1032,76 @@ sds *sdssplitargs(char *line, int *argc) { ...@@ -496,29 +1032,76 @@ sds *sdssplitargs(char *line, int *argc) {
err: err:
while((*argc)--) while((*argc)--)
sdsfree(vector[*argc]); sdsfree(vector[*argc]);
if (vector != NULL) free(vector); s_free(vector);
if (current != NULL) sdsfree(current); if (current) sdsfree(current);
*argc = 0;
return NULL; return NULL;
} }
#ifdef SDS_TEST_MAIN /* Modify the string substituting all the occurrences of the set of
#include <stdio.h> * characters specified in the 'from' string to the corresponding character
* in the 'to' array.
*
* For instance: sdsmapchars(mystring, "ho", "01", 2)
* will have the effect of turning the string "hello" into "0ell1".
*
* The function returns the sds string pointer, that is always the same
* as the input pointer since no resize is needed. */
sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) {
size_t j, i, l = sdslen(s);
for (j = 0; j < l; j++) {
for (i = 0; i < setlen; i++) {
if (s[j] == from[i]) {
s[j] = to[i];
break;
}
}
}
return s;
}
int __failed_tests = 0; /* Join an array of C strings using the specified separator (also a C string).
int __test_num = 0; * Returns the result as an sds string. */
#define test_cond(descr,_c) do { \ sds sdsjoin(char **argv, int argc, char *sep) {
__test_num++; printf("%d - %s: ", __test_num, descr); \ sds join = sdsempty();
if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \ int j;
} while(0);
#define test_report() do { \
printf("%d tests, %d passed, %d failed\n", __test_num, \
__test_num-__failed_tests, __failed_tests); \
if (__failed_tests) { \
printf("=== WARNING === We have failed tests here...\n"); \
} \
} while(0);
int main(void) { for (j = 0; j < argc; j++) {
join = sdscat(join, argv[j]);
if (j != argc-1) join = sdscat(join,sep);
}
return join;
}
/* Like sdsjoin, but joins an array of SDS strings. */
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen) {
sds join = sdsempty();
int j;
for (j = 0; j < argc; j++) {
join = sdscatsds(join, argv[j]);
if (j != argc-1) join = sdscatlen(join,sep,seplen);
}
return join;
}
/* Wrappers to the allocators used by SDS. Note that SDS will actually
* just use the macros defined into sdsalloc.h in order to avoid to pay
* the overhead of function calls. Here we define these wrappers only for
* the programs SDS is linked to, if they want to touch the SDS internals
* even if they use a different allocator. */
void *sds_malloc(size_t size) { return s_malloc(size); }
void *sds_realloc(void *ptr, size_t size) { return s_realloc(ptr,size); }
void sds_free(void *ptr) { s_free(ptr); }
#if defined(SDS_TEST_MAIN)
#include <stdio.h>
#include "testhelp.h"
#include "limits.h"
#define UNUSED(x) (void)(x)
int sdsTest(void) {
{ {
sds x = sdsnew("foo"), y; sds x = sdsnew("foo"), y;
...@@ -546,39 +1129,74 @@ int main(void) { ...@@ -546,39 +1129,74 @@ int main(void) {
sdsfree(x); sdsfree(x);
x = sdscatprintf(sdsempty(),"%d",123); x = sdscatprintf(sdsempty(),"%d",123);
test_cond("sdscatprintf() seems working in the base case", test_cond("sdscatprintf() seems working in the base case",
sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) sdslen(x) == 3 && memcmp(x,"123\0",4) == 0)
sdsfree(x);
x = sdsnew("--");
x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX);
test_cond("sdscatfmt() seems working in the base case",
sdslen(x) == 60 &&
memcmp(x,"--Hello Hi! World -9223372036854775808,"
"9223372036854775807--",60) == 0)
printf("[%s]\n",x);
sdsfree(x);
x = sdsnew("--");
x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX);
test_cond("sdscatfmt() seems working with unsigned numbers",
sdslen(x) == 35 &&
memcmp(x,"--4294967295,18446744073709551615--",35) == 0)
sdsfree(x);
x = sdsnew(" x ");
sdstrim(x," x");
test_cond("sdstrim() works when all chars match",
sdslen(x) == 0)
sdsfree(x);
x = sdsnew(" x ");
sdstrim(x," ");
test_cond("sdstrim() works when a single char remains",
sdslen(x) == 1 && x[0] == 'x')
sdsfree(x); sdsfree(x);
x = sdstrim(sdsnew("xxciaoyyy"),"xy"); x = sdsnew("xxciaoyyy");
sdstrim(x,"xy");
test_cond("sdstrim() correctly trims characters", test_cond("sdstrim() correctly trims characters",
sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0)
y = sdsrange(sdsdup(x),1,1); y = sdsdup(x);
sdsrange(y,1,1);
test_cond("sdsrange(...,1,1)", test_cond("sdsrange(...,1,1)",
sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) sdslen(y) == 1 && memcmp(y,"i\0",2) == 0)
sdsfree(y); sdsfree(y);
y = sdsrange(sdsdup(x),1,-1); y = sdsdup(x);
sdsrange(y,1,-1);
test_cond("sdsrange(...,1,-1)", test_cond("sdsrange(...,1,-1)",
sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
sdsfree(y); sdsfree(y);
y = sdsrange(sdsdup(x),-2,-1); y = sdsdup(x);
sdsrange(y,-2,-1);
test_cond("sdsrange(...,-2,-1)", test_cond("sdsrange(...,-2,-1)",
sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0)
sdsfree(y); sdsfree(y);
y = sdsrange(sdsdup(x),2,1); y = sdsdup(x);
sdsrange(y,2,1);
test_cond("sdsrange(...,2,1)", test_cond("sdsrange(...,2,1)",
sdslen(y) == 0 && memcmp(y,"\0",1) == 0) sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
sdsfree(y); sdsfree(y);
y = sdsrange(sdsdup(x),1,100); y = sdsdup(x);
sdsrange(y,1,100);
test_cond("sdsrange(...,1,100)", test_cond("sdsrange(...,1,100)",
sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0)
sdsfree(y); sdsfree(y);
y = sdsrange(sdsdup(x),100,100); y = sdsdup(x);
sdsrange(y,100,100);
test_cond("sdsrange(...,100,100)", test_cond("sdsrange(...,100,100)",
sdslen(y) == 0 && memcmp(y,"\0",1) == 0) sdslen(y) == 0 && memcmp(y,"\0",1) == 0)
...@@ -599,7 +1217,56 @@ int main(void) { ...@@ -599,7 +1217,56 @@ int main(void) {
x = sdsnew("aar"); x = sdsnew("aar");
y = sdsnew("bar"); y = sdsnew("bar");
test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0)
sdsfree(y);
sdsfree(x);
x = sdsnewlen("\a\n\0foo\r",7);
y = sdscatrepr(sdsempty(),x,sdslen(x));
test_cond("sdscatrepr(...data...)",
memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0)
{
unsigned int oldfree;
char *p;
int step = 10, j, i;
sdsfree(x);
sdsfree(y);
x = sdsnew("0");
test_cond("sdsnew() free/len buffers", sdslen(x) == 1 && sdsavail(x) == 0);
/* Run the test a few times in order to hit the first two
* SDS header types. */
for (i = 0; i < 10; i++) {
int oldlen = sdslen(x);
x = sdsMakeRoomFor(x,step);
int type = x[-1]&SDS_TYPE_MASK;
test_cond("sdsMakeRoomFor() len", sdslen(x) == oldlen);
if (type != SDS_TYPE_5) {
test_cond("sdsMakeRoomFor() free", sdsavail(x) >= step);
oldfree = sdsavail(x);
}
p = x+oldlen;
for (j = 0; j < step; j++) {
p[j] = 'A'+j;
}
sdsIncrLen(x,step);
}
test_cond("sdsMakeRoomFor() content",
memcmp("0ABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJABCDEFGHIJ",x,101) == 0);
test_cond("sdsMakeRoomFor() final length",sdslen(x)==101);
sdsfree(x);
}
} }
test_report() test_report()
return 0;
}
#endif
#ifdef SDS_TEST_MAIN
int main(void) {
return sdsTest();
} }
#endif #endif
/* SDSLib, A C dynamic strings library /* SDSLib 2.0 -- A C dynamic strings library
* *
* Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com> * Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2015, Oran Agra
* Copyright (c) 2015, Redis Labs, Inc
* All rights reserved. * All rights reserved.
* *
* Redistribution and use in source and binary forms, with or without * Redistribution and use in source and binary forms, with or without
...@@ -31,39 +33,198 @@ ...@@ -31,39 +33,198 @@
#ifndef __SDS_H #ifndef __SDS_H
#define __SDS_H #define __SDS_H
#define SDS_MAX_PREALLOC (1024*1024)
#include <sys/types.h> #include <sys/types.h>
#include <stdarg.h> #include <stdarg.h>
#include <stdint.h>
typedef char *sds; typedef char *sds;
struct sdshdr { /* Note: sdshdr5 is never used, we just access the flags byte directly.
int len; * However is here to document the layout of type 5 SDS strings. */
int free; struct __attribute__ ((__packed__)) sdshdr5 {
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr8 {
uint8_t len; /* used */
uint8_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr16 {
uint16_t len; /* used */
uint16_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr32 {
uint32_t len; /* used */
uint32_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[];
};
struct __attribute__ ((__packed__)) sdshdr64 {
uint64_t len; /* used */
uint64_t alloc; /* excluding the header and null terminator */
unsigned char flags; /* 3 lsb of type, 5 unused bits */
char buf[]; char buf[];
}; };
#define SDS_TYPE_5 0
#define SDS_TYPE_8 1
#define SDS_TYPE_16 2
#define SDS_TYPE_32 3
#define SDS_TYPE_64 4
#define SDS_TYPE_MASK 7
#define SDS_TYPE_BITS 3
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
static inline size_t sdslen(const sds s) { static inline size_t sdslen(const sds s) {
struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); unsigned char flags = s[-1];
return sh->len; switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8,s)->len;
case SDS_TYPE_16:
return SDS_HDR(16,s)->len;
case SDS_TYPE_32:
return SDS_HDR(32,s)->len;
case SDS_TYPE_64:
return SDS_HDR(64,s)->len;
}
return 0;
} }
static inline size_t sdsavail(const sds s) { static inline size_t sdsavail(const sds s) {
struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); unsigned char flags = s[-1];
return sh->free; switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5: {
return 0;
}
case SDS_TYPE_8: {
SDS_HDR_VAR(8,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_16: {
SDS_HDR_VAR(16,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_32: {
SDS_HDR_VAR(32,s);
return sh->alloc - sh->len;
}
case SDS_TYPE_64: {
SDS_HDR_VAR(64,s);
return sh->alloc - sh->len;
}
}
return 0;
}
static inline void sdssetlen(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s)-1;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->len = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->len = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->len = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->len = newlen;
break;
}
}
static inline void sdsinclen(sds s, size_t inc) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
{
unsigned char *fp = ((unsigned char*)s)-1;
unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc;
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
}
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->len += inc;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->len += inc;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->len += inc;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->len += inc;
break;
}
}
/* sdsalloc() = sdsavail() + sdslen() */
static inline size_t sdsalloc(const sds s) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
return SDS_TYPE_5_LEN(flags);
case SDS_TYPE_8:
return SDS_HDR(8,s)->alloc;
case SDS_TYPE_16:
return SDS_HDR(16,s)->alloc;
case SDS_TYPE_32:
return SDS_HDR(32,s)->alloc;
case SDS_TYPE_64:
return SDS_HDR(64,s)->alloc;
}
return 0;
}
static inline void sdssetalloc(sds s, size_t newlen) {
unsigned char flags = s[-1];
switch(flags&SDS_TYPE_MASK) {
case SDS_TYPE_5:
/* Nothing to do, this type has no total allocation info. */
break;
case SDS_TYPE_8:
SDS_HDR(8,s)->alloc = newlen;
break;
case SDS_TYPE_16:
SDS_HDR(16,s)->alloc = newlen;
break;
case SDS_TYPE_32:
SDS_HDR(32,s)->alloc = newlen;
break;
case SDS_TYPE_64:
SDS_HDR(64,s)->alloc = newlen;
break;
}
} }
sds sdsnewlen(const void *init, size_t initlen); sds sdsnewlen(const void *init, size_t initlen);
sds sdsnew(const char *init); sds sdsnew(const char *init);
sds sdsempty(void); sds sdsempty(void);
size_t sdslen(const sds s);
sds sdsdup(const sds s); sds sdsdup(const sds s);
void sdsfree(sds s); void sdsfree(sds s);
size_t sdsavail(sds s);
sds sdsgrowzero(sds s, size_t len); sds sdsgrowzero(sds s, size_t len);
sds sdscatlen(sds s, const void *t, size_t len); sds sdscatlen(sds s, const void *t, size_t len);
sds sdscat(sds s, const char *t); sds sdscat(sds s, const char *t);
sds sdscpylen(sds s, char *t, size_t len); sds sdscatsds(sds s, const sds t);
sds sdscpy(sds s, char *t); sds sdscpylen(sds s, const char *t, size_t len);
sds sdscpy(sds s, const char *t);
sds sdscatvprintf(sds s, const char *fmt, va_list ap); sds sdscatvprintf(sds s, const char *fmt, va_list ap);
#ifdef __GNUC__ #ifdef __GNUC__
...@@ -73,16 +234,40 @@ sds sdscatprintf(sds s, const char *fmt, ...) ...@@ -73,16 +234,40 @@ sds sdscatprintf(sds s, const char *fmt, ...)
sds sdscatprintf(sds s, const char *fmt, ...); sds sdscatprintf(sds s, const char *fmt, ...);
#endif #endif
sds sdscatfmt(sds s, char const *fmt, ...);
sds sdstrim(sds s, const char *cset); sds sdstrim(sds s, const char *cset);
sds sdsrange(sds s, int start, int end); void sdsrange(sds s, int start, int end);
void sdsupdatelen(sds s); void sdsupdatelen(sds s);
int sdscmp(sds s1, sds s2); void sdsclear(sds s);
sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); int sdscmp(const sds s1, const sds s2);
sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
void sdsfreesplitres(sds *tokens, int count); void sdsfreesplitres(sds *tokens, int count);
void sdstolower(sds s); void sdstolower(sds s);
void sdstoupper(sds s); void sdstoupper(sds s);
sds sdsfromlonglong(long long value); sds sdsfromlonglong(long long value);
sds sdscatrepr(sds s, char *p, size_t len); sds sdscatrepr(sds s, const char *p, size_t len);
sds *sdssplitargs(char *line, int *argc); sds *sdssplitargs(const char *line, int *argc);
sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
sds sdsjoin(char **argv, int argc, char *sep);
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
/* Low level functions exposed to the user API */
sds sdsMakeRoomFor(sds s, size_t addlen);
void sdsIncrLen(sds s, int incr);
sds sdsRemoveFreeSpace(sds s);
size_t sdsAllocSize(sds s);
void *sdsAllocPtr(sds s);
/* Export the allocator used by SDS to the program using SDS.
* Sometimes the program SDS is linked to, may use a different set of
* allocators, but may want to allocate or free things that SDS will
* respectively free or allocate. */
void *sds_malloc(size_t size);
void *sds_realloc(void *ptr, size_t size);
void sds_free(void *ptr);
#ifdef REDIS_TEST
int sdsTest(int argc, char *argv[]);
#endif
#endif #endif
/* SDSLib 2.0 -- A C dynamic strings library
*
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
* Copyright (c) 2015, Oran Agra
* Copyright (c) 2015, Redis Labs, Inc
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* SDS allocator selection.
*
* This file is used in order to change the SDS allocator at compile time.
* Just define the following defines to what you want to use. Also add
* the include of your alternate allocator if needed (not needed in order
* to use the default libc allocator). */
#define s_malloc malloc
#define s_realloc realloc
#define s_free free
...@@ -8,12 +8,15 @@ ...@@ -8,12 +8,15 @@
#include <unistd.h> #include <unistd.h>
#include <signal.h> #include <signal.h>
#include <errno.h> #include <errno.h>
#include <limits.h>
#include "hiredis.h" #include "hiredis.h"
#include "net.h"
enum connection_type { enum connection_type {
CONN_TCP, CONN_TCP,
CONN_UNIX CONN_UNIX,
CONN_FD
}; };
struct config { struct config {
...@@ -22,11 +25,12 @@ struct config { ...@@ -22,11 +25,12 @@ struct config {
struct { struct {
const char *host; const char *host;
int port; int port;
struct timeval timeout;
} tcp; } tcp;
struct { struct {
const char *path; const char *path;
} unix; } unix_sock;
}; };
/* The following lines make up our testing "framework" :) */ /* The following lines make up our testing "framework" :) */
...@@ -40,6 +44,13 @@ static long long usec(void) { ...@@ -40,6 +44,13 @@ static long long usec(void) {
return (((long long)tv.tv_sec)*1000000)+tv.tv_usec; return (((long long)tv.tv_sec)*1000000)+tv.tv_usec;
} }
/* The assert() calls below have side effects, so we need assert()
* even if we are compiling without asserts (-DNDEBUG). */
#ifdef NDEBUG
#undef assert
#define assert(e) (void)(e)
#endif
static redisContext *select_database(redisContext *c) { static redisContext *select_database(redisContext *c) {
redisReply *reply; redisReply *reply;
...@@ -62,7 +73,7 @@ static redisContext *select_database(redisContext *c) { ...@@ -62,7 +73,7 @@ static redisContext *select_database(redisContext *c) {
return c; return c;
} }
static void disconnect(redisContext *c) { static int disconnect(redisContext *c, int keep_fd) {
redisReply *reply; redisReply *reply;
/* Make sure we're on DB 9. */ /* Make sure we're on DB 9. */
...@@ -73,8 +84,11 @@ static void disconnect(redisContext *c) { ...@@ -73,8 +84,11 @@ static void disconnect(redisContext *c) {
assert(reply != NULL); assert(reply != NULL);
freeReplyObject(reply); freeReplyObject(reply);
/* Free the context as well. */ /* Free the context as well, but keep the fd if requested. */
if (keep_fd)
return redisFreeKeepFd(c);
redisFree(c); redisFree(c);
return -1;
} }
static redisContext *connect(struct config config) { static redisContext *connect(struct config config) {
...@@ -83,13 +97,25 @@ static redisContext *connect(struct config config) { ...@@ -83,13 +97,25 @@ static redisContext *connect(struct config config) {
if (config.type == CONN_TCP) { if (config.type == CONN_TCP) {
c = redisConnect(config.tcp.host, config.tcp.port); c = redisConnect(config.tcp.host, config.tcp.port);
} else if (config.type == CONN_UNIX) { } else if (config.type == CONN_UNIX) {
c = redisConnectUnix(config.unix.path); c = redisConnectUnix(config.unix_sock.path);
} else if (config.type == CONN_FD) {
/* Create a dummy connection just to get an fd to inherit */
redisContext *dummy_ctx = redisConnectUnix(config.unix_sock.path);
if (dummy_ctx) {
int fd = disconnect(dummy_ctx, 1);
printf("Connecting to inherited fd %d\n", fd);
c = redisConnectFd(fd);
}
} else { } else {
assert(NULL); assert(NULL);
} }
if (c->err) { if (c == NULL) {
printf("Connection error: can't allocate redis context\n");
exit(1);
} else if (c->err) {
printf("Connection error: %s\n", c->errstr); printf("Connection error: %s\n", c->errstr);
redisFree(c);
exit(1); exit(1);
} }
...@@ -125,13 +151,13 @@ static void test_format_commands(void) { ...@@ -125,13 +151,13 @@ static void test_format_commands(void) {
free(cmd); free(cmd);
test("Format command with %%b string interpolation: "); test("Format command with %%b string interpolation: ");
len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"b\0r",3); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"b\0r",(size_t)3);
test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 && test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nb\0r\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(3+2)+4+(3+2)); len == 4+4+(3+2)+4+(3+2)+4+(3+2));
free(cmd); free(cmd);
test("Format command with %%b and an empty string: "); test("Format command with %%b and an empty string: ");
len = redisFormatCommand(&cmd,"SET %b %b","foo",3,"",0); len = redisFormatCommand(&cmd,"SET %b %b","foo",(size_t)3,"",(size_t)0);
test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 && test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$0\r\n\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(3+2)+4+(0+2)); len == 4+4+(3+2)+4+(3+2)+4+(0+2));
free(cmd); free(cmd);
...@@ -177,7 +203,7 @@ static void test_format_commands(void) { ...@@ -177,7 +203,7 @@ static void test_format_commands(void) {
FLOAT_WIDTH_TEST(double); FLOAT_WIDTH_TEST(double);
test("Format command with invalid printf format: "); test("Format command with invalid printf format: ");
len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",3); len = redisFormatCommand(&cmd,"key:%08p %b",(void*)1234,"foo",(size_t)3);
test_cond(len == -1); test_cond(len == -1);
const char *argv[3]; const char *argv[3];
...@@ -198,12 +224,51 @@ static void test_format_commands(void) { ...@@ -198,12 +224,51 @@ static void test_format_commands(void) {
test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 && test_cond(strncmp(cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2)); len == 4+4+(3+2)+4+(7+2)+4+(3+2));
free(cmd); free(cmd);
sds sds_cmd;
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv without lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,NULL);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(3+2)+4+(3+2));
sdsfree(sds_cmd);
sds_cmd = sdsempty();
test("Format command into sds by passing argc/argv with lengths: ");
len = redisFormatSdsCommandArgv(&sds_cmd,argc,argv,lens);
test_cond(strncmp(sds_cmd,"*3\r\n$3\r\nSET\r\n$7\r\nfoo\0xxx\r\n$3\r\nbar\r\n",len) == 0 &&
len == 4+4+(3+2)+4+(7+2)+4+(3+2));
sdsfree(sds_cmd);
}
static void test_append_formatted_commands(struct config config) {
redisContext *c;
redisReply *reply;
char *cmd;
int len;
c = connect(config);
test("Append format command: ");
len = redisFormatCommand(&cmd, "SET foo bar");
test_cond(redisAppendFormattedCommand(c, cmd, len) == REDIS_OK);
assert(redisGetReply(c, (void*)&reply) == REDIS_OK);
free(cmd);
freeReplyObject(reply);
disconnect(c, 0);
} }
static void test_reply_reader(void) { static void test_reply_reader(void) {
redisReader *reader; redisReader *reader;
void *reply; void *reply;
int ret; int ret;
int i;
test("Error handling in reply parser: "); test("Error handling in reply parser: ");
reader = redisReaderCreate(); reader = redisReaderCreate();
...@@ -225,12 +290,13 @@ static void test_reply_reader(void) { ...@@ -225,12 +290,13 @@ static void test_reply_reader(void) {
strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0); strcasecmp(reader->errstr,"Protocol error, got \"@\" as reply type byte") == 0);
redisReaderFree(reader); redisReaderFree(reader);
test("Set error on nested multi bulks with depth > 2: "); test("Set error on nested multi bulks with depth > 7: ");
reader = redisReaderCreate(); reader = redisReaderCreate();
redisReaderFeed(reader,(char*)"*1\r\n",4);
redisReaderFeed(reader,(char*)"*1\r\n",4); for (i = 0; i < 9; i++) {
redisReaderFeed(reader,(char*)"*1\r\n",4); redisReaderFeed(reader,(char*)"*1\r\n",4);
redisReaderFeed(reader,(char*)"*1\r\n",4); }
ret = redisReaderGetReply(reader,NULL); ret = redisReaderGetReply(reader,NULL);
test_cond(ret == REDIS_ERR && test_cond(ret == REDIS_ERR &&
strncasecmp(reader->errstr,"No support for",14) == 0); strncasecmp(reader->errstr,"No support for",14) == 0);
...@@ -277,14 +343,32 @@ static void test_reply_reader(void) { ...@@ -277,14 +343,32 @@ static void test_reply_reader(void) {
redisReaderFree(reader); redisReaderFree(reader);
} }
static void test_free_null(void) {
void *redisCtx = NULL;
void *reply = NULL;
test("Don't fail when redisFree is passed a NULL value: ");
redisFree(redisCtx);
test_cond(redisCtx == NULL);
test("Don't fail when freeReplyObject is passed a NULL value: ");
freeReplyObject(reply);
test_cond(reply == NULL);
}
static void test_blocking_connection_errors(void) { static void test_blocking_connection_errors(void) {
redisContext *c; redisContext *c;
test("Returns error when host cannot be resolved: "); test("Returns error when host cannot be resolved: ");
c = redisConnect((char*)"idontexist.local", 6379); c = redisConnect((char*)"idontexist.test", 6379);
test_cond(c->err == REDIS_ERR_OTHER && test_cond(c->err == REDIS_ERR_OTHER &&
(strcmp(c->errstr,"Name or service not known") == 0 || (strcmp(c->errstr,"Name or service not known") == 0 ||
strcmp(c->errstr,"Can't resolve: idontexist.local") == 0)); strcmp(c->errstr,"Can't resolve: idontexist.test") == 0 ||
strcmp(c->errstr,"nodename nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"No address associated with hostname") == 0 ||
strcmp(c->errstr,"Temporary failure in name resolution") == 0 ||
strcmp(c->errstr,"hostname nor servname provided, or not known") == 0 ||
strcmp(c->errstr,"no address associated with name") == 0));
redisFree(c); redisFree(c);
test("Returns error when the port is not open: "); test("Returns error when the port is not open: ");
...@@ -293,7 +377,7 @@ static void test_blocking_connection_errors(void) { ...@@ -293,7 +377,7 @@ static void test_blocking_connection_errors(void) {
strcmp(c->errstr,"Connection refused") == 0); strcmp(c->errstr,"Connection refused") == 0);
redisFree(c); redisFree(c);
test("Returns error when the unix socket path doesn't accept connections: "); test("Returns error when the unix_sock socket path doesn't accept connections: ");
c = redisConnectUnix((char*)"/tmp/idontexist.sock"); c = redisConnectUnix((char*)"/tmp/idontexist.sock");
test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */ test_cond(c->err == REDIS_ERR_IO); /* Don't care about the message... */
redisFree(c); redisFree(c);
...@@ -326,7 +410,7 @@ static void test_blocking_connection(struct config config) { ...@@ -326,7 +410,7 @@ static void test_blocking_connection(struct config config) {
freeReplyObject(reply); freeReplyObject(reply);
test("%%b String interpolation works: "); test("%%b String interpolation works: ");
reply = redisCommand(c,"SET %b %b","foo",3,"hello\x00world",11); reply = redisCommand(c,"SET %b %b","foo",(size_t)3,"hello\x00world",(size_t)11);
freeReplyObject(reply); freeReplyObject(reply);
reply = redisCommand(c,"GET foo"); reply = redisCommand(c,"GET foo");
test_cond(reply->type == REDIS_REPLY_STRING && test_cond(reply->type == REDIS_REPLY_STRING &&
...@@ -374,7 +458,53 @@ static void test_blocking_connection(struct config config) { ...@@ -374,7 +458,53 @@ static void test_blocking_connection(struct config config) {
strcasecmp(reply->element[1]->str,"pong") == 0); strcasecmp(reply->element[1]->str,"pong") == 0);
freeReplyObject(reply); freeReplyObject(reply);
disconnect(c); disconnect(c, 0);
}
static void test_blocking_connection_timeouts(struct config config) {
redisContext *c;
redisReply *reply;
ssize_t s;
const char *cmd = "DEBUG SLEEP 3\r\n";
struct timeval tv;
c = connect(config);
test("Successfully completes a command when the timeout is not exceeded: ");
reply = redisCommand(c,"SET foo fast");
freeReplyObject(reply);
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STRING && memcmp(reply->str, "fast", 4) == 0);
freeReplyObject(reply);
disconnect(c, 0);
c = connect(config);
test("Does not return a reply when the command times out: ");
s = write(c->fd, cmd, strlen(cmd));
tv.tv_sec = 0;
tv.tv_usec = 10000;
redisSetTimeout(c, tv);
reply = redisCommand(c, "GET foo");
test_cond(s > 0 && reply == NULL && c->err == REDIS_ERR_IO && strcmp(c->errstr, "Resource temporarily unavailable") == 0);
freeReplyObject(reply);
test("Reconnect properly reconnects after a timeout: ");
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
test("Reconnect properly uses owned parameters: ");
config.tcp.host = "foo";
config.unix_sock.path = "foo";
redisReconnect(c);
reply = redisCommand(c, "PING");
test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0);
freeReplyObject(reply);
disconnect(c, 0);
} }
static void test_blocking_io_errors(struct config config) { static void test_blocking_io_errors(struct config config) {
...@@ -400,7 +530,7 @@ static void test_blocking_io_errors(struct config config) { ...@@ -400,7 +530,7 @@ static void test_blocking_io_errors(struct config config) {
test("Returns I/O error when the connection is lost: "); test("Returns I/O error when the connection is lost: ");
reply = redisCommand(c,"QUIT"); reply = redisCommand(c,"QUIT");
if (major >= 2 && minor > 0) { if (major > 2 || (major == 2 && minor > 0)) {
/* > 2.0 returns OK on QUIT and read() should be issued once more /* > 2.0 returns OK on QUIT and read() should be issued once more
* to know the descriptor is at EOF. */ * to know the descriptor is at EOF. */
test_cond(strcasecmp(reply->str,"OK") == 0 && test_cond(strcasecmp(reply->str,"OK") == 0 &&
...@@ -428,6 +558,30 @@ static void test_blocking_io_errors(struct config config) { ...@@ -428,6 +558,30 @@ static void test_blocking_io_errors(struct config config) {
redisFree(c); redisFree(c);
} }
static void test_invalid_timeout_errors(struct config config) {
redisContext *c;
test("Set error when an invalid timeout usec value is given to redisConnectWithTimeout: ");
config.tcp.timeout.tv_sec = 0;
config.tcp.timeout.tv_usec = 10000001;
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
test("Set error when an invalid timeout sec value is given to redisConnectWithTimeout: ");
config.tcp.timeout.tv_sec = (((LONG_MAX) - 999) / 1000) + 1;
config.tcp.timeout.tv_usec = 0;
c = redisConnectWithTimeout(config.tcp.host, config.tcp.port, config.tcp.timeout);
test_cond(c->err == REDIS_ERR_IO && strcmp(c->errstr, "Invalid timeout specified") == 0);
redisFree(c);
}
static void test_throughput(struct config config) { static void test_throughput(struct config config) {
redisContext *c = connect(config); redisContext *c = connect(config);
redisReply **replies; redisReply **replies;
...@@ -490,7 +644,7 @@ static void test_throughput(struct config config) { ...@@ -490,7 +644,7 @@ static void test_throughput(struct config config) {
free(replies); free(replies);
printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0); printf("\t(%dx LRANGE with 500 elements (pipelined): %.3fs)\n", num, (t2-t1)/1000000.0);
disconnect(c); disconnect(c, 0);
} }
// static long __test_callback_flags = 0; // static long __test_callback_flags = 0;
...@@ -598,11 +752,12 @@ int main(int argc, char **argv) { ...@@ -598,11 +752,12 @@ int main(int argc, char **argv) {
.host = "127.0.0.1", .host = "127.0.0.1",
.port = 6379 .port = 6379
}, },
.unix = { .unix_sock = {
.path = "/tmp/redis.sock" .path = "/tmp/redis.sock"
} }
}; };
int throughput = 1; int throughput = 1;
int test_inherit_fd = 1;
/* Ignore broken pipe signal (for I/O error tests). */ /* Ignore broken pipe signal (for I/O error tests). */
signal(SIGPIPE, SIG_IGN); signal(SIGPIPE, SIG_IGN);
...@@ -618,9 +773,11 @@ int main(int argc, char **argv) { ...@@ -618,9 +773,11 @@ int main(int argc, char **argv) {
cfg.tcp.port = atoi(argv[0]); cfg.tcp.port = atoi(argv[0]);
} else if (argc >= 2 && !strcmp(argv[0],"-s")) { } else if (argc >= 2 && !strcmp(argv[0],"-s")) {
argv++; argc--; argv++; argc--;
cfg.unix.path = argv[0]; cfg.unix_sock.path = argv[0];
} else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) { } else if (argc >= 1 && !strcmp(argv[0],"--skip-throughput")) {
throughput = 0; throughput = 0;
} else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) {
test_inherit_fd = 0;
} else { } else {
fprintf(stderr, "Invalid argument: %s\n", argv[0]); fprintf(stderr, "Invalid argument: %s\n", argv[0]);
exit(1); exit(1);
...@@ -631,19 +788,31 @@ int main(int argc, char **argv) { ...@@ -631,19 +788,31 @@ int main(int argc, char **argv) {
test_format_commands(); test_format_commands();
test_reply_reader(); test_reply_reader();
test_blocking_connection_errors(); test_blocking_connection_errors();
test_free_null();
printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port); printf("\nTesting against TCP connection (%s:%d):\n", cfg.tcp.host, cfg.tcp.port);
cfg.type = CONN_TCP; cfg.type = CONN_TCP;
test_blocking_connection(cfg); test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg); test_blocking_io_errors(cfg);
test_invalid_timeout_errors(cfg);
test_append_formatted_commands(cfg);
if (throughput) test_throughput(cfg); if (throughput) test_throughput(cfg);
printf("\nTesting against Unix socket connection (%s):\n", cfg.unix.path); printf("\nTesting against Unix socket connection (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_UNIX; cfg.type = CONN_UNIX;
test_blocking_connection(cfg); test_blocking_connection(cfg);
test_blocking_connection_timeouts(cfg);
test_blocking_io_errors(cfg); test_blocking_io_errors(cfg);
if (throughput) test_throughput(cfg); if (throughput) test_throughput(cfg);
if (test_inherit_fd) {
printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path);
cfg.type = CONN_FD;
test_blocking_connection(cfg);
}
if (fails) { if (fails) {
printf("*** %d TESTS FAILED ***\n", fails); printf("*** %d TESTS FAILED ***\n", fails);
return 1; return 1;
......
#ifndef _WIN32_HELPER_INCLUDE
#define _WIN32_HELPER_INCLUDE
#ifdef _MSC_VER
#ifndef inline
#define inline __inline
#endif
#ifndef va_copy
#define va_copy(d,s) ((d) = (s))
#endif
#ifndef snprintf
#define snprintf c99_snprintf
__inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap)
{
int count = -1;
if (size != 0)
count = _vsnprintf_s(str, size, _TRUNCATE, format, ap);
if (count == -1)
count = _vscprintf(format, ap);
return count;
}
__inline int c99_snprintf(char* str, size_t size, const char* format, ...)
{
int count;
va_list ap;
va_start(ap, format);
count = c99_vsnprintf(str, size, format, ap);
va_end(ap);
return count;
}
#endif
#endif
#endif
\ No newline at end of file
version: '{build}'
environment:
matrix:
- MSYSTEM: MINGW64
CPU: x86_64
MSVC: amd64
- MSYSTEM: MINGW32
CPU: i686
MSVC: x86
- MSYSTEM: MINGW64
CPU: x86_64
- MSYSTEM: MINGW32
CPU: i686
install:
- set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
- if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
- if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
- pacman --noconfirm -Suy mingw-w64-%CPU%-make
build_script:
- bash -c "autoconf"
- bash -c "./configure"
- mingw32-make -j3
- file lib/jemalloc.dll
- mingw32-make -j3 tests
- mingw32-make -k check
begin-language: "Autoconf-without-aclocal-m4"
args: --no-cache
end-language: "Autoconf-without-aclocal-m4"
/autom4te.cache/ /*.gcov.*
/bin/jemalloc-config
/bin/jemalloc.sh
/bin/jeprof
/config.stamp /config.stamp
/config.log /config.log
/config.status /config.status
/configure
/doc/html.xsl /doc/html.xsl
/doc/manpages.xsl /doc/manpages.xsl
/doc/jemalloc.xml /doc/jemalloc.xml
/doc/jemalloc.html
/doc/jemalloc.3
/jemalloc.pc
/lib/ /lib/
/Makefile /Makefile
/include/jemalloc/internal/jemalloc_internal\.h
/include/jemalloc/internal/size_classes\.h /include/jemalloc/internal/jemalloc_internal.h
/include/jemalloc/jemalloc\.h /include/jemalloc/internal/jemalloc_internal_defs.h
/include/jemalloc/jemalloc_defs\.h /include/jemalloc/internal/private_namespace.h
/include/jemalloc/internal/private_unnamespace.h
/include/jemalloc/internal/public_namespace.h
/include/jemalloc/internal/public_symbols.txt
/include/jemalloc/internal/public_unnamespace.h
/include/jemalloc/internal/size_classes.h
/include/jemalloc/jemalloc.h
/include/jemalloc/jemalloc_defs.h
/include/jemalloc/jemalloc_macros.h
/include/jemalloc/jemalloc_mangle.h
/include/jemalloc/jemalloc_mangle_jet.h
/include/jemalloc/jemalloc_protos.h
/include/jemalloc/jemalloc_protos_jet.h
/include/jemalloc/jemalloc_rename.h
/include/jemalloc/jemalloc_typedefs.h
/src/*.[od] /src/*.[od]
/test/*.[od] /src/*.gcda
/test/*.out /src/*.gcno
/test/[a-zA-Z_]*
!test/*.c /test/test.sh
!test/*.exp test/include/test/jemalloc_test.h
/bin/jemalloc.sh test/include/test/jemalloc_test_defs.h
/test/integration/[A-Za-z]*
!/test/integration/[A-Za-z]*.*
/test/integration/*.[od]
/test/integration/*.gcda
/test/integration/*.gcno
/test/integration/*.out
/test/src/*.[od]
/test/src/*.gcda
/test/src/*.gcno
/test/stress/[A-Za-z]*
!/test/stress/[A-Za-z]*.*
/test/stress/*.[od]
/test/stress/*.gcda
/test/stress/*.gcno
/test/stress/*.out
/test/unit/[A-Za-z]*
!/test/unit/[A-Za-z]*.*
/test/unit/*.[od]
/test/unit/*.gcda
/test/unit/*.gcno
/test/unit/*.out
/VERSION
*.pdb
*.sdf
*.opendb
*.opensdf
*.cachefile
*.suo
*.user
*.sln.docstates
*.tmp
/msvc/Win32/
/msvc/x64/
/msvc/projects/*/*/Debug*/
/msvc/projects/*/*/Release*/
/msvc/projects/*/*/Win32/
/msvc/projects/*/*/x64/
language: c
matrix:
include:
- os: linux
compiler: gcc
- os: linux
compiler: gcc
env:
- EXTRA_FLAGS=-m32
addons:
apt:
packages:
- gcc-multilib
- os: osx
compiler: clang
- os: osx
compiler: clang
env:
- EXTRA_FLAGS=-m32
before_script:
- autoconf
- ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"}
- make -j3
- make -j3 tests
script:
- make check
Unless otherwise specified, files in the jemalloc source distribution are Unless otherwise specified, files in the jemalloc source distribution are
subject to the following license: subject to the following license:
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
Copyright (C) 2002-2012 Jason Evans <jasone@canonware.com>. Copyright (C) 2002-2016 Jason Evans <jasone@canonware.com>.
All rights reserved. All rights reserved.
Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
Copyright (C) 2009-2012 Facebook, Inc. All rights reserved. Copyright (C) 2009-2016 Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met: modification, are permitted provided that the following conditions are met:
......
Following are change highlights associated with official releases. Important Following are change highlights associated with official releases. Important
bug fixes are all mentioned, but internal enhancements are omitted here for bug fixes are all mentioned, but some internal enhancements are omitted here for
brevity (even though they are more fun to write about). Much more detail can be brevity. Much more detail can be found in the git revision history:
found in the git revision history:
http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git https://github.com/jemalloc/jemalloc
git://canonware.com/jemalloc.git
* 4.4.0 (December 3, 2016)
New features:
- Add configure support for *-*-linux-android. (@cferris1000, @jasone)
- Add the --disable-syscall configure option, for use on systems that place
security-motivated limitations on syscall(2). (@jasone)
- Add support for Debian GNU/kFreeBSD. (@thesam)
Optimizations:
- Add extent serial numbers and use them where appropriate as a sort key that
is higher priority than address, so that the allocation policy prefers older
extents. This tends to improve locality (decrease fragmentation) when
memory grows downward. (@jasone)
- Refactor madvise(2) configuration so that MADV_FREE is detected and utilized
on Linux 4.5 and newer. (@jasone)
- Mark partially purged arena chunks as non-huge-page. This improves
interaction with Linux's transparent huge page functionality. (@jasone)
Bug fixes:
- Fix size class computations for edge conditions involving extremely large
allocations. This regression was first released in 4.0.0. (@jasone,
@ingvarha)
- Remove overly restrictive assertions related to the cactive statistic. This
regression was first released in 4.1.0. (@jasone)
- Implement a more reliable detection scheme for os_unfair_lock on macOS.
(@jszakmeister)
* 4.3.1 (November 7, 2016)
Bug fixes:
- Fix a severe virtual memory leak. This regression was first released in
4.3.0. (@interwq, @jasone)
- Refactor atomic and prng APIs to restore support for 32-bit platforms that
use pre-C11 toolchains, e.g. FreeBSD's mips. (@jasone)
* 4.3.0 (November 4, 2016)
This is the first release that passes the test suite for multiple Windows
configurations, thanks in large part to @glandium setting up continuous
integration via AppVeyor (and Travis CI for Linux and OS X).
New features:
- Add "J" (JSON) support to malloc_stats_print(). (@jasone)
- Add Cray compiler support. (@ronawho)
Optimizations:
- Add/use adaptive spinning for bootstrapping and radix tree node
initialization. (@jasone)
Bug fixes:
- Fix large allocation to search starting in the optimal size class heap,
which can substantially reduce virtual memory churn and fragmentation. This
regression was first released in 4.0.0. (@mjp41, @jasone)
- Fix stats.arenas.<i>.nthreads accounting. (@interwq)
- Fix and simplify decay-based purging. (@jasone)
- Make DSS (sbrk(2)-related) operations lockless, which resolves potential
deadlocks during thread exit. (@jasone)
- Fix over-sized allocation of radix tree leaf nodes. (@mjp41, @ogaun,
@jasone)
- Fix over-sized allocation of arena_t (plus associated stats) data
structures. (@jasone, @interwq)
- Fix EXTRA_CFLAGS to not affect configuration. (@jasone)
- Fix a Valgrind integration bug. (@ronawho)
- Disallow 0x5a junk filling when running in Valgrind. (@jasone)
- Fix a file descriptor leak on Linux. This regression was first released in
4.2.0. (@vsarunas, @jasone)
- Fix static linking of jemalloc with glibc. (@djwatson)
- Use syscall(2) rather than {open,read,close}(2) during boot on Linux. This
works around other libraries' system call wrappers performing reentrant
allocation. (@kspinka, @Whissi, @jasone)
- Fix OS X default zone replacement to work with OS X 10.12. (@glandium,
@jasone)
- Fix cached memory management to avoid needless commit/decommit operations
during purging, which resolves permanent virtual memory map fragmentation
issues on Windows. (@mjp41, @jasone)
- Fix TSD fetches to avoid (recursive) allocation. This is relevant to
non-TLS and Windows configurations. (@jasone)
- Fix malloc_conf overriding to work on Windows. (@jasone)
- Forcibly disable lazy-lock on Windows (was forcibly *enabled*). (@jasone)
* 4.2.1 (June 8, 2016)
Bug fixes:
- Fix bootstrapping issues for configurations that require allocation during
tsd initialization (e.g. --disable-tls). (@cferris1000, @jasone)
- Fix gettimeofday() version of nstime_update(). (@ronawho)
- Fix Valgrind regressions in calloc() and chunk_alloc_wrapper(). (@ronawho)
- Fix potential VM map fragmentation regression. (@jasone)
- Fix opt_zero-triggered in-place huge reallocation zeroing. (@jasone)
- Fix heap profiling context leaks in reallocation edge cases. (@jasone)
* 4.2.0 (May 12, 2016)
New features:
- Add the arena.<i>.reset mallctl, which makes it possible to discard all of
an arena's allocations in a single operation. (@jasone)
- Add the stats.retained and stats.arenas.<i>.retained statistics. (@jasone)
- Add the --with-version configure option. (@jasone)
- Support --with-lg-page values larger than actual page size. (@jasone)
Optimizations:
- Use pairing heaps rather than red-black trees for various hot data
structures. (@djwatson, @jasone)
- Streamline fast paths of rtree operations. (@jasone)
- Optimize the fast paths of calloc() and [m,d,sd]allocx(). (@jasone)
- Decommit unused virtual memory if the OS does not overcommit. (@jasone)
- Specify MAP_NORESERVE on Linux if [heuristic] overcommit is active, in order
to avoid unfortunate interactions during fork(2). (@jasone)
Bug fixes:
- Fix chunk accounting related to triggering gdump profiles. (@jasone)
- Link against librt for clock_gettime(2) if glibc < 2.17. (@jasone)
- Scale leak report summary according to sampling probability. (@jasone)
* 4.1.1 (May 3, 2016)
This bugfix release resolves a variety of mostly minor issues, though the
bitmap fix is critical for 64-bit Windows.
Bug fixes:
- Fix the linear scan version of bitmap_sfu() to shift by the proper amount
even when sizeof(long) is not the same as sizeof(void *), as on 64-bit
Windows. (@jasone)
- Fix hashing functions to avoid unaligned memory accesses (and resulting
crashes). This is relevant at least to some ARM-based platforms.
(@rkmisra)
- Fix fork()-related lock rank ordering reversals. These reversals were
unlikely to cause deadlocks in practice except when heap profiling was
enabled and active. (@jasone)
- Fix various chunk leaks in OOM code paths. (@jasone)
- Fix malloc_stats_print() to print opt.narenas correctly. (@jasone)
- Fix MSVC-specific build/test issues. (@rustyx, @yuslepukhin)
- Fix a variety of test failures that were due to test fragility rather than
core bugs. (@jasone)
* 4.1.0 (February 28, 2016)
This release is primarily about optimizations, but it also incorporates a lot
of portability-motivated refactoring and enhancements. Many people worked on
this release, to an extent that even with the omission here of minor changes
(see git revision history), and of the people who reported and diagnosed
issues, so much of the work was contributed that starting with this release,
changes are annotated with author credits to help reflect the collaborative
effort involved.
New features:
- Implement decay-based unused dirty page purging, a major optimization with
mallctl API impact. This is an alternative to the existing ratio-based
unused dirty page purging, and is intended to eventually become the sole
purging mechanism. New mallctls:
+ opt.purge
+ opt.decay_time
+ arena.<i>.decay
+ arena.<i>.decay_time
+ arenas.decay_time
+ stats.arenas.<i>.decay_time
(@jasone, @cevans87)
- Add --with-malloc-conf, which makes it possible to embed a default
options string during configuration. This was motivated by the desire to
specify --with-malloc-conf=purge:decay , since the default must remain
purge:ratio until the 5.0.0 release. (@jasone)
- Add MS Visual Studio 2015 support. (@rustyx, @yuslepukhin)
- Make *allocx() size class overflow behavior defined. The maximum
size class is now less than PTRDIFF_MAX to protect applications against
numerical overflow, and all allocation functions are guaranteed to indicate
errors rather than potentially crashing if the request size exceeds the
maximum size class. (@jasone)
- jeprof:
+ Add raw heap profile support. (@jasone)
+ Add --retain and --exclude for backtrace symbol filtering. (@jasone)
Optimizations:
- Optimize the fast path to combine various bootstrapping and configuration
checks and execute more streamlined code in the common case. (@interwq)
- Use linear scan for small bitmaps (used for small object tracking). In
addition to speeding up bitmap operations on 64-bit systems, this reduces
allocator metadata overhead by approximately 0.2%. (@djwatson)
- Separate arena_avail trees, which substantially speeds up run tree
operations. (@djwatson)
- Use memoization (boot-time-computed table) for run quantization. Separate
arena_avail trees reduced the importance of this optimization. (@jasone)
- Attempt mmap-based in-place huge reallocation. This can dramatically speed
up incremental huge reallocation. (@jasone)
Incompatible changes:
- Make opt.narenas unsigned rather than size_t. (@jasone)
Bug fixes:
- Fix stats.cactive accounting regression. (@rustyx, @jasone)
- Handle unaligned keys in hash(). This caused problems for some ARM systems.
(@jasone, @cferris1000)
- Refactor arenas array. In addition to fixing a fork-related deadlock, this
makes arena lookups faster and simpler. (@jasone)
- Move retained memory allocation out of the default chunk allocation
function, to a location that gets executed even if the application installs
a custom chunk allocation function. This resolves a virtual memory leak.
(@buchgr)
- Fix a potential tsd cleanup leak. (@cferris1000, @jasone)
- Fix run quantization. In practice this bug had no impact unless
applications requested memory with alignment exceeding one page.
(@jasone, @djwatson)
- Fix LinuxThreads-specific bootstrapping deadlock. (Cosmin Paraschiv)
- jeprof:
+ Don't discard curl options if timeout is not defined. (@djwatson)
+ Detect failed profile fetches. (@djwatson)
- Fix stats.arenas.<i>.{dss,lg_dirty_mult,decay_time,pactive,pdirty} for
--disable-stats case. (@jasone)
* 4.0.4 (October 24, 2015)
This bugfix release fixes another xallocx() regression. No other regressions
have come to light in over a month, so this is likely a good starting point
for people who prefer to wait for "dot one" releases with all the major issues
shaken out.
Bug fixes:
- Fix xallocx(..., MALLOCX_ZERO to zero the last full trailing page of large
allocations that have been randomly assigned an offset of 0 when
--enable-cache-oblivious configure option is enabled.
* 4.0.3 (September 24, 2015)
This bugfix release continues the trend of xallocx() and heap profiling fixes.
Bug fixes:
- Fix xallocx(..., MALLOCX_ZERO) to zero all trailing bytes of large
allocations when --enable-cache-oblivious configure option is enabled.
- Fix xallocx(..., MALLOCX_ZERO) to zero trailing bytes of huge allocations
when resizing from/to a size class that is not a multiple of the chunk size.
- Fix prof_tctx_dump_iter() to filter out nodes that were created after heap
profile dumping started.
- Work around a potentially bad thread-specific data initialization
interaction with NPTL (glibc's pthreads implementation).
* 4.0.2 (September 21, 2015)
This bugfix release addresses a few bugs specific to heap profiling.
Bug fixes:
- Fix ixallocx_prof_sample() to never modify nor create sampled small
allocations. xallocx() is in general incapable of moving small allocations,
so this fix removes buggy code without loss of generality.
- Fix irallocx_prof_sample() to always allocate large regions, even when
alignment is non-zero.
- Fix prof_alloc_rollback() to read tdata from thread-specific data rather
than dereferencing a potentially invalid tctx.
* 4.0.1 (September 15, 2015)
This is a bugfix release that is somewhat high risk due to the amount of
refactoring required to address deep xallocx() problems. As a side effect of
these fixes, xallocx() now tries harder to partially fulfill requests for
optional extra space. Note that a couple of minor heap profiling
optimizations are included, but these are better thought of as performance
fixes that were integral to disovering most of the other bugs.
Optimizations:
- Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the
fast path when heap profiling is enabled. Additionally, split a special
case out into arena_prof_tctx_reset(), which also avoids chunk metadata
reads.
- Optimize irallocx_prof() to optimistically update the sampler state. The
prior implementation appears to have been a holdover from when
rallocx()/xallocx() functionality was combined as rallocm().
Bug fixes:
- Fix TLS configuration such that it is enabled by default for platforms on
which it works correctly.
- Fix arenas_cache_cleanup() and arena_get_hard() to handle
allocation/deallocation within the application's thread-specific data
cleanup functions even after arenas_cache is torn down.
- Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS.
- Fix chunk purge hook calls for in-place huge shrinking reallocation to
specify the old chunk size rather than the new chunk size. This bug caused
no correctness issues for the default chunk purge function, but was
visible to custom functions set via the "arena.<i>.chunk_hooks" mallctl.
- Fix heap profiling bugs:
+ Fix heap profiling to distinguish among otherwise identical sample sites
with interposed resets (triggered via the "prof.reset" mallctl). This bug
could cause data structure corruption that would most likely result in a
segfault.
+ Fix irealloc_prof() to prof_alloc_rollback() on OOM.
+ Make one call to prof_active_get_unlocked() per allocation event, and use
the result throughout the relevant functions that handle an allocation
event. Also add a missing check in prof_realloc(). These fixes protect
allocation events against concurrent prof_active changes.
+ Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample()
in the correct order.
+ Fix prof_realloc() to call prof_free_sampled_object() after calling
prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were
the same, the tctx could have been prematurely destroyed.
- Fix portability bugs:
+ Don't bitshift by negative amounts when encoding/decoding run sizes in
chunk header maps. This affected systems with page sizes greater than 8
KiB.
+ Rename index_t to szind_t to avoid an existing type on Solaris.
+ Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to
match glibc and avoid compilation errors when including both
jemalloc/jemalloc.h and malloc.h in C++ code.
+ Don't assume that /bin/sh is appropriate when running size_classes.sh
during configuration.
+ Consider __sparcv9 a synonym for __sparc64__ when defining LG_QUANTUM.
+ Link tests to librt if it contains clock_gettime(2).
* 4.0.0 (August 17, 2015)
This version contains many speed and space optimizations, both minor and
major. The major themes are generalization, unification, and simplification.
Although many of these optimizations cause no visible behavior change, their
cumulative effect is substantial.
New features:
- Normalize size class spacing to be consistent across the complete size
range. By default there are four size classes per size doubling, but this
is now configurable via the --with-lg-size-class-group option. Also add the
--with-lg-page, --with-lg-page-sizes, --with-lg-quantum, and
--with-lg-tiny-min options, which can be used to tweak page and size class
settings. Impacts:
+ Worst case performance for incrementally growing/shrinking reallocation
is improved because there are far fewer size classes, and therefore
copying happens less often.
+ Internal fragmentation is limited to 20% for all but the smallest size
classes (those less than four times the quantum). (1B + 4 KiB)
and (1B + 4 MiB) previously suffered nearly 50% internal fragmentation.
+ Chunk fragmentation tends to be lower because there are fewer distinct run
sizes to pack.
- Add support for explicit tcaches. The "tcache.create", "tcache.flush", and
"tcache.destroy" mallctls control tcache lifetime and flushing, and the
MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to the *allocx() API
control which tcache is used for each operation.
- Implement per thread heap profiling, as well as the ability to
enable/disable heap profiling on a per thread basis. Add the "prof.reset",
"prof.lg_sample", "thread.prof.name", "thread.prof.active",
"opt.prof_thread_active_init", "prof.thread_active_init", and
"thread.prof.active" mallctls.
- Add support for per arena application-specified chunk allocators, configured
via the "arena.<i>.chunk_hooks" mallctl.
- Refactor huge allocation to be managed by arenas, so that arenas now
function as general purpose independent allocators. This is important in
the context of user-specified chunk allocators, aside from the scalability
benefits. Related new statistics:
+ The "stats.arenas.<i>.huge.allocated", "stats.arenas.<i>.huge.nmalloc",
"stats.arenas.<i>.huge.ndalloc", and "stats.arenas.<i>.huge.nrequests"
mallctls provide high level per arena huge allocation statistics.
+ The "arenas.nhchunks", "arenas.hchunk.<i>.size",
"stats.arenas.<i>.hchunks.<j>.nmalloc",
"stats.arenas.<i>.hchunks.<j>.ndalloc",
"stats.arenas.<i>.hchunks.<j>.nrequests", and
"stats.arenas.<i>.hchunks.<j>.curhchunks" mallctls provide per size class
statistics.
- Add the 'util' column to malloc_stats_print() output, which reports the
proportion of available regions that are currently in use for each small
size class.
- Add "alloc" and "free" modes for for junk filling (see the "opt.junk"
mallctl), so that it is possible to separately enable junk filling for
allocation versus deallocation.
- Add the jemalloc-config script, which provides information about how
jemalloc was configured, and how to integrate it into application builds.
- Add metadata statistics, which are accessible via the "stats.metadata",
"stats.arenas.<i>.metadata.mapped", and
"stats.arenas.<i>.metadata.allocated" mallctls.
- Add the "stats.resident" mallctl, which reports the upper limit of
physically resident memory mapped by the allocator.
- Add per arena control over unused dirty page purging, via the
"arenas.lg_dirty_mult", "arena.<i>.lg_dirty_mult", and
"stats.arenas.<i>.lg_dirty_mult" mallctls.
- Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump
feature on/off during program execution.
- Add sdallocx(), which implements sized deallocation. The primary
optimization over dallocx() is the removal of a metadata read, which often
suffers an L1 cache miss.
- Add missing header includes in jemalloc/jemalloc.h, so that applications
only have to #include <jemalloc/jemalloc.h>.
- Add support for additional platforms:
+ Bitrig
+ Cygwin
+ DragonFlyBSD
+ iOS
+ OpenBSD
+ OpenRISC/or1k
Optimizations:
- Maintain dirty runs in per arena LRUs rather than in per arena trees of
dirty-run-containing chunks. In practice this change significantly reduces
dirty page purging volume.
- Integrate whole chunks into the unused dirty page purging machinery. This
reduces the cost of repeated huge allocation/deallocation, because it
effectively introduces a cache of chunks.
- Split the arena chunk map into two separate arrays, in order to increase
cache locality for the frequently accessed bits.
- Move small run metadata out of runs, into arena chunk headers. This reduces
run fragmentation, smaller runs reduce external fragmentation for small size
classes, and packed (less uniformly aligned) metadata layout improves CPU
cache set distribution.
- Randomly distribute large allocation base pointer alignment relative to page
boundaries in order to more uniformly utilize CPU cache sets. This can be
disabled via the --disable-cache-oblivious configure option, and queried via
the "config.cache_oblivious" mallctl.
- Micro-optimize the fast paths for the public API functions.
- Refactor thread-specific data to reside in a single structure. This assures
that only a single TLS read is necessary per call into the public API.
- Implement in-place huge allocation growing and shrinking.
- Refactor rtree (radix tree for chunk lookups) to be lock-free, and make
additional optimizations that reduce maximum lookup depth to one or two
levels. This resolves what was a concurrency bottleneck for per arena huge
allocation, because a global data structure is critical for determining
which arenas own which huge allocations.
Incompatible changes:
- Replace --enable-cc-silence with --disable-cc-silence to suppress spurious
warnings by default.
- Assure that the constness of malloc_usable_size()'s return type matches that
of the system implementation.
- Change the heap profile dump format to support per thread heap profiling,
rename pprof to jeprof, and enhance it with the --thread=<n> option. As a
result, the bundled jeprof must now be used rather than the upstream
(gperftools) pprof.
- Disable "opt.prof_final" by default, in order to avoid atexit(3), which can
internally deadlock on some platforms.
- Change the "arenas.nlruns" mallctl type from size_t to unsigned.
- Replace the "stats.arenas.<i>.bins.<j>.allocated" mallctl with
"stats.arenas.<i>.bins.<j>.curregs".
- Ignore MALLOC_CONF in set{uid,gid,cap} binaries.
- Ignore MALLOCX_ARENA(a) in dallocx(), in favor of using the
MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to control tcache usage.
Removed features:
- Remove the *allocm() API, which is superseded by the *allocx() API.
- Remove the --enable-dss options, and make dss non-optional on all platforms
which support sbrk(2).
- Remove the "arenas.purge" mallctl, which was obsoleted by the
"arena.<i>.purge" mallctl in 3.1.0.
- Remove the unnecessary "opt.valgrind" mallctl; jemalloc automatically
detects whether it is running inside Valgrind.
- Remove the "stats.huge.allocated", "stats.huge.nmalloc", and
"stats.huge.ndalloc" mallctls.
- Remove the --enable-mremap option.
- Remove the "stats.chunks.current", "stats.chunks.total", and
"stats.chunks.high" mallctls.
Bug fixes:
- Fix the cactive statistic to decrease (rather than increase) when active
memory decreases. This regression was first released in 3.5.0.
- Fix OOM handling in memalign() and valloc(). A variant of this bug existed
in all releases since 2.0.0, which introduced these functions.
- Fix an OOM-related regression in arena_tcache_fill_small(), which could
cause cache corruption on OOM. This regression was present in all releases
from 2.2.0 through 3.6.0.
- Fix size class overflow handling for malloc(), posix_memalign(), memalign(),
calloc(), and realloc() when profiling is enabled.
- Fix the "arena.<i>.dss" mallctl to return an error if "primary" or
"secondary" precedence is specified, but sbrk(2) is not supported.
- Fix fallback lg_floor() implementations to handle extremely large inputs.
- Ensure the default purgeable zone is after the default zone on OS X.
- Fix latent bugs in atomic_*().
- Fix the "arena.<i>.dss" mallctl to handle read-only calls.
- Fix tls_model configuration to enable the initial-exec model when possible.
- Mark malloc_conf as a weak symbol so that the application can override it.
- Correctly detect glibc's adaptive pthread mutexes.
- Fix the --without-export configure option.
* 3.6.0 (March 31, 2014)
This version contains a critical bug fix for a regression present in 3.5.0 and
3.5.1.
Bug fixes:
- Fix a regression in arena_chunk_alloc() that caused crashes during
small/large allocation if chunk allocation failed. In the absence of this
bug, chunk allocation failure would result in allocation failure, e.g. NULL
return from malloc(). This regression was introduced in 3.5.0.
- Fix backtracing for gcc intrinsics-based backtracing by specifying
-fno-omit-frame-pointer to gcc. Note that the application (and all the
libraries it links to) must also be compiled with this option for
backtracing to be reliable.
- Use dss allocation precedence for huge allocations as well as small/large
allocations.
- Fix test assertion failure message formatting. This bug did not manifest on
x86_64 systems because of implementation subtleties in va_list.
- Fix inconsequential test failures for hash and SFMT code.
New features:
- Support heap profiling on FreeBSD. This feature depends on the proc
filesystem being mounted during heap profile dumping.
* 3.5.1 (February 25, 2014)
This version primarily addresses minor bugs in test code.
Bug fixes:
- Configure Solaris/Illumos to use MADV_FREE.
- Fix junk filling for mremap(2)-based huge reallocation. This is only
relevant if configuring with the --enable-mremap option specified.
- Avoid compilation failure if 'restrict' C99 keyword is not supported by the
compiler.
- Add a configure test for SSE2 rather than assuming it is usable on i686
systems. This fixes test compilation errors, especially on 32-bit Linux
systems.
- Fix mallctl argument size mismatches (size_t vs. uint64_t) in the stats unit
test.
- Fix/remove flawed alignment-related overflow tests.
- Prevent compiler optimizations that could change backtraces in the
prof_accum unit test.
* 3.5.0 (January 22, 2014)
This version focuses on refactoring and automated testing, though it also
includes some non-trivial heap profiling optimizations not mentioned below.
New features:
- Add the *allocx() API, which is a successor to the experimental *allocm()
API. The *allocx() functions are slightly simpler to use because they have
fewer parameters, they directly return the results of primary interest, and
mallocx()/rallocx() avoid the strict aliasing pitfall that
allocm()/rallocm() share with posix_memalign(). Note that *allocm() is
slated for removal in the next non-bugfix release.
- Add support for LinuxThreads.
Bug fixes:
- Unless heap profiling is enabled, disable floating point code and don't link
with libm. This, in combination with e.g. EXTRA_CFLAGS=-mno-sse on x64
systems, makes it possible to completely disable floating point register
use. Some versions of glibc neglect to save/restore caller-saved floating
point registers during dynamic lazy symbol loading, and the symbol loading
code uses whatever malloc the application happens to have linked/loaded
with, the result being potential floating point register corruption.
- Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling
backtrace creation in imemalign(). This bug impacted posix_memalign() and
aligned_alloc().
- Fix a file descriptor leak in a prof_dump_maps() error path.
- Fix prof_dump() to close the dump file descriptor for all relevant error
paths.
- Fix rallocm() to use the arena specified by the ALLOCM_ARENA(s) flag for
allocation, not just deallocation.
- Fix a data race for large allocation stats counters.
- Fix a potential infinite loop during thread exit. This bug occurred on
Solaris, and could affect other platforms with similar pthreads TSD
implementations.
- Don't junk-fill reallocations unless usable size changes. This fixes a
violation of the *allocx()/*allocm() semantics.
- Fix growing large reallocation to junk fill new space.
- Fix huge deallocation to junk fill when munmap is disabled.
- Change the default private namespace prefix from empty to je_, and change
--with-private-namespace-prefix so that it prepends an additional prefix
rather than replacing je_. This reduces the likelihood of applications
which statically link jemalloc experiencing symbol name collisions.
- Add missing private namespace mangling (relevant when
--with-private-namespace is specified).
- Add and use JEMALLOC_INLINE_C so that static inline functions are marked as
static even for debug builds.
- Add a missing mutex unlock in a malloc_init_hard() error path. In practice
this error path is never executed.
- Fix numerous bugs in malloc_strotumax() error handling/reporting. These
bugs had no impact except for malformed inputs.
- Fix numerous bugs in malloc_snprintf(). These bugs were not exercised by
existing calls, so they had no impact.
* 3.4.1 (October 20, 2013)
Bug fixes:
- Fix a race in the "arenas.extend" mallctl that could cause memory corruption
of internal data structures and subsequent crashes.
- Fix Valgrind integration flaws that caused Valgrind warnings about reads of
uninitialized memory in:
+ arena chunk headers
+ internal zero-initialized data structures (relevant to tcache and prof
code)
- Preserve errno during the first allocation. A readlink(2) call during
initialization fails unless /etc/malloc.conf exists, so errno was typically
set during the first allocation prior to this fix.
- Fix compilation warnings reported by gcc 4.8.1.
* 3.4.0 (June 2, 2013)
This version is essentially a small bugfix release, but the addition of
aarch64 support requires that the minor version be incremented.
Bug fixes:
- Fix race-triggered deadlocks in chunk_record(). These deadlocks were
typically triggered by multiple threads concurrently deallocating huge
objects.
New features:
- Add support for the aarch64 architecture.
* 3.3.1 (March 6, 2013)
This version fixes bugs that are typically encountered only when utilizing
custom run-time options.
Bug fixes:
- Fix a locking order bug that could cause deadlock during fork if heap
profiling were enabled.
- Fix a chunk recycling bug that could cause the allocator to lose track of
whether a chunk was zeroed. On FreeBSD, NetBSD, and OS X, it could cause
corruption if allocating via sbrk(2) (unlikely unless running with the
"dss:primary" option specified). This was completely harmless on Linux
unless using mlockall(2) (and unlikely even then, unless the
--disable-munmap configure option or the "dss:primary" option was
specified). This regression was introduced in 3.1.0 by the
mlockall(2)/madvise(2) interaction fix.
- Fix TLS-related memory corruption that could occur during thread exit if the
thread never allocated memory. Only the quarantine and prof facilities were
susceptible.
- Fix two quarantine bugs:
+ Internal reallocation of the quarantined object array leaked the old
array.
+ Reallocation failure for internal reallocation of the quarantined object
array (very unlikely) resulted in memory corruption.
- Fix Valgrind integration to annotate all internally allocated memory in a
way that keeps Valgrind happy about internal data structure access.
- Fix building for s390 systems.
* 3.3.0 (January 23, 2013)
This version includes a few minor performance improvements in addition to the
listed new features and bug fixes.
New features:
- Add clipping support to lg_chunk option processing.
- Add the --enable-ivsalloc option.
- Add the --without-export option.
- Add the --disable-zone-allocator option.
Bug fixes:
- Fix "arenas.extend" mallctl to output the number of arenas.
- Fix chunk_recycle() to unconditionally inform Valgrind that returned memory
is undefined.
- Fix build break on FreeBSD related to alloca.h.
* 3.2.0 (November 9, 2012) * 3.2.0 (November 9, 2012)
...@@ -348,7 +976,7 @@ found in the git revision history: ...@@ -348,7 +976,7 @@ found in the git revision history:
- Make it possible for the application to manually flush a thread's cache, via - Make it possible for the application to manually flush a thread's cache, via
the "tcache.flush" mallctl. the "tcache.flush" mallctl.
- Base maximum dirty page count on proportion of active memory. - Base maximum dirty page count on proportion of active memory.
- Compute various addtional run-time statistics, including per size class - Compute various additional run-time statistics, including per size class
statistics for large objects. statistics for large objects.
- Expose malloc_stats_print(), which can be called repeatedly by the - Expose malloc_stats_print(), which can be called repeatedly by the
application. application.
......
Building and installing jemalloc can be as simple as typing the following while Building and installing a packaged release of jemalloc can be as simple as
in the root directory of the source tree: typing the following while in the root directory of the source tree:
./configure ./configure
make make
make install make install
If building from unpackaged developer sources, the simplest command sequence
that might work is:
./autogen.sh
make dist
make
make install
Note that documentation is not built by the default target because doing so
would create a dependency on xsltproc in packaged releases, hence the
requirement to either run 'make dist' or avoid installing docs via the various
install_* targets documented below.
=== Advanced configuration ===================================================== === Advanced configuration =====================================================
The 'configure' script supports numerous options that allow control of which The 'configure' script supports numerous options that allow control of which
...@@ -22,6 +35,10 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -22,6 +35,10 @@ any of the following arguments (not a definitive list) to 'configure':
will cause files to be installed into /usr/local/include, /usr/local/lib, will cause files to be installed into /usr/local/include, /usr/local/lib,
and /usr/local/man. and /usr/local/man.
--with-version=<major>.<minor>.<bugfix>-<nrev>-g<gid>
Use the specified version string rather than trying to generate one (if in
a git repository) or use existing the VERSION file (if present).
--with-rpath=<colon-separated-rpath> --with-rpath=<colon-separated-rpath>
Embed one or more library paths, so that libjemalloc can find the libraries Embed one or more library paths, so that libjemalloc can find the libraries
it is linked to. This works only on ELF-based systems. it is linked to. This works only on ELF-based systems.
...@@ -55,30 +72,62 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -55,30 +72,62 @@ any of the following arguments (not a definitive list) to 'configure':
jemalloc overlays the default malloc zone, but makes no attempt to actually jemalloc overlays the default malloc zone, but makes no attempt to actually
replace the "malloc", "calloc", etc. symbols. replace the "malloc", "calloc", etc. symbols.
--without-export
Don't export public APIs. This can be useful when building jemalloc as a
static library, or to avoid exporting public APIs when using the zone
allocator on OSX.
--with-private-namespace=<prefix> --with-private-namespace=<prefix>
Prefix all library-private APIs with <prefix>. For shared libraries, Prefix all library-private APIs with <prefix>je_. For shared libraries,
symbol visibility mechanisms prevent these symbols from being exported, but symbol visibility mechanisms prevent these symbols from being exported, but
for static libraries, naming collisions are a real possibility. By for static libraries, naming collisions are a real possibility. By
default, the prefix is "" (empty string). default, <prefix> is empty, which results in a symbol prefix of je_ .
--with-install-suffix=<suffix> --with-install-suffix=<suffix>
Append <suffix> to the base name of all installed files, such that multiple Append <suffix> to the base name of all installed files, such that multiple
versions of jemalloc can coexist in the same installation directory. For versions of jemalloc can coexist in the same installation directory. For
example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0. example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0.
--enable-cc-silence --with-malloc-conf=<malloc_conf>
Enable code that silences non-useful compiler warnings. This is helpful Embed <malloc_conf> as a run-time options string that is processed prior to
when trying to tell serious warnings from those due to compiler the malloc_conf global variable, the /etc/malloc.conf symlink, and the
limitations, but it potentially incurs a performance penalty. MALLOC_CONF environment variable. For example, to change the default chunk
size to 256 KiB:
--with-malloc-conf=lg_chunk:18
--disable-cc-silence
Disable code that silences non-useful compiler warnings. This is mainly
useful during development when auditing the set of warnings that are being
silenced.
--enable-debug --enable-debug
Enable assertions and validation code. This incurs a substantial Enable assertions and validation code. This incurs a substantial
performance hit, but is very useful during application development. performance hit, but is very useful during application development.
Implies --enable-ivsalloc.
--enable-code-coverage
Enable code coverage support, for use during jemalloc test development.
Additional testing targets are available if this option is enabled:
coverage
coverage_unit
coverage_integration
coverage_stress
These targets do not clear code coverage results from previous runs, and
there are interactions between the various coverage targets, so it is
usually advisable to run 'make clean' between repeated code coverage runs.
--disable-stats --disable-stats
Disable statistics gathering functionality. See the "opt.stats_print" Disable statistics gathering functionality. See the "opt.stats_print"
option documentation for usage details. option documentation for usage details.
--enable-ivsalloc
Enable validation code, which verifies that pointers reside within
jemalloc-owned chunks before dereferencing them. This incurs a minor
performance hit.
--enable-prof --enable-prof
Enable heap profiling and leak detection functionality. See the "opt.prof" Enable heap profiling and leak detection functionality. See the "opt.prof"
option documentation for usage details. When enabled, there are several option documentation for usage details. When enabled, there are several
...@@ -108,12 +157,6 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -108,12 +157,6 @@ any of the following arguments (not a definitive list) to 'configure':
released in bulk, thus reducing the total number of mutex operations. See released in bulk, thus reducing the total number of mutex operations. See
the "opt.tcache" option for usage details. the "opt.tcache" option for usage details.
--enable-mremap
Enable huge realloc() via mremap(2). mremap() is disabled by default
because the flavor used is specific to Linux, which has a quirk in its
virtual memory allocation algorithm that causes semi-permanent VM map holes
under normal jemalloc operation.
--disable-munmap --disable-munmap
Disable virtual memory deallocation via munmap(2); instead keep track of Disable virtual memory deallocation via munmap(2); instead keep track of
the virtual memory for later use. munmap() is disabled by default (i.e. the virtual memory for later use. munmap() is disabled by default (i.e.
...@@ -121,10 +164,6 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -121,10 +164,6 @@ any of the following arguments (not a definitive list) to 'configure':
memory allocation algorithm that causes semi-permanent VM map holes under memory allocation algorithm that causes semi-permanent VM map holes under
normal jemalloc operation. normal jemalloc operation.
--enable-dss
Enable support for page allocation/deallocation via sbrk(2), in addition to
mmap(2).
--disable-fill --disable-fill
Disable support for junk/zero filling of memory, quarantine, and redzones. Disable support for junk/zero filling of memory, quarantine, and redzones.
See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
...@@ -133,8 +172,9 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -133,8 +172,9 @@ any of the following arguments (not a definitive list) to 'configure':
--disable-valgrind --disable-valgrind
Disable support for Valgrind. Disable support for Valgrind.
--disable-experimental --disable-zone-allocator
Disable support for the experimental API (*allocm()). Disable zone allocator for Darwin. This means jemalloc won't be hooked as
the default allocator on OSX/iOS.
--enable-utrace --enable-utrace
Enable utrace(2)-based allocation tracing. This feature is not broadly Enable utrace(2)-based allocation tracing. This feature is not broadly
...@@ -157,10 +197,111 @@ any of the following arguments (not a definitive list) to 'configure': ...@@ -157,10 +197,111 @@ any of the following arguments (not a definitive list) to 'configure':
thread-local variables via the __thread keyword. If TLS is available, thread-local variables via the __thread keyword. If TLS is available,
jemalloc uses it for several purposes. jemalloc uses it for several purposes.
--disable-cache-oblivious
Disable cache-oblivious large allocation alignment for large allocation
requests with no alignment constraints. If this feature is disabled, all
large allocations are page-aligned as an implementation artifact, which can
severely harm CPU cache utilization. However, the cache-oblivious layout
comes at the cost of one extra page per large allocation, which in the
most extreme case increases physical memory usage for the 16 KiB size class
to 20 KiB.
--disable-syscall
Disable use of syscall(2) rather than {open,read,write,close}(2). This is
intended as a workaround for systems that place security limitations on
syscall(2).
--with-xslroot=<path> --with-xslroot=<path>
Specify where to find DocBook XSL stylesheets when building the Specify where to find DocBook XSL stylesheets when building the
documentation. documentation.
--with-lg-page=<lg-page>
Specify the base 2 log of the system page size. This option is only useful
when cross compiling, since the configure script automatically determines
the host's page size by default.
--with-lg-page-sizes=<lg-page-sizes>
Specify the comma-separated base 2 logs of the page sizes to support. This
option may be useful when cross-compiling in combination with
--with-lg-page, but its primary use case is for integration with FreeBSD's
libc, wherein jemalloc is embedded.
--with-lg-size-class-group=<lg-size-class-group>
Specify the base 2 log of how many size classes to use for each doubling in
size. By default jemalloc uses <lg-size-class-group>=2, which results in
e.g. the following size classes:
[...], 64,
80, 96, 112, 128,
160, [...]
<lg-size-class-group>=3 results in e.g. the following size classes:
[...], 64,
72, 80, 88, 96, 104, 112, 120, 128,
144, [...]
The minimal <lg-size-class-group>=0 causes jemalloc to only provide size
classes that are powers of 2:
[...],
64,
128,
256,
[...]
An implementation detail currently limits the total number of small size
classes to 255, and a compilation error will result if the
<lg-size-class-group> you specify cannot be supported. The limit is
roughly <lg-size-class-group>=4, depending on page size.
--with-lg-quantum=<lg-quantum>
Specify the base 2 log of the minimum allocation alignment. jemalloc needs
to know the minimum alignment that meets the following C standard
requirement (quoted from the April 12, 2011 draft of the C11 standard):
The pointer returned if the allocation succeeds is suitably aligned so
that it may be assigned to a pointer to any type of object with a
fundamental alignment requirement and then used to access such an object
or an array of such objects in the space allocated [...]
This setting is architecture-specific, and although jemalloc includes known
safe values for the most commonly used modern architectures, there is a
wrinkle related to GNU libc (glibc) that may impact your choice of
<lg-quantum>. On most modern architectures, this mandates 16-byte alignment
(<lg-quantum>=4), but the glibc developers chose not to meet this
requirement for performance reasons. An old discussion can be found at
https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc,
jemalloc does follow the C standard by default (caveat: jemalloc
technically cheats if --with-lg-tiny-min is smaller than
--with-lg-quantum), but the fact that Linux systems already work around
this allocator noncompliance means that it is generally safe in practice to
let jemalloc's minimum alignment follow glibc's lead. If you specify
--with-lg-quantum=3 during configuration, jemalloc will provide additional
size classes that are not 16-byte-aligned (24, 40, and 56, assuming
--with-lg-size-class-group=2).
--with-lg-tiny-min=<lg-tiny-min>
Specify the base 2 log of the minimum tiny size class to support. Tiny
size classes are powers of 2 less than the quantum, and are only
incorporated if <lg-tiny-min> is less than <lg-quantum> (see
--with-lg-quantum). Tiny size classes technically violate the C standard
requirement for minimum alignment, and crashes could conceivably result if
the compiler were to generate instructions that made alignment assumptions,
both because illegal instruction traps could result, and because accesses
could straddle page boundaries and cause segmentation faults due to
accessing unmapped addresses.
The default of <lg-tiny-min>=3 works well in practice even on architectures
that technically require 16-byte alignment, probably for the same reason
--with-lg-quantum=3 works. Smaller tiny size classes can, and will, cause
crashes (see https://bugzilla.mozilla.org/show_bug.cgi?id=691003 for an
example).
This option is rarely useful, and is mainly provided as documentation of a
subtle implementation detail. If you do use this option, specify a
value in [3, ..., <lg-quantum>].
The following environment variables (not a definitive list) impact configure's The following environment variables (not a definitive list) impact configure's
behavior: behavior:
...@@ -191,6 +332,15 @@ LDFLAGS="?" ...@@ -191,6 +332,15 @@ LDFLAGS="?"
PATH="?" PATH="?"
'configure' uses this to find programs. 'configure' uses this to find programs.
In some cases it may be necessary to work around configuration results that do
not match reality. For example, Linux 4.5 added support for the MADV_FREE flag
to madvise(2), which can cause problems if building on a host with MADV_FREE
support and deploying to a target without. To work around this, use a cache
file to override the relevant configuration variable defined in configure.ac,
e.g.:
echo "je_cv_madv_free=no" > config.cache && ./configure -C
=== Advanced compilation ======================================================= === Advanced compilation =======================================================
To build only parts of jemalloc, use the following targets: To build only parts of jemalloc, use the following targets:
......
...@@ -24,7 +24,8 @@ abs_objroot := @abs_objroot@ ...@@ -24,7 +24,8 @@ abs_objroot := @abs_objroot@
# Build parameters. # Build parameters.
CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include
CFLAGS := @CFLAGS@ EXTRA_CFLAGS := @EXTRA_CFLAGS@
CFLAGS := @CFLAGS@ $(EXTRA_CFLAGS)
LDFLAGS := @LDFLAGS@ LDFLAGS := @LDFLAGS@
EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ EXTRA_LDFLAGS := @EXTRA_LDFLAGS@
LIBS := @LIBS@ LIBS := @LIBS@
...@@ -42,19 +43,29 @@ XSLTPROC := @XSLTPROC@ ...@@ -42,19 +43,29 @@ XSLTPROC := @XSLTPROC@
AUTOCONF := @AUTOCONF@ AUTOCONF := @AUTOCONF@
_RPATH = @RPATH@ _RPATH = @RPATH@
RPATH = $(if $(1),$(call _RPATH,$(1))) RPATH = $(if $(1),$(call _RPATH,$(1)))
cfghdrs_in := @cfghdrs_in@ cfghdrs_in := $(addprefix $(srcroot),@cfghdrs_in@)
cfghdrs_out := @cfghdrs_out@ cfghdrs_out := @cfghdrs_out@
cfgoutputs_in := @cfgoutputs_in@ cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@)
cfgoutputs_out := @cfgoutputs_out@ cfgoutputs_out := @cfgoutputs_out@
enable_autogen := @enable_autogen@ enable_autogen := @enable_autogen@
enable_experimental := @enable_experimental@ enable_code_coverage := @enable_code_coverage@
enable_prof := @enable_prof@
enable_valgrind := @enable_valgrind@
enable_zone_allocator := @enable_zone_allocator@
MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF
link_whole_archive := @link_whole_archive@
DSO_LDFLAGS = @DSO_LDFLAGS@ DSO_LDFLAGS = @DSO_LDFLAGS@
SOREV = @SOREV@ SOREV = @SOREV@
PIC_CFLAGS = @PIC_CFLAGS@ PIC_CFLAGS = @PIC_CFLAGS@
CTARGET = @CTARGET@ CTARGET = @CTARGET@
LDTARGET = @LDTARGET@ LDTARGET = @LDTARGET@
TEST_LD_MODE = @TEST_LD_MODE@
MKLIB = @MKLIB@ MKLIB = @MKLIB@
AR = @AR@
ARFLAGS = @ARFLAGS@
CC_MM = @CC_MM@ CC_MM = @CC_MM@
LM := @LM@
INSTALL = @INSTALL@
ifeq (macho, $(ABI)) ifeq (macho, $(ABI))
TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib" TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib"
...@@ -69,19 +80,41 @@ endif ...@@ -69,19 +80,41 @@ endif
LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
# Lists of files. # Lists of files.
BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
CHDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h \ C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
$(objroot)include/jemalloc/jemalloc_defs$(install_suffix).h C_SRCS := $(srcroot)src/jemalloc.c \
CSRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c $(srcroot)src/atomic.c \ $(srcroot)src/arena.c \
$(srcroot)src/base.c $(srcroot)src/bitmap.c $(srcroot)src/chunk.c \ $(srcroot)src/atomic.c \
$(srcroot)src/chunk_dss.c $(srcroot)src/chunk_mmap.c \ $(srcroot)src/base.c \
$(srcroot)src/ckh.c $(srcroot)src/ctl.c $(srcroot)src/extent.c \ $(srcroot)src/bitmap.c \
$(srcroot)src/hash.c $(srcroot)src/huge.c $(srcroot)src/mb.c \ $(srcroot)src/chunk.c \
$(srcroot)src/mutex.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \ $(srcroot)src/chunk_dss.c \
$(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \ $(srcroot)src/chunk_mmap.c \
$(srcroot)src/util.c $(srcroot)src/tsd.c $(srcroot)src/ckh.c \
ifeq (macho, $(ABI)) $(srcroot)src/ctl.c \
CSRCS += $(srcroot)src/zone.c $(srcroot)src/extent.c \
$(srcroot)src/hash.c \
$(srcroot)src/huge.c \
$(srcroot)src/mb.c \
$(srcroot)src/mutex.c \
$(srcroot)src/nstime.c \
$(srcroot)src/pages.c \
$(srcroot)src/prng.c \
$(srcroot)src/prof.c \
$(srcroot)src/quarantine.c \
$(srcroot)src/rtree.c \
$(srcroot)src/stats.c \
$(srcroot)src/spin.c \
$(srcroot)src/tcache.c \
$(srcroot)src/ticker.c \
$(srcroot)src/tsd.c \
$(srcroot)src/util.c \
$(srcroot)src/witness.c
ifeq ($(enable_valgrind), 1)
C_SRCS += $(srcroot)src/valgrind.c
endif
ifeq ($(enable_zone_allocator), 1)
C_SRCS += $(srcroot)src/zone.c
endif endif
ifeq ($(IMPORTLIB),$(SO)) ifeq ($(IMPORTLIB),$(SO))
STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A) STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A)
...@@ -95,39 +128,115 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) ...@@ -95,39 +128,115 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV)
ifneq ($(SOREV),$(SO)) ifneq ($(SOREV),$(SO))
DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO)
endif endif
ifeq (1, $(link_whole_archive))
LJEMALLOC := -Wl,--whole-archive -L$(objroot)lib -l$(LIBJEMALLOC) -Wl,--no-whole-archive
else
LJEMALLOC := $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
endif
PC := $(objroot)jemalloc.pc
MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 MAN3 := $(objroot)doc/jemalloc$(install_suffix).3
DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml
DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html)
DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3)
DOCS := $(DOCS_HTML) $(DOCS_MAN3) DOCS := $(DOCS_HTML) $(DOCS_MAN3)
CTESTS := $(srcroot)test/aligned_alloc.c $(srcroot)test/allocated.c \ C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \
$(srcroot)test/ALLOCM_ARENA.c $(srcroot)test/bitmap.c \ $(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \
$(srcroot)test/mremap.c $(srcroot)test/posix_memalign.c \ $(srcroot)test/src/mtx.c $(srcroot)test/src/mq.c \
$(srcroot)test/thread_arena.c $(srcroot)test/thread_tcache_enabled.c $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \
ifeq ($(enable_experimental), 1) $(srcroot)test/src/thd.c $(srcroot)test/src/timer.c
CTESTS += $(srcroot)test/allocm.c $(srcroot)test/rallocm.c ifeq (1, $(link_whole_archive))
C_UTIL_INTEGRATION_SRCS :=
else
C_UTIL_INTEGRATION_SRCS := $(srcroot)src/nstime.c $(srcroot)src/util.c
endif endif
TESTS_UNIT := \
COBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.$(O)) $(srcroot)test/unit/a0.c \
CPICOBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.pic.$(O)) $(srcroot)test/unit/arena_reset.c \
CTESTOBJS := $(CTESTS:$(srcroot)%.c=$(objroot)%.$(O)) $(srcroot)test/unit/atomic.c \
$(srcroot)test/unit/bitmap.c \
.PHONY: all dist doc_html doc_man doc $(srcroot)test/unit/ckh.c \
$(srcroot)test/unit/decay.c \
$(srcroot)test/unit/fork.c \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/junk.c \
$(srcroot)test/unit/junk_alloc.c \
$(srcroot)test/unit/junk_free.c \
$(srcroot)test/unit/lg_chunk.c \
$(srcroot)test/unit/mallctl.c \
$(srcroot)test/unit/math.c \
$(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \
$(srcroot)test/unit/ph.c \
$(srcroot)test/unit/prng.c \
$(srcroot)test/unit/prof_accum.c \
$(srcroot)test/unit/prof_active.c \
$(srcroot)test/unit/prof_gdump.c \
$(srcroot)test/unit/prof_idump.c \
$(srcroot)test/unit/prof_reset.c \
$(srcroot)test/unit/prof_thread_name.c \
$(srcroot)test/unit/ql.c \
$(srcroot)test/unit/qr.c \
$(srcroot)test/unit/quarantine.c \
$(srcroot)test/unit/rb.c \
$(srcroot)test/unit/rtree.c \
$(srcroot)test/unit/run_quantize.c \
$(srcroot)test/unit/SFMT.c \
$(srcroot)test/unit/size_classes.c \
$(srcroot)test/unit/smoothstep.c \
$(srcroot)test/unit/stats.c \
$(srcroot)test/unit/ticker.c \
$(srcroot)test/unit/nstime.c \
$(srcroot)test/unit/tsd.c \
$(srcroot)test/unit/util.c \
$(srcroot)test/unit/witness.c \
$(srcroot)test/unit/zero.c
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
$(srcroot)test/integration/allocated.c \
$(srcroot)test/integration/sdallocx.c \
$(srcroot)test/integration/mallocx.c \
$(srcroot)test/integration/MALLOCX_ARENA.c \
$(srcroot)test/integration/overflow.c \
$(srcroot)test/integration/posix_memalign.c \
$(srcroot)test/integration/rallocx.c \
$(srcroot)test/integration/thread_arena.c \
$(srcroot)test/integration/thread_tcache_enabled.c \
$(srcroot)test/integration/xallocx.c \
$(srcroot)test/integration/chunk.c
TESTS_STRESS := $(srcroot)test/stress/microbench.c
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O))
C_PIC_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
C_JET_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.jet.$(O))
C_TESTLIB_UNIT_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.unit.$(O))
C_TESTLIB_INTEGRATION_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
C_UTIL_INTEGRATION_OBJS := $(C_UTIL_INTEGRATION_SRCS:$(srcroot)%.c=$(objroot)%.integration.$(O))
C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O))
C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS)
TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O))
TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS)
.PHONY: all dist build_doc_html build_doc_man build_doc
.PHONY: install_bin install_include install_lib .PHONY: install_bin install_include install_lib
.PHONY: install_html install_man install_doc install .PHONY: install_doc_html install_doc_man install_doc install
.PHONY: tests check clean distclean relclean .PHONY: tests check clean distclean relclean
.SECONDARY : $(CTESTOBJS) .SECONDARY : $(TESTS_OBJS)
# Default target. # Default target.
all: build all: build_lib
dist: build_doc dist: build_doc
$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl $(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl
$(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $<
$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl $(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl
$(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $<
build_doc_html: $(DOCS_HTML) build_doc_html: $(DOCS_HTML)
...@@ -138,30 +247,45 @@ build_doc: $(DOCS) ...@@ -138,30 +247,45 @@ build_doc: $(DOCS)
# Include generated dependency files. # Include generated dependency files.
# #
ifdef CC_MM ifdef CC_MM
-include $(COBJS:%.$(O)=%.d) -include $(C_OBJS:%.$(O)=%.d)
-include $(CPICOBJS:%.$(O)=%.d) -include $(C_PIC_OBJS:%.$(O)=%.d)
-include $(CTESTOBJS:%.$(O)=%.d) -include $(C_JET_OBJS:%.$(O)=%.d)
-include $(C_TESTLIB_OBJS:%.$(O)=%.d)
-include $(TESTS_OBJS:%.$(O)=%.d)
endif endif
$(COBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c $(C_OBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c
$(CPICOBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c $(C_PIC_OBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c
$(CPICOBJS): CFLAGS += $(PIC_CFLAGS) $(C_PIC_OBJS): CFLAGS += $(PIC_CFLAGS)
$(CTESTOBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c $(C_JET_OBJS): $(objroot)src/%.jet.$(O): $(srcroot)src/%.c
$(CTESTOBJS): CPPFLAGS += -I$(objroot)test $(C_JET_OBJS): CFLAGS += -DJEMALLOC_JET
$(C_TESTLIB_UNIT_OBJS): $(objroot)test/src/%.unit.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(C_TESTLIB_INTEGRATION_OBJS): $(objroot)test/src/%.integration.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
$(C_UTIL_INTEGRATION_OBJS): $(objroot)src/%.integration.$(O): $(srcroot)src/%.c
$(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/%.c
$(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB
$(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
$(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST
$(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST
$(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST
$(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c
$(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include
ifneq ($(IMPORTLIB),$(SO)) ifneq ($(IMPORTLIB),$(SO))
$(COBJS): CPPFLAGS += -DDLLEXPORT $(C_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT
endif endif
ifndef CC_MM ifndef CC_MM
# Dependencies # Dependencies.
HEADER_DIRS = $(srcroot)include/jemalloc/internal \ HEADER_DIRS = $(srcroot)include/jemalloc/internal \
$(objroot)include/jemalloc $(objroot)include/jemalloc/internal $(objroot)include/jemalloc $(objroot)include/jemalloc/internal
HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)) HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h))
$(COBJS) $(CPICOBJS) $(CTESTOBJS): $(HEADERS) $(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS)
$(CTESTOBJS): $(objroot)test/jemalloc_test.h $(TESTS_OBJS): $(objroot)test/include/test/jemalloc_test.h
endif endif
$(COBJS) $(CPICOBJS) $(CTESTOBJS): %.$(O): $(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O):
@mkdir -p $(@D) @mkdir -p $(@D)
$(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $< $(CC) $(CFLAGS) -c $(CPPFLAGS) $(CTARGET) $<
ifdef CC_MM ifdef CC_MM
...@@ -174,119 +298,180 @@ ifneq ($(SOREV),$(SO)) ...@@ -174,119 +298,180 @@ ifneq ($(SOREV),$(SO))
ln -sf $(<F) $@ ln -sf $(<F) $@
endif endif
$(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(CPICOBJS),$(COBJS)) $(objroot)lib/$(LIBJEMALLOC).$(SOREV) : $(if $(PIC_CFLAGS),$(C_PIC_OBJS),$(C_OBJS))
@mkdir -p $(@D) @mkdir -p $(@D)
$(CC) $(DSO_LDFLAGS) $(call RPATH,$(RPATH_EXTRA)) $(LDTARGET) $+ $(LDFLAGS) $(LIBS) $(EXTRA_LDFLAGS) $(CC) $(DSO_LDFLAGS) $(call RPATH,$(RPATH_EXTRA)) $(LDTARGET) $+ $(LDFLAGS) $(LIBS) $(EXTRA_LDFLAGS)
$(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(CPICOBJS) $(objroot)lib/$(LIBJEMALLOC)_pic.$(A) : $(C_PIC_OBJS)
$(objroot)lib/$(LIBJEMALLOC).$(A) : $(COBJS) $(objroot)lib/$(LIBJEMALLOC).$(A) : $(C_OBJS)
$(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(COBJS) $(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(C_OBJS)
$(STATIC_LIBS): $(STATIC_LIBS):
@mkdir -p $(@D) @mkdir -p $(@D)
$(MKLIB) $+ $(AR) $(ARFLAGS)@AROUT@ $+
$(objroot)test/bitmap$(EXE): $(objroot)src/bitmap.$(O) $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(TESTS_UNIT_LINK_OBJS) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS)
@mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D)
$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
$(objroot)test/%$(EXE): $(objroot)test/%.$(O) $(objroot)src/util.$(O) $(DSOS) $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
@mkdir -p $(@D) @mkdir -p $(@D)
$(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(filter -lpthread,$(LIBS)) $(EXTRA_LDFLAGS) $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
build_lib_shared: $(DSOS) build_lib_shared: $(DSOS)
build_lib_static: $(STATIC_LIBS) build_lib_static: $(STATIC_LIBS)
build: build_lib_shared build_lib_static build_lib: build_lib_shared build_lib_static
install_bin: install_bin:
install -d $(BINDIR) $(INSTALL) -d $(BINDIR)
@for b in $(BINS); do \ @for b in $(BINS); do \
echo "install -m 755 $$b $(BINDIR)"; \ echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
install -m 755 $$b $(BINDIR); \ $(INSTALL) -m 755 $$b $(BINDIR); \
done done
install_include: install_include:
install -d $(INCLUDEDIR)/jemalloc $(INSTALL) -d $(INCLUDEDIR)/jemalloc
@for h in $(CHDRS); do \ @for h in $(C_HDRS); do \
echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \ echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
install -m 644 $$h $(INCLUDEDIR)/jemalloc; \ $(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
done done
install_lib_shared: $(DSOS) install_lib_shared: $(DSOS)
install -d $(LIBDIR) $(INSTALL) -d $(LIBDIR)
install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR) $(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
ifneq ($(SOREV),$(SO)) ifneq ($(SOREV),$(SO))
ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO) ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
endif endif
install_lib_static: $(STATIC_LIBS) install_lib_static: $(STATIC_LIBS)
install -d $(LIBDIR) $(INSTALL) -d $(LIBDIR)
@for l in $(STATIC_LIBS); do \ @for l in $(STATIC_LIBS); do \
echo "install -m 755 $$l $(LIBDIR)"; \ echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
install -m 755 $$l $(LIBDIR); \ $(INSTALL) -m 755 $$l $(LIBDIR); \
done done
install_lib: install_lib_shared install_lib_static install_lib_pc: $(PC)
$(INSTALL) -d $(LIBDIR)/pkgconfig
@for l in $(PC); do \
echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
done
install_lib: install_lib_shared install_lib_static install_lib_pc
install_doc_html: install_doc_html:
install -d $(DATADIR)/doc/jemalloc$(install_suffix) $(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
@for d in $(DOCS_HTML); do \ @for d in $(DOCS_HTML); do \
echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \ echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \ $(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
done done
install_doc_man: install_doc_man:
install -d $(MANDIR)/man3 $(INSTALL) -d $(MANDIR)/man3
@for d in $(DOCS_MAN3); do \ @for d in $(DOCS_MAN3); do \
echo "install -m 644 $$d $(MANDIR)/man3"; \ echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
install -m 644 $$d $(MANDIR)/man3; \ $(INSTALL) -m 644 $$d $(MANDIR)/man3; \
done done
install_doc: install_doc_html install_doc_man install_doc: install_doc_html install_doc_man
install: install_bin install_include install_lib install_doc install: install_bin install_include install_lib install_doc
tests: $(CTESTS:$(srcroot)%.c=$(objroot)%$(EXE)) tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE))
tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE))
check: tests tests_stress: $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%$(EXE))
@mkdir -p $(objroot)test tests: tests_unit tests_integration tests_stress
@$(SHELL) -c 'total=0; \
failures=0; \ check_unit_dir:
echo "========================================="; \ @mkdir -p $(objroot)test/unit
for t in $(CTESTS:$(srcroot)%.c=$(objroot)%); do \ check_integration_dir:
total=`expr $$total + 1`; \ @mkdir -p $(objroot)test/integration
/bin/echo -n "$${t} ... "; \ stress_dir:
$(TEST_LIBRARY_PATH) $${t}$(EXE) $(abs_srcroot) \ @mkdir -p $(objroot)test/stress
$(abs_objroot) > $(objroot)$${t}.out 2>&1; \ check_dir: check_unit_dir check_integration_dir
if test -e "$(srcroot)$${t}.exp"; then \
diff -w -u $(srcroot)$${t}.exp \ check_unit: tests_unit check_unit_dir
$(objroot)$${t}.out >/dev/null 2>&1; \ $(MALLOC_CONF)="purge:ratio" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
fail=$$?; \ $(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
if test "$${fail}" -eq "1" ; then \ check_integration_prof: tests_integration check_integration_dir
failures=`expr $${failures} + 1`; \ ifeq ($(enable_prof), 1)
echo "*** FAIL ***"; \ $(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
else \ $(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
echo "pass"; \ endif
fi; \ check_integration_decay: tests_integration check_integration_dir
else \ $(MALLOC_CONF)="purge:decay,decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
echo "*** FAIL *** (.exp file is missing)"; \ $(MALLOC_CONF)="purge:decay,decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
failures=`expr $${failures} + 1`; \ $(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
fi; \ check_integration: tests_integration check_integration_dir
done; \ $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
echo "========================================="; \ stress: tests_stress stress_dir
echo "Failures: $${failures}/$${total}"' $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
check: check_unit check_integration check_integration_decay check_integration_prof
ifeq ($(enable_code_coverage), 1)
coverage_unit: check_unit
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS)
coverage_integration: check_integration
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
coverage_stress: stress
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress stress $(TESTS_STRESS_OBJS)
coverage: check
$(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)src integration $(C_UTIL_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src unit $(C_TESTLIB_UNIT_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/unit unit $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS)
$(SHELL) $(srcroot)coverage.sh $(srcroot)test/stress integration $(TESTS_STRESS_OBJS)
endif
clean: clean:
rm -f $(COBJS) rm -f $(C_OBJS)
rm -f $(CPICOBJS) rm -f $(C_PIC_OBJS)
rm -f $(COBJS:%.$(O)=%.d) rm -f $(C_JET_OBJS)
rm -f $(CPICOBJS:%.$(O)=%.d) rm -f $(C_TESTLIB_OBJS)
rm -f $(CTESTOBJS:%.$(O)=%$(EXE)) rm -f $(C_OBJS:%.$(O)=%.d)
rm -f $(CTESTOBJS) rm -f $(C_OBJS:%.$(O)=%.gcda)
rm -f $(CTESTOBJS:%.$(O)=%.d) rm -f $(C_OBJS:%.$(O)=%.gcno)
rm -f $(CTESTOBJS:%.$(O)=%.out) rm -f $(C_PIC_OBJS:%.$(O)=%.d)
rm -f $(C_PIC_OBJS:%.$(O)=%.gcda)
rm -f $(C_PIC_OBJS:%.$(O)=%.gcno)
rm -f $(C_JET_OBJS:%.$(O)=%.d)
rm -f $(C_JET_OBJS:%.$(O)=%.gcda)
rm -f $(C_JET_OBJS:%.$(O)=%.gcno)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.d)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcda)
rm -f $(C_TESTLIB_OBJS:%.$(O)=%.gcno)
rm -f $(TESTS_OBJS:%.$(O)=%$(EXE))
rm -f $(TESTS_OBJS)
rm -f $(TESTS_OBJS:%.$(O)=%.d)
rm -f $(TESTS_OBJS:%.$(O)=%.gcda)
rm -f $(TESTS_OBJS:%.$(O)=%.gcno)
rm -f $(TESTS_OBJS:%.$(O)=%.out)
rm -f $(DSOS) $(STATIC_LIBS) rm -f $(DSOS) $(STATIC_LIBS)
rm -f $(objroot)*.gcov.*
distclean: clean distclean: clean
rm -rf $(objroot)autom4te.cache rm -f $(objroot)bin/jemalloc-config
rm -f $(objroot)bin/jemalloc.sh
rm -f $(objroot)bin/jeprof
rm -f $(objroot)config.log rm -f $(objroot)config.log
rm -f $(objroot)config.status rm -f $(objroot)config.status
rm -f $(objroot)config.stamp rm -f $(objroot)config.stamp
...@@ -295,7 +480,7 @@ distclean: clean ...@@ -295,7 +480,7 @@ distclean: clean
relclean: distclean relclean: distclean
rm -f $(objroot)configure rm -f $(objroot)configure
rm -f $(srcroot)VERSION rm -f $(objroot)VERSION
rm -f $(DOCS_HTML) rm -f $(DOCS_HTML)
rm -f $(DOCS_MAN3) rm -f $(DOCS_MAN3)
......
jemalloc is a general-purpose scalable concurrent malloc(3) implementation. jemalloc is a general purpose malloc(3) implementation that emphasizes
This distribution is a "portable" implementation that currently targets fragmentation avoidance and scalable concurrency support. jemalloc first came
FreeBSD, Linux, Apple OS X, and MinGW. jemalloc is included as the default into use as the FreeBSD libc allocator in 2005, and since then it has found its
allocator in the FreeBSD and NetBSD operating systems, and it is used by the way into numerous applications that rely on its predictable behavior. In 2010
Mozilla Firefox web browser on Microsoft Windows-related platforms. Depending jemalloc development efforts broadened to include developer support features
on your needs, one of the other divergent versions may suit your needs better such as heap profiling, Valgrind integration, and extensive monitoring/tuning
than this distribution. hooks. Modern jemalloc releases continue to be integrated back into FreeBSD,
and therefore versatility remains critical. Ongoing development efforts trend
toward making jemalloc among the best allocators for a broad range of demanding
applications, and eliminating/mitigating weaknesses that have practical
repercussions for real world applications.
The COPYING file contains copyright and licensing information. The COPYING file contains copyright and licensing information.
...@@ -13,4 +17,4 @@ jemalloc. ...@@ -13,4 +17,4 @@ jemalloc.
The ChangeLog file contains a brief summary of changes for each release. The ChangeLog file contains a brief summary of changes for each release.
URL: http://www.canonware.com/jemalloc/ URL: http://jemalloc.net/
3.2.0-0-g87499f6748ebe4817571e817e9f680ccb5bf54a9 4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc
#!/bin/sh
usage() {
cat <<EOF
Usage:
@BINDIR@/jemalloc-config <option>
Options:
--help | -h : Print usage.
--version : Print jemalloc version.
--revision : Print shared library revision number.
--config : Print configure options used to build jemalloc.
--prefix : Print installation directory prefix.
--bindir : Print binary installation directory.
--datadir : Print data installation directory.
--includedir : Print include installation directory.
--libdir : Print library installation directory.
--mandir : Print manual page installation directory.
--cc : Print compiler used to build jemalloc.
--cflags : Print compiler flags used to build jemalloc.
--cppflags : Print preprocessor flags used to build jemalloc.
--ldflags : Print library flags used to build jemalloc.
--libs : Print libraries jemalloc was linked against.
EOF
}
prefix="@prefix@"
exec_prefix="@exec_prefix@"
case "$1" in
--help | -h)
usage
exit 0
;;
--version)
echo "@jemalloc_version@"
;;
--revision)
echo "@rev@"
;;
--config)
echo "@CONFIG@"
;;
--prefix)
echo "@PREFIX@"
;;
--bindir)
echo "@BINDIR@"
;;
--datadir)
echo "@DATADIR@"
;;
--includedir)
echo "@INCLUDEDIR@"
;;
--libdir)
echo "@LIBDIR@"
;;
--mandir)
echo "@MANDIR@"
;;
--cc)
echo "@CC@"
;;
--cflags)
echo "@CFLAGS@"
;;
--cppflags)
echo "@CPPFLAGS@"
;;
--ldflags)
echo "@LDFLAGS@ @EXTRA_LDFLAGS@"
;;
--libs)
echo "@LIBS@"
;;
*)
usage
exit 1
esac
...@@ -2,11 +2,11 @@ ...@@ -2,11 +2,11 @@
# Copyright (c) 1998-2007, Google Inc. # Copyright (c) 1998-2007, Google Inc.
# All rights reserved. # All rights reserved.
# #
# Redistribution and use in source and binary forms, with or without # Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are # modification, are permitted provided that the following conditions are
# met: # met:
# #
# * Redistributions of source code must retain the above copyright # * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer. # notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above # * Redistributions in binary form must reproduce the above
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
# * Neither the name of Google Inc. nor the names of its # * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from # contributors may be used to endorse or promote products derived from
# this software without specific prior written permission. # this software without specific prior written permission.
# #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
...@@ -40,28 +40,28 @@ ...@@ -40,28 +40,28 @@
# #
# Examples: # Examples:
# #
# % tools/pprof "program" "profile" # % tools/jeprof "program" "profile"
# Enters "interactive" mode # Enters "interactive" mode
# #
# % tools/pprof --text "program" "profile" # % tools/jeprof --text "program" "profile"
# Generates one line per procedure # Generates one line per procedure
# #
# % tools/pprof --gv "program" "profile" # % tools/jeprof --gv "program" "profile"
# Generates annotated call-graph and displays via "gv" # Generates annotated call-graph and displays via "gv"
# #
# % tools/pprof --gv --focus=Mutex "program" "profile" # % tools/jeprof --gv --focus=Mutex "program" "profile"
# Restrict to code paths that involve an entry that matches "Mutex" # Restrict to code paths that involve an entry that matches "Mutex"
# #
# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" # % tools/jeprof --gv --focus=Mutex --ignore=string "program" "profile"
# Restrict to code paths that involve an entry that matches "Mutex" # Restrict to code paths that involve an entry that matches "Mutex"
# and does not match "string" # and does not match "string"
# #
# % tools/pprof --list=IBF_CheckDocid "program" "profile" # % tools/jeprof --list=IBF_CheckDocid "program" "profile"
# Generates disassembly listing of all routines with at least one # Generates disassembly listing of all routines with at least one
# sample that match the --list=<regexp> pattern. The listing is # sample that match the --list=<regexp> pattern. The listing is
# annotated with the flat and cumulative sample counts at each line. # annotated with the flat and cumulative sample counts at each line.
# #
# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" # % tools/jeprof --disasm=IBF_CheckDocid "program" "profile"
# Generates disassembly listing of all routines with at least one # Generates disassembly listing of all routines with at least one
# sample that match the --disasm=<regexp> pattern. The listing is # sample that match the --disasm=<regexp> pattern. The listing is
# annotated with the flat and cumulative sample counts at each PC value. # annotated with the flat and cumulative sample counts at each PC value.
...@@ -72,10 +72,11 @@ use strict; ...@@ -72,10 +72,11 @@ use strict;
use warnings; use warnings;
use Getopt::Long; use Getopt::Long;
my $JEPROF_VERSION = "@jemalloc_version@";
my $PPROF_VERSION = "2.0"; my $PPROF_VERSION = "2.0";
# These are the object tools we use which can come from a # These are the object tools we use which can come from a
# user-specified location using --tools, from the PPROF_TOOLS # user-specified location using --tools, from the JEPROF_TOOLS
# environment variable, or from the environment. # environment variable, or from the environment.
my %obj_tool_map = ( my %obj_tool_map = (
"objdump" => "objdump", "objdump" => "objdump",
...@@ -94,7 +95,7 @@ my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread ...@@ -94,7 +95,7 @@ my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread
my @KCACHEGRIND = ("kcachegrind"); my @KCACHEGRIND = ("kcachegrind");
my @PS2PDF = ("ps2pdf"); my @PS2PDF = ("ps2pdf");
# These are used for dynamic profiles # These are used for dynamic profiles
my @URL_FETCHER = ("curl", "-s"); my @URL_FETCHER = ("curl", "-s", "--fail");
# These are the web pages that servers need to support for dynamic profiles # These are the web pages that servers need to support for dynamic profiles
my $HEAP_PAGE = "/pprof/heap"; my $HEAP_PAGE = "/pprof/heap";
...@@ -144,13 +145,13 @@ my $sep_address = undef; ...@@ -144,13 +145,13 @@ my $sep_address = undef;
sub usage_string { sub usage_string {
return <<EOF; return <<EOF;
Usage: Usage:
pprof [options] <program> <profiles> jeprof [options] <program> <profiles>
<profiles> is a space separated list of profile names. <profiles> is a space separated list of profile names.
pprof [options] <symbolized-profiles> jeprof [options] <symbolized-profiles>
<symbolized-profiles> is a list of profile files where each file contains <symbolized-profiles> is a list of profile files where each file contains
the necessary symbol mappings as well as profile data (likely generated the necessary symbol mappings as well as profile data (likely generated
with --raw). with --raw).
pprof [options] <profile> jeprof [options] <profile>
<profile> is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE <profile> is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE
Each name can be: Each name can be:
...@@ -161,9 +162,9 @@ pprof [options] <profile> ...@@ -161,9 +162,9 @@ pprof [options] <profile>
$GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall,
$CENSUSPROFILE_PAGE, or /pprof/filteredprofile. $CENSUSPROFILE_PAGE, or /pprof/filteredprofile.
For instance: For instance:
pprof http://myserver.com:80$HEAP_PAGE jeprof http://myserver.com:80$HEAP_PAGE
If /<service> is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). If /<service> is omitted, the service defaults to $PROFILE_PAGE (cpu profiling).
pprof --symbols <program> jeprof --symbols <program>
Maps addresses to symbol names. In this mode, stdin should be a Maps addresses to symbol names. In this mode, stdin should be a
list of library mappings, in the same format as is found in the heap- list of library mappings, in the same format as is found in the heap-
and cpu-profile files (this loosely matches that of /proc/self/maps and cpu-profile files (this loosely matches that of /proc/self/maps
...@@ -202,7 +203,7 @@ Output type: ...@@ -202,7 +203,7 @@ Output type:
--pdf Generate PDF to stdout --pdf Generate PDF to stdout
--svg Generate SVG to stdout --svg Generate SVG to stdout
--gif Generate GIF to stdout --gif Generate GIF to stdout
--raw Generate symbolized pprof data (useful with remote fetch) --raw Generate symbolized jeprof data (useful with remote fetch)
Heap-Profile Options: Heap-Profile Options:
--inuse_space Display in-use (mega)bytes [default] --inuse_space Display in-use (mega)bytes [default]
...@@ -222,11 +223,14 @@ Call-graph Options: ...@@ -222,11 +223,14 @@ Call-graph Options:
--nodefraction=<f> Hide nodes below <f>*total [default=.005] --nodefraction=<f> Hide nodes below <f>*total [default=.005]
--edgefraction=<f> Hide edges below <f>*total [default=.001] --edgefraction=<f> Hide edges below <f>*total [default=.001]
--maxdegree=<n> Max incoming/outgoing edges per node [default=8] --maxdegree=<n> Max incoming/outgoing edges per node [default=8]
--focus=<regexp> Focus on nodes matching <regexp> --focus=<regexp> Focus on backtraces with nodes matching <regexp>
--ignore=<regexp> Ignore nodes matching <regexp> --thread=<n> Show profile for thread <n>
--ignore=<regexp> Ignore backtraces with nodes matching <regexp>
--scale=<n> Set GV scaling [default=0] --scale=<n> Set GV scaling [default=0]
--heapcheck Make nodes with non-0 object counts --heapcheck Make nodes with non-0 object counts
(i.e. direct leak generators) more visible (i.e. direct leak generators) more visible
--retain=<regexp> Retain only nodes that match <regexp>
--exclude=<regexp> Exclude all nodes that match <regexp>
Miscellaneous: Miscellaneous:
--tools=<prefix or binary:fullpath>[,...] \$PATH for object tool pathnames --tools=<prefix or binary:fullpath>[,...] \$PATH for object tool pathnames
...@@ -235,34 +239,34 @@ Miscellaneous: ...@@ -235,34 +239,34 @@ Miscellaneous:
--version Version information --version Version information
Environment Variables: Environment Variables:
PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof JEPROF_TMPDIR Profiles directory. Defaults to \$HOME/jeprof
PPROF_TOOLS Prefix for object tools pathnames JEPROF_TOOLS Prefix for object tools pathnames
Examples: Examples:
pprof /bin/ls ls.prof jeprof /bin/ls ls.prof
Enters "interactive" mode Enters "interactive" mode
pprof --text /bin/ls ls.prof jeprof --text /bin/ls ls.prof
Outputs one line per procedure Outputs one line per procedure
pprof --web /bin/ls ls.prof jeprof --web /bin/ls ls.prof
Displays annotated call-graph in web browser Displays annotated call-graph in web browser
pprof --gv /bin/ls ls.prof jeprof --gv /bin/ls ls.prof
Displays annotated call-graph via 'gv' Displays annotated call-graph via 'gv'
pprof --gv --focus=Mutex /bin/ls ls.prof jeprof --gv --focus=Mutex /bin/ls ls.prof
Restricts to code paths including a .*Mutex.* entry Restricts to code paths including a .*Mutex.* entry
pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof jeprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof
Code paths including Mutex but not string Code paths including Mutex but not string
pprof --list=getdir /bin/ls ls.prof jeprof --list=getdir /bin/ls ls.prof
(Per-line) annotated source listing for getdir() (Per-line) annotated source listing for getdir()
pprof --disasm=getdir /bin/ls ls.prof jeprof --disasm=getdir /bin/ls ls.prof
(Per-PC) annotated disassembly for getdir() (Per-PC) annotated disassembly for getdir()
pprof http://localhost:1234/ jeprof http://localhost:1234/
Enters "interactive" mode Enters "interactive" mode
pprof --text localhost:1234 jeprof --text localhost:1234
Outputs one line per procedure for localhost:1234 Outputs one line per procedure for localhost:1234
pprof --raw localhost:1234 > ./local.raw jeprof --raw localhost:1234 > ./local.raw
pprof --text ./local.raw jeprof --text ./local.raw
Fetches a remote profile for later analysis and then Fetches a remote profile for later analysis and then
analyzes it in text mode. analyzes it in text mode.
EOF EOF
...@@ -270,7 +274,8 @@ EOF ...@@ -270,7 +274,8 @@ EOF
sub version_string { sub version_string {
return <<EOF return <<EOF
pprof (part of gperftools $PPROF_VERSION) jeprof (part of jemalloc $JEPROF_VERSION)
based on pprof (part of gperftools $PPROF_VERSION)
Copyright 1998-2007 Google Inc. Copyright 1998-2007 Google Inc.
...@@ -293,8 +298,8 @@ sub Init() { ...@@ -293,8 +298,8 @@ sub Init() {
# Setup tmp-file name and handler to clean it up. # Setup tmp-file name and handler to clean it up.
# We do this in the very beginning so that we can use # We do this in the very beginning so that we can use
# error() and cleanup() function anytime here after. # error() and cleanup() function anytime here after.
$main::tmpfile_sym = "/tmp/pprof$$.sym"; $main::tmpfile_sym = "/tmp/jeprof$$.sym";
$main::tmpfile_ps = "/tmp/pprof$$"; $main::tmpfile_ps = "/tmp/jeprof$$";
$main::next_tmpfile = 0; $main::next_tmpfile = 0;
$SIG{'INT'} = \&sighandler; $SIG{'INT'} = \&sighandler;
...@@ -332,9 +337,12 @@ sub Init() { ...@@ -332,9 +337,12 @@ sub Init() {
$main::opt_edgefraction = 0.001; $main::opt_edgefraction = 0.001;
$main::opt_maxdegree = 8; $main::opt_maxdegree = 8;
$main::opt_focus = ''; $main::opt_focus = '';
$main::opt_thread = undef;
$main::opt_ignore = ''; $main::opt_ignore = '';
$main::opt_scale = 0; $main::opt_scale = 0;
$main::opt_heapcheck = 0; $main::opt_heapcheck = 0;
$main::opt_retain = '';
$main::opt_exclude = '';
$main::opt_seconds = 30; $main::opt_seconds = 30;
$main::opt_lib = ""; $main::opt_lib = "";
...@@ -402,9 +410,12 @@ sub Init() { ...@@ -402,9 +410,12 @@ sub Init() {
"edgefraction=f" => \$main::opt_edgefraction, "edgefraction=f" => \$main::opt_edgefraction,
"maxdegree=i" => \$main::opt_maxdegree, "maxdegree=i" => \$main::opt_maxdegree,
"focus=s" => \$main::opt_focus, "focus=s" => \$main::opt_focus,
"thread=s" => \$main::opt_thread,
"ignore=s" => \$main::opt_ignore, "ignore=s" => \$main::opt_ignore,
"scale=i" => \$main::opt_scale, "scale=i" => \$main::opt_scale,
"heapcheck" => \$main::opt_heapcheck, "heapcheck" => \$main::opt_heapcheck,
"retain=s" => \$main::opt_retain,
"exclude=s" => \$main::opt_exclude,
"inuse_space!" => \$main::opt_inuse_space, "inuse_space!" => \$main::opt_inuse_space,
"inuse_objects!" => \$main::opt_inuse_objects, "inuse_objects!" => \$main::opt_inuse_objects,
"alloc_space!" => \$main::opt_alloc_space, "alloc_space!" => \$main::opt_alloc_space,
...@@ -562,66 +573,12 @@ sub Init() { ...@@ -562,66 +573,12 @@ sub Init() {
} }
} }
sub Main() { sub FilterAndPrint {
Init(); my ($profile, $symbols, $libs, $thread) = @_;
$main::collected_profile = undef;
@main::profile_files = ();
$main::op_time = time();
# Printing symbols is special and requires a lot less info that most.
if ($main::opt_symbols) {
PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin
return;
}
# Fetch all profile data
FetchDynamicProfiles();
# this will hold symbols that we read from the profile files
my $symbol_map = {};
# Read one profile, pick the last item on the list
my $data = ReadProfile($main::prog, pop(@main::profile_files));
my $profile = $data->{profile};
my $pcs = $data->{pcs};
my $libs = $data->{libs}; # Info about main program and shared libraries
$symbol_map = MergeSymbols($symbol_map, $data->{symbols});
# Add additional profiles, if available.
if (scalar(@main::profile_files) > 0) {
foreach my $pname (@main::profile_files) {
my $data2 = ReadProfile($main::prog, $pname);
$profile = AddProfile($profile, $data2->{profile});
$pcs = AddPcs($pcs, $data2->{pcs});
$symbol_map = MergeSymbols($symbol_map, $data2->{symbols});
}
}
# Subtract base from profile, if specified
if ($main::opt_base ne '') {
my $base = ReadProfile($main::prog, $main::opt_base);
$profile = SubtractProfile($profile, $base->{profile});
$pcs = AddPcs($pcs, $base->{pcs});
$symbol_map = MergeSymbols($symbol_map, $base->{symbols});
}
# Get total data in profile # Get total data in profile
my $total = TotalProfile($profile); my $total = TotalProfile($profile);
# Collect symbols
my $symbols;
if ($main::use_symbolized_profile) {
$symbols = FetchSymbols($pcs, $symbol_map);
} elsif ($main::use_symbol_page) {
$symbols = FetchSymbols($pcs);
} else {
# TODO(csilvers): $libs uses the /proc/self/maps data from profile1,
# which may differ from the data from subsequent profiles, especially
# if they were run on different machines. Use appropriate libs for
# each pc somehow.
$symbols = ExtractSymbols($libs, $pcs);
}
# Remove uniniteresting stack items # Remove uniniteresting stack items
$profile = RemoveUninterestingFrames($symbols, $profile); $profile = RemoveUninterestingFrames($symbols, $profile);
...@@ -656,7 +613,9 @@ sub Main() { ...@@ -656,7 +613,9 @@ sub Main() {
# (only matters when --heapcheck is given but we must be # (only matters when --heapcheck is given but we must be
# compatible with old branches that did not pass --heapcheck always): # compatible with old branches that did not pass --heapcheck always):
if ($total != 0) { if ($total != 0) {
printf("Total: %s %s\n", Unparse($total), Units()); printf("Total%s: %s %s\n",
(defined($thread) ? " (t$thread)" : ""),
Unparse($total), Units());
} }
PrintText($symbols, $flat, $cumulative, -1); PrintText($symbols, $flat, $cumulative, -1);
} elsif ($main::opt_raw) { } elsif ($main::opt_raw) {
...@@ -692,6 +651,77 @@ sub Main() { ...@@ -692,6 +651,77 @@ sub Main() {
} else { } else {
InteractiveMode($profile, $symbols, $libs, $total); InteractiveMode($profile, $symbols, $libs, $total);
} }
}
sub Main() {
Init();
$main::collected_profile = undef;
@main::profile_files = ();
$main::op_time = time();
# Printing symbols is special and requires a lot less info that most.
if ($main::opt_symbols) {
PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin
return;
}
# Fetch all profile data
FetchDynamicProfiles();
# this will hold symbols that we read from the profile files
my $symbol_map = {};
# Read one profile, pick the last item on the list
my $data = ReadProfile($main::prog, pop(@main::profile_files));
my $profile = $data->{profile};
my $pcs = $data->{pcs};
my $libs = $data->{libs}; # Info about main program and shared libraries
$symbol_map = MergeSymbols($symbol_map, $data->{symbols});
# Add additional profiles, if available.
if (scalar(@main::profile_files) > 0) {
foreach my $pname (@main::profile_files) {
my $data2 = ReadProfile($main::prog, $pname);
$profile = AddProfile($profile, $data2->{profile});
$pcs = AddPcs($pcs, $data2->{pcs});
$symbol_map = MergeSymbols($symbol_map, $data2->{symbols});
}
}
# Subtract base from profile, if specified
if ($main::opt_base ne '') {
my $base = ReadProfile($main::prog, $main::opt_base);
$profile = SubtractProfile($profile, $base->{profile});
$pcs = AddPcs($pcs, $base->{pcs});
$symbol_map = MergeSymbols($symbol_map, $base->{symbols});
}
# Collect symbols
my $symbols;
if ($main::use_symbolized_profile) {
$symbols = FetchSymbols($pcs, $symbol_map);
} elsif ($main::use_symbol_page) {
$symbols = FetchSymbols($pcs);
} else {
# TODO(csilvers): $libs uses the /proc/self/maps data from profile1,
# which may differ from the data from subsequent profiles, especially
# if they were run on different machines. Use appropriate libs for
# each pc somehow.
$symbols = ExtractSymbols($libs, $pcs);
}
if (!defined($main::opt_thread)) {
FilterAndPrint($profile, $symbols, $libs);
}
if (defined($data->{threads})) {
foreach my $thread (sort { $a <=> $b } keys(%{$data->{threads}})) {
if (defined($main::opt_thread) &&
($main::opt_thread eq '*' || $main::opt_thread == $thread)) {
my $thread_profile = $data->{threads}{$thread};
FilterAndPrint($thread_profile, $symbols, $libs, $thread);
}
}
}
cleanup(); cleanup();
exit(0); exit(0);
...@@ -780,14 +810,14 @@ sub InteractiveMode { ...@@ -780,14 +810,14 @@ sub InteractiveMode {
$| = 1; # Make output unbuffered for interactive mode $| = 1; # Make output unbuffered for interactive mode
my ($orig_profile, $symbols, $libs, $total) = @_; my ($orig_profile, $symbols, $libs, $total) = @_;
print STDERR "Welcome to pprof! For help, type 'help'.\n"; print STDERR "Welcome to jeprof! For help, type 'help'.\n";
# Use ReadLine if it's installed and input comes from a console. # Use ReadLine if it's installed and input comes from a console.
if ( -t STDIN && if ( -t STDIN &&
!ReadlineMightFail() && !ReadlineMightFail() &&
defined(eval {require Term::ReadLine}) ) { defined(eval {require Term::ReadLine}) ) {
my $term = new Term::ReadLine 'pprof'; my $term = new Term::ReadLine 'jeprof';
while ( defined ($_ = $term->readline('(pprof) '))) { while ( defined ($_ = $term->readline('(jeprof) '))) {
$term->addhistory($_) if /\S/; $term->addhistory($_) if /\S/;
if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) {
last; # exit when we get an interactive command to quit last; # exit when we get an interactive command to quit
...@@ -795,7 +825,7 @@ sub InteractiveMode { ...@@ -795,7 +825,7 @@ sub InteractiveMode {
} }
} else { # don't have readline } else { # don't have readline
while (1) { while (1) {
print STDERR "(pprof) "; print STDERR "(jeprof) ";
$_ = <STDIN>; $_ = <STDIN>;
last if ! defined $_ ; last if ! defined $_ ;
s/\r//g; # turn windows-looking lines into unix-looking lines s/\r//g; # turn windows-looking lines into unix-looking lines
...@@ -988,7 +1018,7 @@ sub ProcessProfile { ...@@ -988,7 +1018,7 @@ sub ProcessProfile {
sub InteractiveHelpMessage { sub InteractiveHelpMessage {
print STDERR <<ENDOFHELP; print STDERR <<ENDOFHELP;
Interactive pprof mode Interactive jeprof mode
Commands: Commands:
gv gv
...@@ -1031,7 +1061,7 @@ Commands: ...@@ -1031,7 +1061,7 @@ Commands:
Generates callgrind file. If no filename is given, kcachegrind is called. Generates callgrind file. If no filename is given, kcachegrind is called.
help - This listing help - This listing
quit or ^D - End pprof quit or ^D - End jeprof
For commands that accept optional -ignore tags, samples where any routine in For commands that accept optional -ignore tags, samples where any routine in
the stack trace matches the regular expression in any of the -ignore the stack trace matches the regular expression in any of the -ignore
...@@ -1136,8 +1166,21 @@ sub PrintSymbolizedProfile { ...@@ -1136,8 +1166,21 @@ sub PrintSymbolizedProfile {
} }
print '---', "\n"; print '---', "\n";
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash my $profile_marker;
my $profile_marker = $&; if ($main::profile_type eq 'heap') {
$HEAP_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} elsif ($main::profile_type eq 'growth') {
$GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} elsif ($main::profile_type eq 'contention') {
$CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
} else { # elsif ($main::profile_type eq 'cpu')
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash
$profile_marker = $&;
}
print '--- ', $profile_marker, "\n"; print '--- ', $profile_marker, "\n";
if (defined($main::collected_profile)) { if (defined($main::collected_profile)) {
# if used with remote fetch, simply dump the collected profile to output. # if used with remote fetch, simply dump the collected profile to output.
...@@ -1147,6 +1190,12 @@ sub PrintSymbolizedProfile { ...@@ -1147,6 +1190,12 @@ sub PrintSymbolizedProfile {
} }
close(SRC); close(SRC);
} else { } else {
# --raw/http: For everything to work correctly for non-remote profiles, we
# would need to extend PrintProfileData() to handle all possible profile
# types, re-enable the code that is currently disabled in ReadCPUProfile()
# and FixCallerAddresses(), and remove the remote profile dumping code in
# the block above.
die "--raw/http: jeprof can only dump remote profiles for --raw\n";
# dump a cpu-format profile to standard out # dump a cpu-format profile to standard out
PrintProfileData($profile); PrintProfileData($profile);
} }
...@@ -1476,7 +1525,7 @@ h1 { ...@@ -1476,7 +1525,7 @@ h1 {
} }
</style> </style>
<script type="text/javascript"> <script type="text/javascript">
function pprof_toggle_asm(e) { function jeprof_toggle_asm(e) {
var target; var target;
if (!e) e = window.event; if (!e) e = window.event;
if (e.target) target = e.target; if (e.target) target = e.target;
...@@ -1683,23 +1732,23 @@ sub PrintSource { ...@@ -1683,23 +1732,23 @@ sub PrintSource {
HtmlPrintNumber($c2), HtmlPrintNumber($c2),
UnparseAddress($offset, $e->[0]), UnparseAddress($offset, $e->[0]),
CleanDisassembly($e->[3])); CleanDisassembly($e->[3]));
# Append the most specific source line associated with this instruction # Append the most specific source line associated with this instruction
if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) }; if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) };
$dis = HtmlEscape($dis); $dis = HtmlEscape($dis);
my $f = $e->[5]; my $f = $e->[5];
my $l = $e->[6]; my $l = $e->[6];
if ($f ne $last_dis_filename) { if ($f ne $last_dis_filename) {
$dis .= sprintf("<span class=disasmloc>%s:%d</span>", $dis .= sprintf("<span class=disasmloc>%s:%d</span>",
HtmlEscape(CleanFileName($f)), $l); HtmlEscape(CleanFileName($f)), $l);
} elsif ($l ne $last_dis_linenum) { } elsif ($l ne $last_dis_linenum) {
# De-emphasize the unchanged file name portion # De-emphasize the unchanged file name portion
$dis .= sprintf("<span class=unimportant>%s</span>" . $dis .= sprintf("<span class=unimportant>%s</span>" .
"<span class=disasmloc>:%d</span>", "<span class=disasmloc>:%d</span>",
HtmlEscape(CleanFileName($f)), $l); HtmlEscape(CleanFileName($f)), $l);
} else { } else {
# De-emphasize the entire location # De-emphasize the entire location
$dis .= sprintf("<span class=unimportant>%s:%d</span>", $dis .= sprintf("<span class=unimportant>%s:%d</span>",
HtmlEscape(CleanFileName($f)), $l); HtmlEscape(CleanFileName($f)), $l);
} }
$last_dis_filename = $f; $last_dis_filename = $f;
...@@ -1745,7 +1794,7 @@ sub PrintSource { ...@@ -1745,7 +1794,7 @@ sub PrintSource {
if ($html) { if ($html) {
printf $output ( printf $output (
"<h1>%s</h1>%s\n<pre onClick=\"pprof_toggle_asm()\">\n" . "<h1>%s</h1>%s\n<pre onClick=\"jeprof_toggle_asm()\">\n" .
"Total:%6s %6s (flat / cumulative %s)\n", "Total:%6s %6s (flat / cumulative %s)\n",
HtmlEscape(ShortFunctionName($routine)), HtmlEscape(ShortFunctionName($routine)),
HtmlEscape(CleanFileName($filename)), HtmlEscape(CleanFileName($filename)),
...@@ -1788,8 +1837,8 @@ sub PrintSource { ...@@ -1788,8 +1837,8 @@ sub PrintSource {
if (defined($dis) && $dis ne '') { if (defined($dis) && $dis ne '') {
$asm = "<span class=\"asm\">" . $dis . "</span>"; $asm = "<span class=\"asm\">" . $dis . "</span>";
} }
my $source_class = (($n1 + $n2 > 0) my $source_class = (($n1 + $n2 > 0)
? "livesrc" ? "livesrc"
: (($asm ne "") ? "deadsrc" : "nop")); : (($asm ne "") ? "deadsrc" : "nop"));
printf $output ( printf $output (
"<span class=\"line\">%5d</span> " . "<span class=\"line\">%5d</span> " .
...@@ -2797,6 +2846,43 @@ sub ExtractCalls { ...@@ -2797,6 +2846,43 @@ sub ExtractCalls {
return $calls; return $calls;
} }
sub FilterFrames {
my $symbols = shift;
my $profile = shift;
if ($main::opt_retain eq '' && $main::opt_exclude eq '') {
return $profile;
}
my $result = {};
foreach my $k (keys(%{$profile})) {
my $count = $profile->{$k};
my @addrs = split(/\n/, $k);
my @path = ();
foreach my $a (@addrs) {
my $sym;
if (exists($symbols->{$a})) {
$sym = $symbols->{$a}->[0];
} else {
$sym = $a;
}
if ($main::opt_retain ne '' && $sym !~ m/$main::opt_retain/) {
next;
}
if ($main::opt_exclude ne '' && $sym =~ m/$main::opt_exclude/) {
next;
}
push(@path, $a);
}
if (scalar(@path) > 0) {
my $reduced_path = join("\n", @path);
AddEntry($result, $reduced_path, $count);
}
}
return $result;
}
sub RemoveUninterestingFrames { sub RemoveUninterestingFrames {
my $symbols = shift; my $symbols = shift;
my $profile = shift; my $profile = shift;
...@@ -2811,9 +2897,15 @@ sub RemoveUninterestingFrames { ...@@ -2811,9 +2897,15 @@ sub RemoveUninterestingFrames {
'free', 'free',
'memalign', 'memalign',
'posix_memalign', 'posix_memalign',
'aligned_alloc',
'pvalloc', 'pvalloc',
'valloc', 'valloc',
'realloc', 'realloc',
'mallocx', # jemalloc
'rallocx', # jemalloc
'xallocx', # jemalloc
'dallocx', # jemalloc
'sdallocx', # jemalloc
'tc_calloc', 'tc_calloc',
'tc_cfree', 'tc_cfree',
'tc_malloc', 'tc_malloc',
...@@ -2923,6 +3015,10 @@ sub RemoveUninterestingFrames { ...@@ -2923,6 +3015,10 @@ sub RemoveUninterestingFrames {
if (exists($symbols->{$a})) { if (exists($symbols->{$a})) {
my $func = $symbols->{$a}->[0]; my $func = $symbols->{$a}->[0];
if ($skip{$func} || ($func =~ m/$skip_regexp/)) { if ($skip{$func} || ($func =~ m/$skip_regexp/)) {
# Throw away the portion of the backtrace seen so far, under the
# assumption that previous frames were for functions internal to the
# allocator.
@path = ();
next; next;
} }
} }
...@@ -2931,6 +3027,9 @@ sub RemoveUninterestingFrames { ...@@ -2931,6 +3027,9 @@ sub RemoveUninterestingFrames {
my $reduced_path = join("\n", @path); my $reduced_path = join("\n", @path);
AddEntry($result, $reduced_path, $count); AddEntry($result, $reduced_path, $count);
} }
$result = FilterFrames($symbols, $result);
return $result; return $result;
} }
...@@ -3240,7 +3339,7 @@ sub ResolveRedirectionForCurl { ...@@ -3240,7 +3339,7 @@ sub ResolveRedirectionForCurl {
# Add a timeout flat to URL_FETCHER. Returns a new list. # Add a timeout flat to URL_FETCHER. Returns a new list.
sub AddFetchTimeout { sub AddFetchTimeout {
my $timeout = shift; my $timeout = shift;
my @fetcher = shift; my @fetcher = @_;
if (defined($timeout)) { if (defined($timeout)) {
if (join(" ", @fetcher) =~ m/\bcurl -s/) { if (join(" ", @fetcher) =~ m/\bcurl -s/) {
push(@fetcher, "--max-time", sprintf("%d", $timeout)); push(@fetcher, "--max-time", sprintf("%d", $timeout));
...@@ -3286,6 +3385,27 @@ sub ReadSymbols { ...@@ -3286,6 +3385,27 @@ sub ReadSymbols {
return $map; return $map;
} }
sub URLEncode {
my $str = shift;
$str =~ s/([^A-Za-z0-9\-_.!~*'()])/ sprintf "%%%02x", ord $1 /eg;
return $str;
}
sub AppendSymbolFilterParams {
my $url = shift;
my @params = ();
if ($main::opt_retain ne '') {
push(@params, sprintf("retain=%s", URLEncode($main::opt_retain)));
}
if ($main::opt_exclude ne '') {
push(@params, sprintf("exclude=%s", URLEncode($main::opt_exclude)));
}
if (scalar @params > 0) {
$url = sprintf("%s?%s", $url, join("&", @params));
}
return $url;
}
# Fetches and processes symbols to prepare them for use in the profile output # Fetches and processes symbols to prepare them for use in the profile output
# code. If the optional 'symbol_map' arg is not given, fetches symbols from # code. If the optional 'symbol_map' arg is not given, fetches symbols from
# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols # $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols
...@@ -3310,9 +3430,11 @@ sub FetchSymbols { ...@@ -3310,9 +3430,11 @@ sub FetchSymbols {
my $command_line; my $command_line;
if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) { if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) {
$url = ResolveRedirectionForCurl($url); $url = ResolveRedirectionForCurl($url);
$url = AppendSymbolFilterParams($url);
$command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym", $command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym",
$url); $url);
} else { } else {
$url = AppendSymbolFilterParams($url);
$command_line = (ShellEscape(@URL_FETCHER, "--post", $url) $command_line = (ShellEscape(@URL_FETCHER, "--post", $url)
. " < " . ShellEscape($main::tmpfile_sym)); . " < " . ShellEscape($main::tmpfile_sym));
} }
...@@ -3393,15 +3515,25 @@ sub FetchDynamicProfile { ...@@ -3393,15 +3515,25 @@ sub FetchDynamicProfile {
} }
$url .= sprintf("seconds=%d", $main::opt_seconds); $url .= sprintf("seconds=%d", $main::opt_seconds);
$fetch_timeout = $main::opt_seconds * 1.01 + 60; $fetch_timeout = $main::opt_seconds * 1.01 + 60;
# Set $profile_type for consumption by PrintSymbolizedProfile.
$main::profile_type = 'cpu';
} else { } else {
# For non-CPU profiles, we add a type-extension to # For non-CPU profiles, we add a type-extension to
# the target profile file name. # the target profile file name.
my $suffix = $path; my $suffix = $path;
$suffix =~ s,/,.,g; $suffix =~ s,/,.,g;
$profile_file .= $suffix; $profile_file .= $suffix;
# Set $profile_type for consumption by PrintSymbolizedProfile.
if ($path =~ m/$HEAP_PAGE/) {
$main::profile_type = 'heap';
} elsif ($path =~ m/$GROWTH_PAGE/) {
$main::profile_type = 'growth';
} elsif ($path =~ m/$CONTENTION_PAGE/) {
$main::profile_type = 'contention';
}
} }
my $profile_dir = $ENV{"PPROF_TMPDIR"} || ($ENV{HOME} . "/pprof"); my $profile_dir = $ENV{"JEPROF_TMPDIR"} || ($ENV{HOME} . "/jeprof");
if (! -d $profile_dir) { if (! -d $profile_dir) {
mkdir($profile_dir) mkdir($profile_dir)
|| die("Unable to create profile directory $profile_dir: $!\n"); || die("Unable to create profile directory $profile_dir: $!\n");
...@@ -3617,7 +3749,7 @@ BEGIN { ...@@ -3617,7 +3749,7 @@ BEGIN {
# Reads the top, 'header' section of a profile, and returns the last # Reads the top, 'header' section of a profile, and returns the last
# line of the header, commonly called a 'header line'. The header # line of the header, commonly called a 'header line'. The header
# section of a profile consists of zero or more 'command' lines that # section of a profile consists of zero or more 'command' lines that
# are instructions to pprof, which pprof executes when reading the # are instructions to jeprof, which jeprof executes when reading the
# header. All 'command' lines start with a %. After the command # header. All 'command' lines start with a %. After the command
# lines is the 'header line', which is a profile-specific line that # lines is the 'header line', which is a profile-specific line that
# indicates what type of profile it is, and perhaps other global # indicates what type of profile it is, and perhaps other global
...@@ -3680,6 +3812,7 @@ sub IsSymbolizedProfileFile { ...@@ -3680,6 +3812,7 @@ sub IsSymbolizedProfileFile {
# $result->{version} Version number of profile file # $result->{version} Version number of profile file
# $result->{period} Sampling period (in microseconds) # $result->{period} Sampling period (in microseconds)
# $result->{profile} Profile object # $result->{profile} Profile object
# $result->{threads} Map of thread IDs to profile objects
# $result->{map} Memory map info from profile # $result->{map} Memory map info from profile
# $result->{pcs} Hash of all PC values seen, key is hex address # $result->{pcs} Hash of all PC values seen, key is hex address
sub ReadProfile { sub ReadProfile {
...@@ -3695,6 +3828,8 @@ sub ReadProfile { ...@@ -3695,6 +3828,8 @@ sub ReadProfile {
my $symbol_marker = $&; my $symbol_marker = $&;
$PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $profile_marker = $&; my $profile_marker = $&;
$HEAP_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $heap_marker = $&;
# Look at first line to see if it is a heap or a CPU profile. # Look at first line to see if it is a heap or a CPU profile.
# CPU profile may start with no header at all, and just binary data # CPU profile may start with no header at all, and just binary data
...@@ -3721,13 +3856,22 @@ sub ReadProfile { ...@@ -3721,13 +3856,22 @@ sub ReadProfile {
$header = ReadProfileHeader(*PROFILE) || ""; $header = ReadProfileHeader(*PROFILE) || "";
} }
if ($header =~ m/^--- *($heap_marker|$growth_marker)/o) {
# Skip "--- ..." line for profile types that have their own headers.
$header = ReadProfileHeader(*PROFILE) || "";
}
$main::profile_type = ''; $main::profile_type = '';
if ($header =~ m/^heap profile:.*$growth_marker/o) { if ($header =~ m/^heap profile:.*$growth_marker/o) {
$main::profile_type = 'growth'; $main::profile_type = 'growth';
$result = ReadHeapProfile($prog, *PROFILE, $header); $result = ReadHeapProfile($prog, *PROFILE, $header);
} elsif ($header =~ m/^heap profile:/) { } elsif ($header =~ m/^heap profile:/) {
$main::profile_type = 'heap'; $main::profile_type = 'heap';
$result = ReadHeapProfile($prog, *PROFILE, $header); $result = ReadHeapProfile($prog, *PROFILE, $header);
} elsif ($header =~ m/^heap/) {
$main::profile_type = 'heap';
$result = ReadThreadedHeapProfile($prog, $fname, $header);
} elsif ($header =~ m/^--- *$contention_marker/o) { } elsif ($header =~ m/^--- *$contention_marker/o) {
$main::profile_type = 'contention'; $main::profile_type = 'contention';
$result = ReadSynchProfile($prog, *PROFILE); $result = ReadSynchProfile($prog, *PROFILE);
...@@ -3770,9 +3914,9 @@ sub ReadProfile { ...@@ -3770,9 +3914,9 @@ sub ReadProfile {
# independent implementation. # independent implementation.
sub FixCallerAddresses { sub FixCallerAddresses {
my $stack = shift; my $stack = shift;
if ($main::use_symbolized_profile) { # --raw/http: Always subtract one from pc's, because PrintSymbolizedProfile()
return $stack; # dumps unadjusted profiles.
} else { {
$stack =~ /(\s)/; $stack =~ /(\s)/;
my $delimiter = $1; my $delimiter = $1;
my @addrs = split(' ', $stack); my @addrs = split(' ', $stack);
...@@ -3840,12 +3984,7 @@ sub ReadCPUProfile { ...@@ -3840,12 +3984,7 @@ sub ReadCPUProfile {
for (my $j = 0; $j < $d; $j++) { for (my $j = 0; $j < $d; $j++) {
my $pc = $slots->get($i+$j); my $pc = $slots->get($i+$j);
# Subtract one from caller pc so we map back to call instr. # Subtract one from caller pc so we map back to call instr.
# However, don't do this if we're reading a symbolized profile $pc--;
# file, in which case the subtract-one was done when the file
# was written.
if ($j > 0 && !$main::use_symbolized_profile) {
$pc--;
}
$pc = sprintf("%0*x", $address_length, $pc); $pc = sprintf("%0*x", $address_length, $pc);
$pcs->{$pc} = 1; $pcs->{$pc} = 1;
push @k, $pc; push @k, $pc;
...@@ -3870,11 +4009,7 @@ sub ReadCPUProfile { ...@@ -3870,11 +4009,7 @@ sub ReadCPUProfile {
return $r; return $r;
} }
sub ReadHeapProfile { sub HeapProfileIndex {
my $prog = shift;
local *PROFILE = shift;
my $header = shift;
my $index = 1; my $index = 1;
if ($main::opt_inuse_space) { if ($main::opt_inuse_space) {
$index = 1; $index = 1;
...@@ -3885,6 +4020,84 @@ sub ReadHeapProfile { ...@@ -3885,6 +4020,84 @@ sub ReadHeapProfile {
} elsif ($main::opt_alloc_objects) { } elsif ($main::opt_alloc_objects) {
$index = 2; $index = 2;
} }
return $index;
}
sub ReadMappedLibraries {
my $fh = shift;
my $map = "";
# Read the /proc/self/maps data
while (<$fh>) {
s/\r//g; # turn windows-looking lines into unix-looking lines
$map .= $_;
}
return $map;
}
sub ReadMemoryMap {
my $fh = shift;
my $map = "";
# Read /proc/self/maps data as formatted by DumpAddressMap()
my $buildvar = "";
while (<PROFILE>) {
s/\r//g; # turn windows-looking lines into unix-looking lines
# Parse "build=<dir>" specification if supplied
if (m/^\s*build=(.*)\n/) {
$buildvar = $1;
}
# Expand "$build" variable if available
$_ =~ s/\$build\b/$buildvar/g;
$map .= $_;
}
return $map;
}
sub AdjustSamples {
my ($sample_adjustment, $sampling_algorithm, $n1, $s1, $n2, $s2) = @_;
if ($sample_adjustment) {
if ($sampling_algorithm == 2) {
# Remote-heap version 2
# The sampling frequency is the rate of a Poisson process.
# This means that the probability of sampling an allocation of
# size X with sampling rate Y is 1 - exp(-X/Y)
if ($n1 != 0) {
my $ratio = (($s1*1.0)/$n1)/($sample_adjustment);
my $scale_factor = 1/(1 - exp(-$ratio));
$n1 *= $scale_factor;
$s1 *= $scale_factor;
}
if ($n2 != 0) {
my $ratio = (($s2*1.0)/$n2)/($sample_adjustment);
my $scale_factor = 1/(1 - exp(-$ratio));
$n2 *= $scale_factor;
$s2 *= $scale_factor;
}
} else {
# Remote-heap version 1
my $ratio;
$ratio = (($s1*1.0)/$n1)/($sample_adjustment);
if ($ratio < 1) {
$n1 /= $ratio;
$s1 /= $ratio;
}
$ratio = (($s2*1.0)/$n2)/($sample_adjustment);
if ($ratio < 1) {
$n2 /= $ratio;
$s2 /= $ratio;
}
}
}
return ($n1, $s1, $n2, $s2);
}
sub ReadHeapProfile {
my $prog = shift;
local *PROFILE = shift;
my $header = shift;
my $index = HeapProfileIndex();
# Find the type of this profile. The header line looks like: # Find the type of this profile. The header line looks like:
# heap profile: 1246: 8800744 [ 1246: 8800744] @ <heap-url>/266053 # heap profile: 1246: 8800744 [ 1246: 8800744] @ <heap-url>/266053
...@@ -3974,29 +4187,12 @@ sub ReadHeapProfile { ...@@ -3974,29 +4187,12 @@ sub ReadHeapProfile {
while (<PROFILE>) { while (<PROFILE>) {
s/\r//g; # turn windows-looking lines into unix-looking lines s/\r//g; # turn windows-looking lines into unix-looking lines
if (/^MAPPED_LIBRARIES:/) { if (/^MAPPED_LIBRARIES:/) {
# Read the /proc/self/maps data $map .= ReadMappedLibraries(*PROFILE);
while (<PROFILE>) {
s/\r//g; # turn windows-looking lines into unix-looking lines
$map .= $_;
}
last; last;
} }
if (/^--- Memory map:/) { if (/^--- Memory map:/) {
# Read /proc/self/maps data as formatted by DumpAddressMap() $map .= ReadMemoryMap(*PROFILE);
my $buildvar = "";
while (<PROFILE>) {
s/\r//g; # turn windows-looking lines into unix-looking lines
# Parse "build=<dir>" specification if supplied
if (m/^\s*build=(.*)\n/) {
$buildvar = $1;
}
# Expand "$build" variable if available
$_ =~ s/\$build\b/$buildvar/g;
$map .= $_;
}
last; last;
} }
...@@ -4007,43 +4203,85 @@ sub ReadHeapProfile { ...@@ -4007,43 +4203,85 @@ sub ReadHeapProfile {
if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) {
my $stack = $5; my $stack = $5;
my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4);
my @counts = AdjustSamples($sample_adjustment, $sampling_algorithm,
$n1, $s1, $n2, $s2);
AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]);
}
}
if ($sample_adjustment) { my $r = {};
if ($sampling_algorithm == 2) { $r->{version} = "heap";
# Remote-heap version 2 $r->{period} = 1;
# The sampling frequency is the rate of a Poisson process. $r->{profile} = $profile;
# This means that the probability of sampling an allocation of $r->{libs} = ParseLibraries($prog, $map, $pcs);
# size X with sampling rate Y is 1 - exp(-X/Y) $r->{pcs} = $pcs;
if ($n1 != 0) { return $r;
my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); }
my $scale_factor = 1/(1 - exp(-$ratio));
$n1 *= $scale_factor; sub ReadThreadedHeapProfile {
$s1 *= $scale_factor; my ($prog, $fname, $header) = @_;
}
if ($n2 != 0) { my $index = HeapProfileIndex();
my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); my $sampling_algorithm = 0;
my $scale_factor = 1/(1 - exp(-$ratio)); my $sample_adjustment = 0;
$n2 *= $scale_factor; chomp($header);
$s2 *= $scale_factor; my $type = "unknown";
} # Assuming a very specific type of header for now.
} else { if ($header =~ m"^heap_v2/(\d+)") {
# Remote-heap version 1 $type = "_v2";
my $ratio; $sampling_algorithm = 2;
$ratio = (($s1*1.0)/$n1)/($sample_adjustment); $sample_adjustment = int($1);
if ($ratio < 1) { }
$n1 /= $ratio; if ($type ne "_v2" || !defined($sample_adjustment)) {
$s1 /= $ratio; die "Threaded heap profiles require v2 sampling with a sample rate\n";
} }
$ratio = (($s2*1.0)/$n2)/($sample_adjustment);
if ($ratio < 1) { my $profile = {};
$n2 /= $ratio; my $thread_profiles = {};
$s2 /= $ratio; my $pcs = {};
} my $map = "";
my $stack = "";
while (<PROFILE>) {
s/\r//g;
if (/^MAPPED_LIBRARIES:/) {
$map .= ReadMappedLibraries(*PROFILE);
last;
}
if (/^--- Memory map:/) {
$map .= ReadMemoryMap(*PROFILE);
last;
}
# Read entry of the form:
# @ a1 a2 ... an
# t*: <count1>: <bytes1> [<count2>: <bytes2>]
# t1: <count1>: <bytes1> [<count2>: <bytes2>]
# ...
# tn: <count1>: <bytes1> [<count2>: <bytes2>]
s/^\s*//;
s/\s*$//;
if (m/^@\s+(.*)$/) {
$stack = $1;
} elsif (m/^\s*(t(\*|\d+)):\s+(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]$/) {
if ($stack eq "") {
# Still in the header, so this is just a per-thread summary.
next;
}
my $thread = $2;
my ($n1, $s1, $n2, $s2) = ($3, $4, $5, $6);
my @counts = AdjustSamples($sample_adjustment, $sampling_algorithm,
$n1, $s1, $n2, $s2);
if ($thread eq "*") {
AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]);
} else {
if (!exists($thread_profiles->{$thread})) {
$thread_profiles->{$thread} = {};
} }
AddEntries($thread_profiles->{$thread}, $pcs,
FixCallerAddresses($stack), $counts[$index]);
} }
my @counts = ($n1, $s1, $n2, $s2);
AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]);
} }
} }
...@@ -4051,6 +4289,7 @@ sub ReadHeapProfile { ...@@ -4051,6 +4289,7 @@ sub ReadHeapProfile {
$r->{version} = "heap"; $r->{version} = "heap";
$r->{period} = 1; $r->{period} = 1;
$r->{profile} = $profile; $r->{profile} = $profile;
$r->{threads} = $thread_profiles;
$r->{libs} = ParseLibraries($prog, $map, $pcs); $r->{libs} = ParseLibraries($prog, $map, $pcs);
$r->{pcs} = $pcs; $r->{pcs} = $pcs;
return $r; return $r;
...@@ -4120,10 +4359,10 @@ sub ReadSynchProfile { ...@@ -4120,10 +4359,10 @@ sub ReadSynchProfile {
} elsif ($variable eq "sampling period") { } elsif ($variable eq "sampling period") {
$sampling_period = $value; $sampling_period = $value;
} elsif ($variable eq "ms since reset") { } elsif ($variable eq "ms since reset") {
# Currently nothing is done with this value in pprof # Currently nothing is done with this value in jeprof
# So we just silently ignore it for now # So we just silently ignore it for now
} elsif ($variable eq "discarded samples") { } elsif ($variable eq "discarded samples") {
# Currently nothing is done with this value in pprof # Currently nothing is done with this value in jeprof
# So we just silently ignore it for now # So we just silently ignore it for now
} else { } else {
printf STDERR ("Ignoring unnknown variable in /contention output: " . printf STDERR ("Ignoring unnknown variable in /contention output: " .
...@@ -4197,8 +4436,12 @@ sub FindLibrary { ...@@ -4197,8 +4436,12 @@ sub FindLibrary {
# For libc libraries, the copy in /usr/lib/debug contains debugging symbols # For libc libraries, the copy in /usr/lib/debug contains debugging symbols
sub DebuggingLibrary { sub DebuggingLibrary {
my $file = shift; my $file = shift;
if ($file =~ m|^/| && -f "/usr/lib/debug$file") { if ($file =~ m|^/|) {
return "/usr/lib/debug$file"; if (-f "/usr/lib/debug$file") {
return "/usr/lib/debug$file";
} elsif (-f "/usr/lib/debug$file.debug") {
return "/usr/lib/debug$file.debug";
}
} }
return undef; return undef;
} }
...@@ -4360,6 +4603,19 @@ sub ParseLibraries { ...@@ -4360,6 +4603,19 @@ sub ParseLibraries {
$finish = HexExtend($2); $finish = HexExtend($2);
$offset = $zero_offset; $offset = $zero_offset;
$lib = $3; $lib = $3;
}
# FreeBSD 10.0 virtual memory map /proc/curproc/map as defined in
# function procfs_doprocmap (sys/fs/procfs/procfs_map.c)
#
# Example:
# 0x800600000 0x80061a000 26 0 0xfffff800035a0000 r-x 75 33 0x1004 COW NC vnode /libexec/ld-elf.s
# o.1 NCH -1
elsif ($l =~ /^(0x$h)\s(0x$h)\s\d+\s\d+\s0x$h\sr-x\s\d+\s\d+\s0x\d+\s(COW|NCO)\s(NC|NNC)\svnode\s(\S+\.so(\.\d+)*)/) {
$start = HexExtend($1);
$finish = HexExtend($2);
$offset = $zero_offset;
$lib = FindLibrary($5);
} else { } else {
next; next;
} }
...@@ -4382,6 +4638,7 @@ sub ParseLibraries { ...@@ -4382,6 +4638,7 @@ sub ParseLibraries {
} }
} }
if($main::opt_debug) { printf STDERR "$start:$finish ($offset) $lib\n"; }
push(@{$result}, [$lib, $start, $finish, $offset]); push(@{$result}, [$lib, $start, $finish, $offset]);
} }
...@@ -4411,7 +4668,7 @@ sub ParseLibraries { ...@@ -4411,7 +4668,7 @@ sub ParseLibraries {
} }
# Add two hex addresses of length $address_length. # Add two hex addresses of length $address_length.
# Run pprof --test for unit test if this is changed. # Run jeprof --test for unit test if this is changed.
sub AddressAdd { sub AddressAdd {
my $addr1 = shift; my $addr1 = shift;
my $addr2 = shift; my $addr2 = shift;
...@@ -4465,7 +4722,7 @@ sub AddressAdd { ...@@ -4465,7 +4722,7 @@ sub AddressAdd {
# Subtract two hex addresses of length $address_length. # Subtract two hex addresses of length $address_length.
# Run pprof --test for unit test if this is changed. # Run jeprof --test for unit test if this is changed.
sub AddressSub { sub AddressSub {
my $addr1 = shift; my $addr1 = shift;
my $addr2 = shift; my $addr2 = shift;
...@@ -4517,7 +4774,7 @@ sub AddressSub { ...@@ -4517,7 +4774,7 @@ sub AddressSub {
} }
# Increment a hex addresses of length $address_length. # Increment a hex addresses of length $address_length.
# Run pprof --test for unit test if this is changed. # Run jeprof --test for unit test if this is changed.
sub AddressInc { sub AddressInc {
my $addr = shift; my $addr = shift;
my $sum; my $sum;
...@@ -4589,6 +4846,12 @@ sub ExtractSymbols { ...@@ -4589,6 +4846,12 @@ sub ExtractSymbols {
my $finish = $lib->[2]; my $finish = $lib->[2];
my $offset = $lib->[3]; my $offset = $lib->[3];
# Use debug library if it exists
my $debug_libname = DebuggingLibrary($libname);
if ($debug_libname) {
$libname = $debug_libname;
}
# Get list of pcs that belong in this library. # Get list of pcs that belong in this library.
my $contained = []; my $contained = [];
my ($start_pc_index, $finish_pc_index); my ($start_pc_index, $finish_pc_index);
...@@ -4723,7 +4986,7 @@ sub MapToSymbols { ...@@ -4723,7 +4986,7 @@ sub MapToSymbols {
} }
} }
} }
# Prepend to accumulated symbols for pcstr # Prepend to accumulated symbols for pcstr
# (so that caller comes before callee) # (so that caller comes before callee)
my $sym = $symbols->{$pcstr}; my $sym = $symbols->{$pcstr};
...@@ -4829,7 +5092,7 @@ sub UnparseAddress { ...@@ -4829,7 +5092,7 @@ sub UnparseAddress {
# 32-bit or ELF 64-bit executable file. The location of the tools # 32-bit or ELF 64-bit executable file. The location of the tools
# is determined by considering the following options in this order: # is determined by considering the following options in this order:
# 1) --tools option, if set # 1) --tools option, if set
# 2) PPROF_TOOLS environment variable, if set # 2) JEPROF_TOOLS environment variable, if set
# 3) the environment # 3) the environment
sub ConfigureObjTools { sub ConfigureObjTools {
my $prog_file = shift; my $prog_file = shift;
...@@ -4862,7 +5125,7 @@ sub ConfigureObjTools { ...@@ -4862,7 +5125,7 @@ sub ConfigureObjTools {
# For windows, we provide a version of nm and addr2line as part of # For windows, we provide a version of nm and addr2line as part of
# the opensource release, which is capable of parsing # the opensource release, which is capable of parsing
# Windows-style PDB executables. It should live in the path, or # Windows-style PDB executables. It should live in the path, or
# in the same directory as pprof. # in the same directory as jeprof.
$obj_tool_map{"nm_pdb"} = "nm-pdb"; $obj_tool_map{"nm_pdb"} = "nm-pdb";
$obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb";
} }
...@@ -4881,20 +5144,20 @@ sub ConfigureObjTools { ...@@ -4881,20 +5144,20 @@ sub ConfigureObjTools {
} }
# Returns the path of a caller-specified object tool. If --tools or # Returns the path of a caller-specified object tool. If --tools or
# PPROF_TOOLS are specified, then returns the full path to the tool # JEPROF_TOOLS are specified, then returns the full path to the tool
# with that prefix. Otherwise, returns the path unmodified (which # with that prefix. Otherwise, returns the path unmodified (which
# means we will look for it on PATH). # means we will look for it on PATH).
sub ConfigureTool { sub ConfigureTool {
my $tool = shift; my $tool = shift;
my $path; my $path;
# --tools (or $PPROF_TOOLS) is a comma separated list, where each # --tools (or $JEPROF_TOOLS) is a comma separated list, where each
# item is either a) a pathname prefix, or b) a map of the form # item is either a) a pathname prefix, or b) a map of the form
# <tool>:<path>. First we look for an entry of type (b) for our # <tool>:<path>. First we look for an entry of type (b) for our
# tool. If one is found, we use it. Otherwise, we consider all the # tool. If one is found, we use it. Otherwise, we consider all the
# pathname prefixes in turn, until one yields an existing file. If # pathname prefixes in turn, until one yields an existing file. If
# none does, we use a default path. # none does, we use a default path.
my $tools = $main::opt_tools || $ENV{"PPROF_TOOLS"} || ""; my $tools = $main::opt_tools || $ENV{"JEPROF_TOOLS"} || "";
if ($tools =~ m/(,|^)\Q$tool\E:([^,]*)/) { if ($tools =~ m/(,|^)\Q$tool\E:([^,]*)/) {
$path = $2; $path = $2;
# TODO(csilvers): sanity-check that $path exists? Hard if it's relative. # TODO(csilvers): sanity-check that $path exists? Hard if it's relative.
...@@ -4908,16 +5171,16 @@ sub ConfigureTool { ...@@ -4908,16 +5171,16 @@ sub ConfigureTool {
} }
if (!$path) { if (!$path) {
error("No '$tool' found with prefix specified by " . error("No '$tool' found with prefix specified by " .
"--tools (or \$PPROF_TOOLS) '$tools'\n"); "--tools (or \$JEPROF_TOOLS) '$tools'\n");
} }
} else { } else {
# ... otherwise use the version that exists in the same directory as # ... otherwise use the version that exists in the same directory as
# pprof. If there's nothing there, use $PATH. # jeprof. If there's nothing there, use $PATH.
$0 =~ m,[^/]*$,; # this is everything after the last slash $0 =~ m,[^/]*$,; # this is everything after the last slash
my $dirname = $`; # this is everything up to and including the last slash my $dirname = $`; # this is everything up to and including the last slash
if (-x "$dirname$tool") { if (-x "$dirname$tool") {
$path = "$dirname$tool"; $path = "$dirname$tool";
} else { } else {
$path = $tool; $path = $tool;
} }
} }
...@@ -4942,7 +5205,7 @@ sub cleanup { ...@@ -4942,7 +5205,7 @@ sub cleanup {
unlink($main::tmpfile_sym); unlink($main::tmpfile_sym);
unlink(keys %main::tempnames); unlink(keys %main::tempnames);
# We leave any collected profiles in $HOME/pprof in case the user wants # We leave any collected profiles in $HOME/jeprof in case the user wants
# to look at them later. We print a message informing them of this. # to look at them later. We print a message informing them of this.
if ((scalar(@main::profile_files) > 0) && if ((scalar(@main::profile_files) > 0) &&
defined($main::collected_profile)) { defined($main::collected_profile)) {
...@@ -4951,7 +5214,7 @@ sub cleanup { ...@@ -4951,7 +5214,7 @@ sub cleanup {
} }
print STDERR "If you want to investigate this profile further, you can do:\n"; print STDERR "If you want to investigate this profile further, you can do:\n";
print STDERR "\n"; print STDERR "\n";
print STDERR " pprof \\\n"; print STDERR " jeprof \\\n";
print STDERR " $main::prog \\\n"; print STDERR " $main::prog \\\n";
print STDERR " $main::collected_profile\n"; print STDERR " $main::collected_profile\n";
print STDERR "\n"; print STDERR "\n";
...@@ -5019,7 +5282,7 @@ sub GetProcedureBoundariesViaNm { ...@@ -5019,7 +5282,7 @@ sub GetProcedureBoundariesViaNm {
# Tag this routine with the starting address in case the image # Tag this routine with the starting address in case the image
# has multiple occurrences of this routine. We use a syntax # has multiple occurrences of this routine. We use a syntax
# that resembles template paramters that are automatically # that resembles template parameters that are automatically
# stripped out by ShortFunctionName() # stripped out by ShortFunctionName()
$this_routine .= "<$start_val>"; $this_routine .= "<$start_val>";
...@@ -5136,7 +5399,7 @@ sub GetProcedureBoundaries { ...@@ -5136,7 +5399,7 @@ sub GetProcedureBoundaries {
# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. # The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings.
# To make them more readable, we add underscores at interesting places. # To make them more readable, we add underscores at interesting places.
# This routine removes the underscores, producing the canonical representation # This routine removes the underscores, producing the canonical representation
# used by pprof to represent addresses, particularly in the tested routines. # used by jeprof to represent addresses, particularly in the tested routines.
sub CanonicalHex { sub CanonicalHex {
my $arg = shift; my $arg = shift;
return join '', (split '_',$arg); return join '', (split '_',$arg);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment