Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
eb46f4bd
Commit
eb46f4bd
authored
Feb 27, 2010
by
antirez
Browse files
initial zipmap.c implementation
parent
5921aa36
Changes
1
Hide whitespace changes
Inline
Side-by-side
zipmap.c
0 → 100644
View file @
eb46f4bd
/* String -> String Map data structure optimized for size.
* This file implements a data structure mapping strings to other strings
* implementing an O(n) lookup data structure designed to be very memory
* efficient.
*
* The Redis Hash type uses this data structure for hashes composed of a small
* number of elements, to switch to an hash table once a given number of
* elements is reached.
*
* Given that many times Redis Hashes are used to represent objects composed
* of few fields, this is a very big win in terms of used memory.
*
* --------------------------------------------------------------------------
*
* Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
*
* <status><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
*
* <status> is 1 byte status. Currently only 1 bit is used: if the least
* significant bit is set, it means the zipmap needs to be defragmented.
*
* <len> is the length of the following string (key or value).
* <len> lengths are encoded in a single value or in a 5 bytes value.
* If the first byte value (as an unsigned 8 bit value) is between 0 and
* 252, it's a single-byte length. If it is 253 then a four bytes unsigned
* integer follows (in the host byte ordering). A value fo 255 is used to
* signal the end of the hash. The special value 254 is used to mark
* empty space that can be used to add new key/value pairs.
*
* <free> is the number of free unused bytes
* after the string, resulting from modification of values associated to a
* key (for instance if "foo" is set to "bar', and later "foo" will be se to
* "hi", I'll have a free byte to use if the value will enlarge again later,
* or even in order to add a key/value pair if it fits.
*
* <free> is always an unsigned 8 bit number, because if after an
* update operation there are more than a few free bytes, they'll be converted
* into empty space prefixed by the special value 254.
*
* The most compact representation of the above two elements hash is actually:
*
* "\x00\x03\x00foo\x03\x00bar\x05\x00hello\x05\x00world\xff"
*
* Empty space is marked using a 254 bytes + a <len> (coded as already
* specified). The length includes the 254 bytes in the count and the
* space taken by the <len> field. So for instance removing the "foo" key
* from the zipmap above will lead to the following representation:
*
* "\xfd\x10........\x05\x00hello\x05\x00world\xff"
*
* Note that because empty space, keys, values, are all prefixed length
* "objects", the lookup will take O(N) where N is the numeber of elements
* in the zipmap and *not* the number of bytes needed to represent the zipmap.
* This lowers the constant times considerably.
*/
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "zmalloc.h"
#define ZIPMAP_BIGLEN 253
#define ZIPMAP_EMPTY 254
#define ZIPMAP_END 255
#define ZIPMAP_STATUS_FRAGMENTED 1
/* The following defines the max value for the <free> field described in the
* comments above, that is, the max number of trailing bytes in a value. */
#define ZIPMAP_VALUE_MAX_FREE 5
/* Create a new empty zipmap. */
unsigned
char
*
zipmapNew
(
void
)
{
unsigned
char
*
zm
=
zmalloc
(
2
);
zm
[
0
]
=
0
;
/* Status */
zm
[
1
]
=
ZIPMAP_END
;
return
zm
;
}
/* Decode the encoded length pointed by 'p' */
static
unsigned
int
zipmapDecodeLength
(
unsigned
char
*
p
)
{
unsigned
int
len
=
*
p
;
if
(
len
<
ZIPMAP_BIGLEN
)
return
len
;
memcpy
(
&
len
,
p
,
sizeof
(
unsigned
int
));
return
len
;
}
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */
static
unsigned
int
zipmapEncodeLength
(
unsigned
char
*
p
,
unsigned
int
len
)
{
if
(
p
==
NULL
)
{
return
(
len
<
ZIPMAP_BIGLEN
)
?
1
:
1
+
sizeof
(
unsigned
int
);
}
else
{
if
(
len
<
ZIPMAP_BIGLEN
)
{
p
[
0
]
=
len
;
return
1
;
}
else
{
p
[
0
]
=
ZIPMAP_BIGLEN
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
return
1
+
sizeof
(
len
);
}
}
}
/* Search for a matching key, returning a pointer to the entry inside the
* zipmap. Returns NULL if the key is not found.
*
* If NULL is returned, and totlen is not NULL, it is set to the entire
* size of the zimap, so that the calling function will be able to
* reallocate the original zipmap to make room for more entries.
*
* If NULL is returned, and freeoff and freelen are not NULL, they are set
* to the offset of the first empty space that can hold '*freelen' bytes
* (freelen is an integer pointer used both to signal the required length
* and to get the reply from the function). If there is not a suitable
* free space block to hold the requested bytes, *freelen is set to 0. */
static
unsigned
char
*
zipmapLookupRaw
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
int
*
totlen
,
unsigned
int
*
freeoff
,
unsigned
int
*
freelen
)
{
unsigned
char
*
p
=
zm
+
1
;
unsigned
int
l
;
unsigned
int
reqfreelen
;
if
(
freelen
)
{
reqfreelen
=
*
freelen
;
*
freelen
=
0
;
assert
(
reqfreelen
!=
0
);
}
while
(
*
p
!=
ZIPMAP_END
)
{
if
(
*
p
==
ZIPMAP_EMPTY
)
{
l
=
zipmapDecodeLength
(
p
+
1
);
/* if the user want a free space report, and this space is
* enough, and we did't already found a suitable space... */
if
(
freelen
&&
l
>=
reqfreelen
&&
*
freelen
==
0
)
{
*
freelen
=
l
;
*
freeoff
=
p
-
zm
;
}
p
+=
l
;
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
}
else
{
unsigned
char
free
;
/* Match or skip the key */
l
=
zipmapDecodeLength
(
p
);
if
(
l
==
klen
&&
!
memcmp
(
p
+
1
,
key
,
l
))
return
p
;
p
+=
zipmapEncodeLength
(
NULL
,
l
)
+
l
;
/* Skip the value as well */
l
=
zipmapDecodeLength
(
p
);
p
+=
zipmapEncodeLength
(
NULL
,
l
);
free
=
p
[
0
];
p
+=
l
+
1
+
free
;
/* +1 to skip the free byte */
}
}
if
(
totlen
!=
NULL
)
*
totlen
=
(
unsigned
int
)(
p
-
zm
)
+
1
;
return
NULL
;
}
static
unsigned
long
zipmapRequiredLength
(
unsigned
int
klen
,
unsigned
int
vlen
)
{
unsigned
int
l
;
l
=
klen
+
vlen
+
3
;
if
(
klen
>=
ZIPMAP_BIGLEN
)
l
+=
4
;
if
(
vlen
>=
ZIPMAP_BIGLEN
)
l
+=
4
;
return
l
;
}
/* Return the total amonut used by a key (encoded length + payload) */
static
unsigned
int
zipmapRawKeyLength
(
unsigned
char
*
p
)
{
unsigned
int
l
=
zipmapDecodeLength
(
p
);
return
zipmapEncodeLength
(
NULL
,
l
)
+
l
;
}
/* Return the total amonut used by a value
* (encoded length + single byte free count + payload) */
static
unsigned
int
zipmapRawValueLength
(
unsigned
char
*
p
)
{
unsigned
int
l
=
zipmapDecodeLength
(
p
);
unsigned
int
used
;
used
=
zipmapEncodeLength
(
NULL
,
l
);
used
+=
p
[
used
]
+
1
+
l
;
return
used
;
}
/* Set key to value, creating the key if it does not already exist. */
unsigned
char
*
zipmapSet
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
char
*
val
,
unsigned
int
vlen
)
{
unsigned
int
oldlen
=
0
,
freeoff
=
0
,
freelen
;
unsigned
int
reqlen
=
zipmapRequiredLength
(
klen
,
vlen
);
unsigned
int
empty
,
vempty
;
unsigned
char
*
p
;
freelen
=
reqlen
;
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
&
oldlen
,
&
freeoff
,
&
freelen
);
if
(
p
==
NULL
&&
freelen
==
0
)
{
printf
(
"HERE oldlen:%u required:%u
\n
"
,
oldlen
,
reqlen
);
/* Key not found, and not space for the new key. Enlarge */
zm
=
zrealloc
(
zm
,
oldlen
+
reqlen
);
p
=
zm
+
oldlen
-
1
;
zm
[
oldlen
+
reqlen
-
1
]
=
ZIPMAP_END
;
freelen
=
reqlen
;
printf
(
"New total length is: %u
\n
"
,
oldlen
+
reqlen
);
}
else
if
(
p
==
NULL
)
{
/* Key not found, but there is enough free space. */
p
=
zm
+
freeoff
;
/* note: freelen is already set in this case */
}
else
{
unsigned
char
*
b
=
p
;
/* Key found. Is there enough space for the new value? */
/* Compute the total length: */
freelen
=
zipmapRawKeyLength
(
b
);
b
+=
freelen
;
freelen
+=
zipmapRawValueLength
(
b
);
if
(
freelen
<
reqlen
)
{
/* Mark this blog as free and recurse */
p
[
0
]
=
ZIPMAP_EMPTY
;
zipmapEncodeLength
(
p
+
1
,
freelen
);
return
zipmapSet
(
zm
,
key
,
klen
,
val
,
vlen
);
}
}
/* Ok we have a suitable block where to write the new key/value
* entry. */
empty
=
freelen
-
reqlen
;
/* If there is too much free space mark it as a free block instead
* of adding it as trailing empty space for the value, as we want
* zipmaps to be very space efficient. */
if
(
empty
>
ZIPMAP_VALUE_MAX_FREE
)
{
unsigned
char
*
e
;
e
=
p
+
reqlen
;
e
[
0
]
=
ZIPMAP_EMPTY
;
zipmapEncodeLength
(
e
+
1
,
empty
);
vempty
=
0
;
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
}
else
{
vempty
=
empty
;
}
/* Just write the key + value and we are done. */
/* Key: */
p
+=
zipmapEncodeLength
(
p
,
klen
);
memcpy
(
p
,
key
,
klen
);
p
+=
klen
;
/* Value: */
p
+=
zipmapEncodeLength
(
p
,
vlen
);
*
p
++
=
vempty
;
memcpy
(
p
,
val
,
vlen
);
return
zm
;
}
void
zipmapRepr
(
unsigned
char
*
p
)
{
unsigned
int
l
;
printf
(
"{status %u}"
,
*
p
++
);
while
(
1
)
{
if
(
p
[
0
]
==
ZIPMAP_END
)
{
printf
(
"{end}"
);
break
;
}
else
if
(
p
[
0
]
==
ZIPMAP_EMPTY
)
{
l
=
zipmapDecodeLength
(
p
+
1
);
printf
(
"{%u empty block}"
,
l
);
p
+=
l
;
}
else
{
unsigned
char
e
;
l
=
zipmapDecodeLength
(
p
);
printf
(
"{key %u}"
,
l
);
p
+=
zipmapEncodeLength
(
NULL
,
l
);
fwrite
(
p
,
l
,
1
,
stdout
);
p
+=
l
;
l
=
zipmapDecodeLength
(
p
);
printf
(
"{value %u}"
,
l
);
p
+=
zipmapEncodeLength
(
NULL
,
l
);
e
=
*
p
++
;
fwrite
(
p
,
l
,
1
,
stdout
);
p
+=
l
;
if
(
e
)
{
printf
(
"["
);
while
(
e
--
)
printf
(
"."
);
printf
(
"]"
);
}
}
}
printf
(
"
\n
"
);
}
int
main
(
void
)
{
unsigned
char
*
zm
;
zm
=
zipmapNew
();
zm
=
zipmapSet
(
zm
,(
unsigned
char
*
)
"hello"
,
5
,
(
unsigned
char
*
)
"world!"
,
6
);
zm
=
zipmapSet
(
zm
,(
unsigned
char
*
)
"foo"
,
3
,
(
unsigned
char
*
)
"bar"
,
3
);
zipmapRepr
(
zm
);
return
0
;
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment