Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
14ae26d8
Commit
14ae26d8
authored
Apr 01, 2010
by
antirez
Browse files
Merge branch 'zipmap' of
git://github.com/pietern/redis
parents
ffc6b7f8
e9484a85
Changes
2
Show whitespace changes
Inline
Side-by-side
redis.c
View file @
14ae26d8
...
...
@@ -6000,10 +6000,8 @@ static void hsetCommand(redisClient *c) {
decrRefCount(valobj);
o->ptr = zm;
/* And here there is the second check for hash conversion...
* we want to do it only if the operation was not just an update as
* zipmapLen() is O(N). */
if (!update && zipmapLen(zm) > server.hash_max_zipmap_entries)
/* And here there is the second check for hash conversion. */
if (zipmapLen(zm) > server.hash_max_zipmap_entries)
convertToRealHash(o);
} else {
tryObjectEncoding(c->argv[2]);
...
...
@@ -6021,7 +6019,6 @@ static void hsetCommand(redisClient *c) {
}
static void hincrbyCommand(redisClient *c) {
int update = 0;
long long value = 0, incr = 0;
robj *o = lookupKeyWrite(c->db,c->argv[1]);
...
...
@@ -6058,13 +6055,12 @@ static void hincrbyCommand(redisClient *c) {
value += incr;
sds svalue = sdscatprintf(sdsempty(),"%lld",value);
zm = zipmapSet(zm,c->argv[2]->ptr,sdslen(c->argv[2]->ptr),
(unsigned char*)svalue,sdslen(svalue),
&update
);
(unsigned char*)svalue,sdslen(svalue),
NULL
);
sdsfree(svalue);
o->ptr = zm;
/* Check if the zipmap needs to be converted
* if this was not an update. */
if (!update && zipmapLen(zm) > server.hash_max_zipmap_entries)
/* Check if the zipmap needs to be converted. */
if (zipmapLen(zm) > server.hash_max_zipmap_entries)
convertToRealHash(o);
} else {
robj *hval;
...
...
zipmap.c
View file @
14ae26d8
...
...
@@ -42,10 +42,11 @@
/* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
*
* <
status
><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
* <
zmlen
><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
*
* <status> is 1 byte status. Currently only 1 bit is used: if the least
* significant bit is set, it means the zipmap needs to be defragmented.
* <zmlen> is 1 byte length that holds the current size of the zipmap.
* When the zipmap length is greater than or equal to 254, this value
* is not used and the zipmap needs to be traversed to find out the length.
*
* <len> is the length of the following string (key or value).
* <len> lengths are encoded in a single value or in a 5 bytes value.
...
...
@@ -62,22 +63,15 @@
* or even in order to add a key/value pair if it fits.
*
* <free> is always an unsigned 8 bit number, because if after an
* update operation there are more than a few free bytes, the
y'll be converted
*
into empty space prefixed by the special value 254
.
* update operation there are more than a few free bytes, the
zipmap will be
*
reallocated to make sure it is as small as possible
.
*
* The most compact representation of the above two elements hash is actually:
*
* "\x0
0
\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
* "\x0
2
\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
*
* Empty space is marked using a 254 bytes + a <len> (coded as already
* specified). The length includes the 254 bytes in the count and the
* space taken by the <len> field. So for instance removing the "foo" key
* from the zipmap above will lead to the following representation:
*
* "\x00\xfd\x10........\x05hello\x05\x00world\xff"
*
* Note that because empty space, keys, values, are all prefixed length
* "objects", the lookup will take O(N) where N is the numeber of elements
* Note that because keys and values are prefixed length "objects",
* the lookup will take O(N) where N is the number of elements
* in the zipmap and *not* the number of bytes needed to represent the zipmap.
* This lowers the constant times considerably.
*/
...
...
@@ -87,15 +81,12 @@
#include <assert.h>
#include "zmalloc.h"
#define ZIPMAP_BIGLEN 253
#define ZIPMAP_EMPTY 254
#define ZIPMAP_BIGLEN 254
#define ZIPMAP_END 255
#define ZIPMAP_STATUS_FRAGMENTED 1
/* The following defines the max value for the <free> field described in the
* comments above, that is, the max number of trailing bytes in a value. */
#define ZIPMAP_VALUE_MAX_FREE
5
#define ZIPMAP_VALUE_MAX_FREE
4
/* The following macro returns the number of bytes needed to encode the length
* for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
...
...
@@ -106,7 +97,7 @@
unsigned
char
*
zipmapNew
(
void
)
{
unsigned
char
*
zm
=
zmalloc
(
2
);
zm
[
0
]
=
0
;
/*
Status
*/
zm
[
0
]
=
0
;
/*
Length
*/
zm
[
1
]
=
ZIPMAP_END
;
return
zm
;
}
...
...
@@ -142,40 +133,25 @@ static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
*
* If NULL is returned, and totlen is not NULL, it is set to the entire
* size of the zimap, so that the calling function will be able to
* reallocate the original zipmap to make room for more entries.
*
* If NULL is returned, and freeoff and freelen are not NULL, they are set
* to the offset of the first empty space that can hold '*freelen' bytes
* (freelen is an integer pointer used both to signal the required length
* and to get the reply from the function). If there is not a suitable
* free space block to hold the requested bytes, *freelen is set to 0. */
static
unsigned
char
*
zipmapLookupRaw
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
int
*
totlen
,
unsigned
int
*
freeoff
,
unsigned
int
*
freelen
)
{
unsigned
char
*
p
=
zm
+
1
;
* reallocate the original zipmap to make room for more entries. */
static
unsigned
char
*
zipmapLookupRaw
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
int
*
totlen
)
{
unsigned
char
*
p
=
zm
+
1
,
*
k
=
NULL
;
unsigned
int
l
;
unsigned
int
reqfreelen
=
0
;
/* initialized just to prevent warning */
if
(
freelen
)
{
reqfreelen
=
*
freelen
;
*
freelen
=
0
;
assert
(
reqfreelen
!=
0
);
}
while
(
*
p
!=
ZIPMAP_END
)
{
if
(
*
p
==
ZIPMAP_EMPTY
)
{
l
=
zipmapDecodeLength
(
p
+
1
);
/* if the user want a free space report, and this space is
* enough, and we did't already found a suitable space... */
if
(
freelen
&&
l
>=
reqfreelen
&&
*
freelen
==
0
)
{
*
freelen
=
l
;
*
freeoff
=
p
-
zm
;
}
p
+=
l
;
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
}
else
{
unsigned
char
free
;
/* Match or skip the key */
l
=
zipmapDecodeLength
(
p
);
if
(
l
==
klen
&&
!
memcmp
(
p
+
1
,
key
,
l
))
return
p
;
if
(
k
==
NULL
&&
l
==
klen
&&
!
memcmp
(
p
+
1
,
key
,
l
))
{
/* Only return when the user doesn't care
* for the total length of the zipmap. */
if
(
totlen
!=
NULL
)
{
k
=
p
;
}
else
{
return
p
;
}
}
p
+=
zipmapEncodeLength
(
NULL
,
l
)
+
l
;
/* Skip the value as well */
l
=
zipmapDecodeLength
(
p
);
...
...
@@ -183,9 +159,8 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
free
=
p
[
0
];
p
+=
l
+
1
+
free
;
/* +1 to skip the free byte */
}
}
if
(
totlen
!=
NULL
)
*
totlen
=
(
unsigned
int
)(
p
-
zm
)
+
1
;
return
NULL
;
return
k
;
}
static
unsigned
long
zipmapRequiredLength
(
unsigned
int
klen
,
unsigned
int
vlen
)
{
...
...
@@ -220,64 +195,70 @@ static unsigned int zipmapRawValueLength(unsigned char *p) {
* free space if any). */
static
unsigned
int
zipmapRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
l
=
zipmapRawKeyLength
(
p
);
return
l
+
zipmapRawValueLength
(
p
+
l
);
}
static
inline
unsigned
char
*
zipmapResize
(
unsigned
char
*
zm
,
unsigned
int
len
)
{
zm
=
zrealloc
(
zm
,
len
);
zm
[
len
-
1
]
=
ZIPMAP_END
;
return
zm
;
}
/* Set key to value, creating the key if it does not already exist.
* If 'update' is not NULL, *update is set to 1 if the key was
* already preset, otherwise to 0. */
unsigned
char
*
zipmapSet
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
char
*
val
,
unsigned
int
vlen
,
int
*
update
)
{
unsigned
int
oldlen
=
0
,
freeoff
=
0
,
freelen
;
unsigned
int
reqlen
=
zipmapRequiredLength
(
klen
,
vlen
);
unsigned
int
zmlen
,
offset
;
unsigned
int
freelen
,
reqlen
=
zipmapRequiredLength
(
klen
,
vlen
);
unsigned
int
empty
,
vempty
;
unsigned
char
*
p
;
freelen
=
reqlen
;
if
(
update
)
*
update
=
0
;
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
&
oldlen
,
&
freeoff
,
&
freelen
);
if
(
p
==
NULL
&&
freelen
==
0
)
{
/* Key not found, and not space for the new key. Enlarge */
zm
=
zrealloc
(
zm
,
oldlen
+
reqlen
);
p
=
zm
+
oldlen
-
1
;
zm
[
oldlen
+
reqlen
-
1
]
=
ZIPMAP_END
;
freelen
=
reqlen
;
}
else
if
(
p
==
NULL
)
{
/* Key not found, but there is enough free space. */
p
=
zm
+
freeoff
;
/* note: freelen is already set in this case */
}
else
{
unsigned
char
*
b
=
p
;
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
&
zmlen
);
if
(
p
==
NULL
)
{
/* Key not found: enlarge */
zm
=
zipmapResize
(
zm
,
zmlen
+
reqlen
);
p
=
zm
+
zmlen
-
1
;
zmlen
=
zmlen
+
reqlen
;
/* Increase zipmap length (this is an insert) */
if
(
zm
[
0
]
<
ZIPMAP_BIGLEN
)
zm
[
0
]
++
;
}
else
{
/* Key found. Is there enough space for the new value? */
/* Compute the total length: */
if
(
update
)
*
update
=
1
;
freelen
=
zipmapRawKeyLength
(
b
);
b
+=
freelen
;
freelen
+=
zipmapRawValueLength
(
b
);
freelen
=
zipmapRawEntryLength
(
p
);
if
(
freelen
<
reqlen
)
{
/* Mark this entry as free and recurse */
p
[
0
]
=
ZIPMAP_EMPTY
;
zipmapEncodeLength
(
p
+
1
,
freelen
);
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
return
zipmapSet
(
zm
,
key
,
klen
,
val
,
vlen
,
NULL
);
/* Store the offset of this key within the current zipmap, so
* it can be resized. Then, move the tail backwards so this
* pair fits at the current position. */
offset
=
p
-
zm
;
zm
=
zipmapResize
(
zm
,
zmlen
-
freelen
+
reqlen
);
p
=
zm
+
offset
;
/* The +1 in the number of bytes to be moved is caused by the
* end-of-zipmap byte. Note: the *original* zmlen is used. */
memmove
(
p
+
reqlen
,
p
+
freelen
,
zmlen
-
(
offset
+
freelen
+
1
));
zmlen
=
zmlen
-
freelen
+
reqlen
;
freelen
=
reqlen
;
}
}
/* Ok we have a suitable block where to write the new key/value
* entry. */
/* We now have a suitable block where the key/value entry can
* be written. If there is too much free space, move the tail
* of the zipmap a few bytes to the front and shrink the zipmap,
* as we want zipmaps to be very space efficient. */
empty
=
freelen
-
reqlen
;
/* If there is too much free space mark it as a free block instead
* of adding it as trailing empty space for the value, as we want
* zipmaps to be very space efficient. */
if
(
empty
>
ZIPMAP_VALUE_MAX_FREE
)
{
unsigned
char
*
e
;
e
=
p
+
reqlen
;
e
[
0
]
=
ZIPMAP_EMPTY
;
zipmapEncodeLength
(
e
+
1
,
empty
);
if
(
empty
>=
ZIPMAP_VALUE_MAX_FREE
)
{
/* First, move the tail <empty> bytes to the front, then resize
* the zipmap to be <empty> bytes smaller. */
offset
=
p
-
zm
;
memmove
(
p
+
reqlen
,
p
+
freelen
,
zmlen
-
(
offset
+
freelen
+
1
));
zmlen
-=
empty
;
zm
=
zipmapResize
(
zm
,
zmlen
);
p
=
zm
+
offset
;
vempty
=
0
;
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
}
else
{
vempty
=
empty
;
}
...
...
@@ -297,13 +278,16 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
/* Remove the specified key. If 'deleted' is not NULL the pointed integer is
* set to 0 if the key was not found, to 1 if it was found and deleted. */
unsigned
char
*
zipmapDel
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
int
*
deleted
)
{
unsigned
char
*
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
NULL
,
NULL
,
NULL
);
unsigned
int
zmlen
,
freelen
;
unsigned
char
*
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
&
zmlen
);
if
(
p
)
{
unsigned
int
freelen
=
zipmapRawEntryLength
(
p
);
freelen
=
zipmapRawEntryLength
(
p
);
memmove
(
p
,
p
+
freelen
,
zmlen
-
((
p
-
zm
)
+
freelen
+
1
));
zm
=
zipmapResize
(
zm
,
zmlen
-
freelen
);
/* Decrease zipmap length */
if
(
zm
[
0
]
<
ZIPMAP_BIGLEN
)
zm
[
0
]
--
;
p
[
0
]
=
ZIPMAP_EMPTY
;
zipmapEncodeLength
(
p
+
1
,
freelen
);
zm
[
0
]
|=
ZIPMAP_STATUS_FRAGMENTED
;
if
(
deleted
)
*
deleted
=
1
;
}
else
{
if
(
deleted
)
*
deleted
=
0
;
...
...
@@ -328,8 +312,6 @@ unsigned char *zipmapRewind(unsigned char *zm) {
* }
*/
unsigned
char
*
zipmapNext
(
unsigned
char
*
zm
,
unsigned
char
**
key
,
unsigned
int
*
klen
,
unsigned
char
**
value
,
unsigned
int
*
vlen
)
{
while
(
zm
[
0
]
==
ZIPMAP_EMPTY
)
zm
+=
zipmapDecodeLength
(
zm
+
1
);
if
(
zm
[
0
]
==
ZIPMAP_END
)
return
NULL
;
if
(
key
)
{
*
key
=
zm
;
...
...
@@ -351,7 +333,7 @@ unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *
int
zipmapGet
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
,
unsigned
char
**
value
,
unsigned
int
*
vlen
)
{
unsigned
char
*
p
;
if
((
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
NULL
,
NULL
,
NULL
))
==
NULL
)
return
0
;
if
((
p
=
zipmapLookupRaw
(
zm
,
key
,
klen
,
NULL
))
==
NULL
)
return
0
;
p
+=
zipmapRawKeyLength
(
p
);
*
vlen
=
zipmapDecodeLength
(
p
);
*
value
=
p
+
ZIPMAP_LEN_BYTES
(
*
vlen
)
+
1
;
...
...
@@ -360,15 +342,21 @@ int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned
/* Return 1 if the key exists, otherwise 0 is returned. */
int
zipmapExists
(
unsigned
char
*
zm
,
unsigned
char
*
key
,
unsigned
int
klen
)
{
return
zipmapLookupRaw
(
zm
,
key
,
klen
,
NULL
,
NULL
,
NULL
)
!=
NULL
;
return
zipmapLookupRaw
(
zm
,
key
,
klen
,
NULL
)
!=
NULL
;
}
/* Return the number of entries inside a zipmap */
unsigned
int
zipmapLen
(
unsigned
char
*
zm
)
{
unsigned
char
*
p
=
zipmapRewind
(
zm
);
unsigned
int
len
=
0
;
if
(
zm
[
0
]
<
ZIPMAP_BIGLEN
)
{
len
=
zm
[
0
];
}
else
{
unsigned
char
*
p
=
zipmapRewind
(
zm
);
while
((
p
=
zipmapNext
(
p
,
NULL
,
NULL
,
NULL
,
NULL
))
!=
NULL
)
len
++
;
/* Re-store length if small enough */
if
(
len
<
ZIPMAP_BIGLEN
)
zm
[
0
]
=
len
;
}
return
len
;
}
...
...
@@ -380,10 +368,6 @@ void zipmapRepr(unsigned char *p) {
if
(
p
[
0
]
==
ZIPMAP_END
)
{
printf
(
"{end}"
);
break
;
}
else
if
(
p
[
0
]
==
ZIPMAP_EMPTY
)
{
l
=
zipmapDecodeLength
(
p
+
1
);
printf
(
"{%u empty block}"
,
l
);
p
+=
l
;
}
else
{
unsigned
char
e
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment