Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
29b14d5f
Commit
29b14d5f
authored
May 22, 2010
by
Pieter Noordhuis
Browse files
initial work for integer encoding in ziplists
parent
f6eb1747
Changes
2
Hide whitespace changes
Inline
Side-by-side
zip.c
View file @
29b14d5f
#define ZIP_BIGLEN 254
#define ZIP_BIGLEN 254
#define ZIP_END 255
#define ZIP_END 255
/* The following macro returns the number of bytes needed to encode the length
/* Entry encoding */
* for the integer value _l, that is, 1 byte for lengths < ZIP_BIGLEN and
#define ZIP_ENC_RAW 0
* 5 bytes for all the other lengths. */
#define ZIP_ENC_SHORT 1
#define ZIP_LEN_BYTES(_l) (((_l) < ZIP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
#define ZIP_ENC_INT 2
#define ZIP_ENC_LLONG 3
#define ZIP_ENCODING(p) ((p)[0] >> 6)
/* Decode the encoded length pointed by 'p' */
/* Length encoding for raw entries */
static
unsigned
int
zipDecodeLength
(
unsigned
char
*
p
)
{
#define ZIP_LEN_INLINE 0
unsigned
int
len
=
*
p
;
#define ZIP_LEN_UINT16 1
#define ZIP_LEN_UINT32 2
if
(
len
<
ZIP_BIGLEN
)
return
len
;
static
unsigned
int
zipEncodingSize
(
char
encoding
)
{
memcpy
(
&
len
,
p
+
1
,
sizeof
(
unsigned
int
));
if
(
encoding
==
ZIP_ENC_SHORT
)
{
return
sizeof
(
short
int
);
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
return
sizeof
(
int
);
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
return
sizeof
(
long
long
);
}
assert
(
NULL
);
}
/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
* provided, it is set to the number of bytes required to encode the length. */
static
unsigned
int
zipDecodeLength
(
unsigned
char
*
p
,
unsigned
int
*
lensize
)
{
unsigned
char
encoding
=
ZIP_ENCODING
(
p
),
lenenc
;
unsigned
int
len
;
if
(
encoding
==
ZIP_ENC_RAW
)
{
lenenc
=
(
p
[
0
]
>>
4
)
&
0x3
;
if
(
lenenc
==
ZIP_LEN_INLINE
)
{
len
=
p
[
0
]
&
0xf
;
if
(
lensize
)
*
lensize
=
1
;
}
else
if
(
lenenc
==
ZIP_LEN_UINT16
)
{
len
=
p
[
1
]
|
(
p
[
2
]
<<
8
);
if
(
lensize
)
*
lensize
=
3
;
}
else
{
len
=
p
[
1
]
|
(
p
[
2
]
<<
8
)
|
(
p
[
3
]
<<
16
)
|
(
p
[
4
]
<<
24
);
if
(
lensize
)
*
lensize
=
5
;
}
}
else
{
len
=
zipEncodingSize
(
encoding
);
if
(
lensize
)
*
lensize
=
1
;
}
return
len
;
return
len
;
}
}
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */
* the amount of bytes required to encode such a length. */
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
unsigned
int
len
)
{
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
char
encoding
,
unsigned
int
rawlen
)
{
if
(
p
==
NULL
)
{
unsigned
char
len
=
1
,
lenenc
,
buf
[
5
];
return
ZIP_LEN_BYTES
(
len
);
if
(
encoding
==
ZIP_ENC_RAW
)
{
}
else
{
if
(
rawlen
<=
0xf
)
{
if
(
len
<
ZIP_BIGLEN
)
{
if
(
!
p
)
return
len
;
p
[
0
]
=
len
;
lenenc
=
ZIP_LEN_INLINE
;
return
1
;
buf
[
0
]
=
rawlen
;
}
else
if
(
rawlen
<=
0xffff
)
{
len
+=
2
;
if
(
!
p
)
return
len
;
lenenc
=
ZIP_LEN_UINT16
;
buf
[
1
]
=
(
rawlen
)
&
0xff
;
buf
[
2
]
=
(
rawlen
>>
8
)
&
0xff
;
}
else
{
}
else
{
p
[
0
]
=
ZIP_BIGLEN
;
len
+=
4
;
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
if
(
!
p
)
return
len
;
return
1
+
sizeof
(
len
);
lenenc
=
ZIP_LEN_UINT32
;
buf
[
1
]
=
(
rawlen
)
&
0xff
;
buf
[
2
]
=
(
rawlen
>>
8
)
&
0xff
;
buf
[
3
]
=
(
rawlen
>>
16
)
&
0xff
;
buf
[
4
]
=
(
rawlen
>>
24
)
&
0xff
;
}
}
buf
[
0
]
=
(
lenenc
<<
4
)
|
(
buf
[
0
]
&
0xf
);
}
if
(
!
p
)
return
len
;
/* Apparently we need to store the length in 'p' */
buf
[
0
]
=
(
encoding
<<
6
)
|
(
buf
[
0
]
&
0x3f
);
memcpy
(
p
,
buf
,
len
);
return
len
;
}
/* Check if string pointed to by 'entry' can be encoded as an integer.
* Stores the integer value in 'v' and its encoding in 'encoding'.
* Warning: this function requires a NULL-terminated string! */
static
int
zipTryEncoding
(
unsigned
char
*
entry
,
long
long
*
v
,
char
*
encoding
)
{
long
long
value
;
char
*
eptr
;
if
(
entry
[
0
]
==
'-'
||
(
entry
[
0
]
>=
'0'
&&
entry
[
0
]
<=
'9'
))
{
value
=
strtoll
(
entry
,
&
eptr
,
10
);
if
(
eptr
[
0
]
!=
'\0'
)
return
0
;
if
(
value
>=
SHRT_MIN
&&
value
<=
SHRT_MAX
)
{
*
encoding
=
ZIP_ENC_SHORT
;
}
else
if
(
value
>=
INT_MIN
&&
value
<=
INT_MAX
)
{
*
encoding
=
ZIP_ENC_INT
;
}
else
{
*
encoding
=
ZIP_ENC_LLONG
;
}
*
v
=
value
;
return
1
;
}
return
0
;
}
static
void
zipSaveInteger
(
unsigned
char
*
p
,
long
long
value
,
char
encoding
)
{
short
int
s
;
int
i
;
long
long
l
;
if
(
encoding
==
ZIP_ENC_SHORT
)
{
s
=
value
;
memcpy
(
p
,
&
s
,
sizeof
(
s
));
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
i
=
value
;
memcpy
(
p
,
&
i
,
sizeof
(
i
));
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
l
=
value
;
memcpy
(
p
,
&
l
,
sizeof
(
l
));
}
else
{
assert
(
NULL
);
}
}
static
long
long
zipLoadInteger
(
unsigned
char
*
p
,
char
encoding
)
{
short
int
s
;
int
i
;
long
long
l
,
ret
;
if
(
encoding
==
ZIP_ENC_SHORT
)
{
memcpy
(
&
s
,
p
,
sizeof
(
s
));
ret
=
s
;
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
memcpy
(
&
i
,
p
,
sizeof
(
i
));
ret
=
i
;
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
memcpy
(
&
l
,
p
,
sizeof
(
l
));
ret
=
l
;
}
else
{
assert
(
NULL
);
}
}
return
ret
;
}
}
/* Return the total amount used by an entry (encoded length + payload). */
/* Return the total amount used by an entry (encoded length + payload). */
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
l
=
zipDecodeLength
(
p
);
unsigned
int
lensize
,
len
;
return
zipEncodeLength
(
NULL
,
l
)
+
l
;
len
=
zipDecodeLength
(
p
,
&
lensize
);
return
lensize
+
len
;
}
}
/* Resize the zip* structure. */
/* Resize the zip* structure. */
...
...
ziplist.c
View file @
29b14d5f
...
@@ -15,8 +15,10 @@
...
@@ -15,8 +15,10 @@
*/
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string.h>
#include <assert.h>
#include <assert.h>
#include <limits.h>
#include "zmalloc.h"
#include "zmalloc.h"
#include "sds.h"
#include "sds.h"
#include "ziplist.h"
#include "ziplist.h"
...
@@ -60,13 +62,21 @@ static unsigned char *ziplistTail(unsigned char *zl) {
...
@@ -60,13 +62,21 @@ static unsigned char *ziplistTail(unsigned char *zl) {
}
}
unsigned
char
*
ziplistPush
(
unsigned
char
*
zl
,
unsigned
char
*
entry
,
unsigned
int
elen
,
int
where
)
{
unsigned
char
*
ziplistPush
(
unsigned
char
*
zl
,
unsigned
char
*
entry
,
unsigned
int
elen
,
int
where
)
{
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
);
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
reqlen
;
unsigned
int
reqlen
=
zipEncodeLength
(
NULL
,
elen
)
+
elen
;
unsigned
char
*
p
;
unsigned
char
*
p
;
char
encoding
=
ZIP_ENC_RAW
;
long
long
value
;
/* Resize the ziplist */
/* See if the entry can be encoded */
zl
=
ziplistResize
(
zl
,
curlen
+
reqlen
);
if
(
zipTryEncoding
(
entry
,
&
value
,
&
encoding
))
{
reqlen
=
zipEncodingSize
(
encoding
);
}
else
{
reqlen
=
elen
;
}
reqlen
+=
zipEncodeLength
(
NULL
,
encoding
,
elen
);
/* Resize the ziplist and move if needed */
zl
=
ziplistResize
(
zl
,
curlen
+
reqlen
);
if
(
where
==
ZIPLIST_HEAD
)
{
if
(
where
==
ZIPLIST_HEAD
)
{
p
=
zl
+
ZIPLIST_HEADER_SIZE
;
p
=
zl
+
ZIPLIST_HEADER_SIZE
;
if
(
*
p
!=
ZIP_END
)
{
if
(
*
p
!=
ZIP_END
)
{
...
@@ -78,31 +88,44 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
...
@@ -78,31 +88,44 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
}
}
/* Write the entry */
/* Write the entry */
p
+=
zipEncodeLength
(
p
,
elen
);
p
+=
zipEncodeLength
(
p
,
encoding
,
elen
);
memcpy
(
p
,
entry
,
elen
);
if
(
encoding
!=
ZIP_ENC_RAW
)
{
zipSaveInteger
(
p
,
value
,
encoding
);
}
else
{
memcpy
(
p
,
entry
,
elen
);
}
ZIPLIST_INCR_LENGTH
(
zl
,
1
);
ZIPLIST_INCR_LENGTH
(
zl
,
1
);
return
zl
;
return
zl
;
}
}
unsigned
char
*
ziplistPop
(
unsigned
char
*
zl
,
sds
*
value
,
int
where
)
{
unsigned
char
*
ziplistPop
(
unsigned
char
*
zl
,
sds
*
target
,
int
where
)
{
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
len
,
rlen
;
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
rawlen
;
unsigned
int
len
,
lensize
;
unsigned
char
*
p
;
unsigned
char
*
p
;
if
(
value
)
*
value
=
NULL
;
long
long
value
;
if
(
target
)
*
target
=
NULL
;
/* Get pointer to element to remove */
/* Get pointer to element to remove */
p
=
(
where
==
ZIPLIST_HEAD
)
?
ziplistHead
(
zl
)
:
ziplistTail
(
zl
);
p
=
(
where
==
ZIPLIST_HEAD
)
?
ziplistHead
(
zl
)
:
ziplistTail
(
zl
);
if
(
*
p
==
ZIP_END
)
return
zl
;
if
(
*
p
==
ZIP_END
)
return
zl
;
len
=
zipDecodeLength
(
p
);
len
=
zipDecodeLength
(
p
,
&
lensize
);
if
(
value
)
*
value
=
sdsnewlen
(
p
+
zipEncodeLength
(
NULL
,
len
),
len
);
if
(
target
)
{
if
(
ZIP_ENCODING
(
p
)
==
ZIP_ENC_RAW
)
{
*
target
=
sdsnewlen
(
p
+
lensize
,
len
);
}
else
{
value
=
zipLoadInteger
(
p
+
lensize
,
ZIP_ENCODING
(
p
));
*
target
=
sdscatprintf
(
sdsempty
(),
"%lld"
,
value
);
}
}
/* Move list to front when popping from the head */
/* Move list to front when popping from the head */
rlen
=
zipRawEntryLength
(
p
)
;
r
aw
len
=
lensize
+
len
;
if
(
where
==
ZIPLIST_HEAD
)
{
if
(
where
==
ZIPLIST_HEAD
)
{
memmove
(
p
,
p
+
rlen
,
curlen
-
ZIPLIST_HEADER_SIZE
-
len
);
memmove
(
p
,
p
+
r
aw
len
,
curlen
-
ZIPLIST_HEADER_SIZE
-
len
);
}
}
/* Resize and update length */
/* Resize and update length */
zl
=
ziplistResize
(
zl
,
curlen
-
rlen
);
zl
=
ziplistResize
(
zl
,
curlen
-
r
aw
len
);
ZIPLIST_INCR_LENGTH
(
zl
,
-
1
);
ZIPLIST_INCR_LENGTH
(
zl
,
-
1
);
return
zl
;
return
zl
;
}
}
...
@@ -121,10 +144,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
...
@@ -121,10 +144,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
/* Store entry at current position in sds *value and return pointer
/* Store entry at current position in sds *value and return pointer
* to the next entry. */
* to the next entry. */
unsigned
char
*
ziplistNext
(
unsigned
char
*
p
,
unsigned
char
**
q
,
unsigned
char
**
entry
,
unsigned
int
*
elen
)
{
unsigned
char
*
ziplistNext
(
unsigned
char
*
p
,
unsigned
char
**
q
,
unsigned
char
**
entry
,
unsigned
int
*
elen
)
{
unsigned
int
lensize
;
if
(
*
p
==
ZIP_END
)
return
NULL
;
if
(
*
p
==
ZIP_END
)
return
NULL
;
if
(
entry
)
{
if
(
entry
)
{
*
elen
=
zipDecodeLength
(
p
);
*
elen
=
zipDecodeLength
(
p
,
&
lensize
);
*
entry
=
p
+
ZIP_LEN_BYTES
(
*
elen
)
;
*
entry
=
p
+
lensize
;
}
}
if
(
q
!=
NULL
)
*
q
=
p
;
if
(
q
!=
NULL
)
*
q
=
p
;
p
+=
zipRawEntryLength
(
p
);
p
+=
zipRawEntryLength
(
p
);
...
@@ -174,16 +198,22 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
...
@@ -174,16 +198,22 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
}
}
void
ziplistRepr
(
unsigned
char
*
zl
)
{
void
ziplistRepr
(
unsigned
char
*
zl
)
{
unsigned
char
*
p
;
unsigned
char
*
p
,
encoding
;
unsigned
int
l
;
unsigned
int
l
,
lsize
;
long
long
value
;
printf
(
"{bytes %d} {length %u}
\n
"
,
ZIPLIST_BYTES
(
zl
),
ZIPLIST_LENGTH
(
zl
));
printf
(
"{
total
bytes %d} {length %u}
\n
"
,
ZIPLIST_BYTES
(
zl
),
ZIPLIST_LENGTH
(
zl
));
p
=
ziplistHead
(
zl
);
p
=
ziplistHead
(
zl
);
while
(
*
p
!=
ZIP_END
)
{
while
(
*
p
!=
ZIP_END
)
{
l
=
zipDecodeLength
(
p
);
l
=
zipDecodeLength
(
p
,
&
lsize
);
printf
(
"{key %u}"
,
l
);
printf
(
"{header %u, payload %u} "
,
lsize
,
l
);
p
+=
zipEncodeLength
(
NULL
,
l
);
encoding
=
ZIP_ENCODING
(
p
);
fwrite
(
p
,
l
,
1
,
stdout
);
p
+=
lsize
;
if
(
encoding
==
ZIP_ENC_RAW
)
{
fwrite
(
p
,
l
,
1
,
stdout
);
}
else
{
printf
(
"%lld"
,
zipLoadInteger
(
p
,
encoding
));
}
printf
(
"
\n
"
);
printf
(
"
\n
"
);
p
+=
l
;
p
+=
l
;
}
}
...
@@ -200,11 +230,33 @@ unsigned char *createList() {
...
@@ -200,11 +230,33 @@ unsigned char *createList() {
return
zl
;
return
zl
;
}
}
unsigned
char
*
createIntList
()
{
unsigned
char
*
zl
=
ziplistNew
();
char
buf
[
32
];
sprintf
(
buf
,
"100"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"128000"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"-100"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_HEAD
);
sprintf
(
buf
,
"4294967296"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_HEAD
);
sprintf
(
buf
,
"non integer"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"much much longer non integer"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
return
zl
;
}
int
main
(
int
argc
,
char
**
argv
)
{
int
main
(
int
argc
,
char
**
argv
)
{
unsigned
char
*
zl
,
*
p
,
*
q
,
*
entry
;
unsigned
char
*
zl
,
*
p
,
*
q
,
*
entry
;
unsigned
int
elen
;
unsigned
int
elen
;
sds
s
;
sds
s
;
zl
=
createIntList
();
ziplistRepr
(
zl
);
zl
=
createList
();
zl
=
createList
();
ziplistRepr
(
zl
);
ziplistRepr
(
zl
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment