Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
29b14d5f
"vscode:/vscode.git/clone" did not exist on "db6c763d8bebe62ab0bee78bc274826ab8fef177"
Commit
29b14d5f
authored
May 22, 2010
by
Pieter Noordhuis
Browse files
initial work for integer encoding in ziplists
parent
f6eb1747
Changes
2
Show whitespace changes
Inline
Side-by-side
zip.c
View file @
29b14d5f
#define ZIP_BIGLEN 254
#define ZIP_BIGLEN 254
#define ZIP_END 255
#define ZIP_END 255
/* The following macro returns the number of bytes needed to encode the length
/* Entry encoding */
* for the integer value _l, that is, 1 byte for lengths < ZIP_BIGLEN and
#define ZIP_ENC_RAW 0
* 5 bytes for all the other lengths. */
#define ZIP_ENC_SHORT 1
#define ZIP_LEN_BYTES(_l) (((_l) < ZIP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
#define ZIP_ENC_INT 2
#define ZIP_ENC_LLONG 3
#define ZIP_ENCODING(p) ((p)[0] >> 6)
/* Decode the encoded length pointed by 'p' */
/* Length encoding for raw entries */
static
unsigned
int
zipDecodeLength
(
unsigned
char
*
p
)
{
#define ZIP_LEN_INLINE 0
unsigned
int
len
=
*
p
;
#define ZIP_LEN_UINT16 1
#define ZIP_LEN_UINT32 2
if
(
len
<
ZIP_BIGLEN
)
return
len
;
static
unsigned
int
zipEncodingSize
(
char
encoding
)
{
memcpy
(
&
len
,
p
+
1
,
sizeof
(
unsigned
int
));
if
(
encoding
==
ZIP_ENC_SHORT
)
{
return
sizeof
(
short
int
);
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
return
sizeof
(
int
);
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
return
sizeof
(
long
long
);
}
assert
(
NULL
);
}
/* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
* provided, it is set to the number of bytes required to encode the length. */
static
unsigned
int
zipDecodeLength
(
unsigned
char
*
p
,
unsigned
int
*
lensize
)
{
unsigned
char
encoding
=
ZIP_ENCODING
(
p
),
lenenc
;
unsigned
int
len
;
if
(
encoding
==
ZIP_ENC_RAW
)
{
lenenc
=
(
p
[
0
]
>>
4
)
&
0x3
;
if
(
lenenc
==
ZIP_LEN_INLINE
)
{
len
=
p
[
0
]
&
0xf
;
if
(
lensize
)
*
lensize
=
1
;
}
else
if
(
lenenc
==
ZIP_LEN_UINT16
)
{
len
=
p
[
1
]
|
(
p
[
2
]
<<
8
);
if
(
lensize
)
*
lensize
=
3
;
}
else
{
len
=
p
[
1
]
|
(
p
[
2
]
<<
8
)
|
(
p
[
3
]
<<
16
)
|
(
p
[
4
]
<<
24
);
if
(
lensize
)
*
lensize
=
5
;
}
}
else
{
len
=
zipEncodingSize
(
encoding
);
if
(
lensize
)
*
lensize
=
1
;
}
return
len
;
return
len
;
}
}
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. */
* the amount of bytes required to encode such a length. */
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
unsigned
int
len
)
{
static
unsigned
int
zipEncodeLength
(
unsigned
char
*
p
,
char
encoding
,
unsigned
int
rawlen
)
{
if
(
p
==
NULL
)
{
unsigned
char
len
=
1
,
lenenc
,
buf
[
5
];
return
ZIP_LEN_BYTES
(
len
);
if
(
encoding
==
ZIP_ENC_RAW
)
{
if
(
rawlen
<=
0xf
)
{
if
(
!
p
)
return
len
;
lenenc
=
ZIP_LEN_INLINE
;
buf
[
0
]
=
rawlen
;
}
else
if
(
rawlen
<=
0xffff
)
{
len
+=
2
;
if
(
!
p
)
return
len
;
lenenc
=
ZIP_LEN_UINT16
;
buf
[
1
]
=
(
rawlen
)
&
0xff
;
buf
[
2
]
=
(
rawlen
>>
8
)
&
0xff
;
}
else
{
}
else
{
if
(
len
<
ZIP_BIGLEN
)
{
len
+=
4
;
p
[
0
]
=
len
;
if
(
!
p
)
return
len
;
lenenc
=
ZIP_LEN_UINT32
;
buf
[
1
]
=
(
rawlen
)
&
0xff
;
buf
[
2
]
=
(
rawlen
>>
8
)
&
0xff
;
buf
[
3
]
=
(
rawlen
>>
16
)
&
0xff
;
buf
[
4
]
=
(
rawlen
>>
24
)
&
0xff
;
}
buf
[
0
]
=
(
lenenc
<<
4
)
|
(
buf
[
0
]
&
0xf
);
}
if
(
!
p
)
return
len
;
/* Apparently we need to store the length in 'p' */
buf
[
0
]
=
(
encoding
<<
6
)
|
(
buf
[
0
]
&
0x3f
);
memcpy
(
p
,
buf
,
len
);
return
len
;
}
/* Check if string pointed to by 'entry' can be encoded as an integer.
* Stores the integer value in 'v' and its encoding in 'encoding'.
* Warning: this function requires a NULL-terminated string! */
static
int
zipTryEncoding
(
unsigned
char
*
entry
,
long
long
*
v
,
char
*
encoding
)
{
long
long
value
;
char
*
eptr
;
if
(
entry
[
0
]
==
'-'
||
(
entry
[
0
]
>=
'0'
&&
entry
[
0
]
<=
'9'
))
{
value
=
strtoll
(
entry
,
&
eptr
,
10
);
if
(
eptr
[
0
]
!=
'\0'
)
return
0
;
if
(
value
>=
SHRT_MIN
&&
value
<=
SHRT_MAX
)
{
*
encoding
=
ZIP_ENC_SHORT
;
}
else
if
(
value
>=
INT_MIN
&&
value
<=
INT_MAX
)
{
*
encoding
=
ZIP_ENC_INT
;
}
else
{
*
encoding
=
ZIP_ENC_LLONG
;
}
*
v
=
value
;
return
1
;
return
1
;
}
return
0
;
}
static
void
zipSaveInteger
(
unsigned
char
*
p
,
long
long
value
,
char
encoding
)
{
short
int
s
;
int
i
;
long
long
l
;
if
(
encoding
==
ZIP_ENC_SHORT
)
{
s
=
value
;
memcpy
(
p
,
&
s
,
sizeof
(
s
));
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
i
=
value
;
memcpy
(
p
,
&
i
,
sizeof
(
i
));
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
l
=
value
;
memcpy
(
p
,
&
l
,
sizeof
(
l
));
}
else
{
}
else
{
p
[
0
]
=
ZIP_BIGLEN
;
assert
(
NULL
);
memcpy
(
p
+
1
,
&
len
,
sizeof
(
len
));
return
1
+
sizeof
(
len
);
}
}
}
static
long
long
zipLoadInteger
(
unsigned
char
*
p
,
char
encoding
)
{
short
int
s
;
int
i
;
long
long
l
,
ret
;
if
(
encoding
==
ZIP_ENC_SHORT
)
{
memcpy
(
&
s
,
p
,
sizeof
(
s
));
ret
=
s
;
}
else
if
(
encoding
==
ZIP_ENC_INT
)
{
memcpy
(
&
i
,
p
,
sizeof
(
i
));
ret
=
i
;
}
else
if
(
encoding
==
ZIP_ENC_LLONG
)
{
memcpy
(
&
l
,
p
,
sizeof
(
l
));
ret
=
l
;
}
else
{
assert
(
NULL
);
}
}
return
ret
;
}
}
/* Return the total amount used by an entry (encoded length + payload). */
/* Return the total amount used by an entry (encoded length + payload). */
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
static
unsigned
int
zipRawEntryLength
(
unsigned
char
*
p
)
{
unsigned
int
l
=
zipDecodeLength
(
p
);
unsigned
int
lensize
,
len
;
return
zipEncodeLength
(
NULL
,
l
)
+
l
;
len
=
zipDecodeLength
(
p
,
&
lensize
);
return
lensize
+
len
;
}
}
/* Resize the zip* structure. */
/* Resize the zip* structure. */
...
...
ziplist.c
View file @
29b14d5f
...
@@ -15,8 +15,10 @@
...
@@ -15,8 +15,10 @@
*/
*/
#include <stdio.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string.h>
#include <assert.h>
#include <assert.h>
#include <limits.h>
#include "zmalloc.h"
#include "zmalloc.h"
#include "sds.h"
#include "sds.h"
#include "ziplist.h"
#include "ziplist.h"
...
@@ -60,13 +62,21 @@ static unsigned char *ziplistTail(unsigned char *zl) {
...
@@ -60,13 +62,21 @@ static unsigned char *ziplistTail(unsigned char *zl) {
}
}
unsigned
char
*
ziplistPush
(
unsigned
char
*
zl
,
unsigned
char
*
entry
,
unsigned
int
elen
,
int
where
)
{
unsigned
char
*
ziplistPush
(
unsigned
char
*
zl
,
unsigned
char
*
entry
,
unsigned
int
elen
,
int
where
)
{
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
);
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
reqlen
;
unsigned
int
reqlen
=
zipEncodeLength
(
NULL
,
elen
)
+
elen
;
unsigned
char
*
p
;
unsigned
char
*
p
;
char
encoding
=
ZIP_ENC_RAW
;
long
long
value
;
/* Resize the ziplist */
/* See if the entry can be encoded */
zl
=
ziplistResize
(
zl
,
curlen
+
reqlen
);
if
(
zipTryEncoding
(
entry
,
&
value
,
&
encoding
))
{
reqlen
=
zipEncodingSize
(
encoding
);
}
else
{
reqlen
=
elen
;
}
reqlen
+=
zipEncodeLength
(
NULL
,
encoding
,
elen
);
/* Resize the ziplist and move if needed */
zl
=
ziplistResize
(
zl
,
curlen
+
reqlen
);
if
(
where
==
ZIPLIST_HEAD
)
{
if
(
where
==
ZIPLIST_HEAD
)
{
p
=
zl
+
ZIPLIST_HEADER_SIZE
;
p
=
zl
+
ZIPLIST_HEADER_SIZE
;
if
(
*
p
!=
ZIP_END
)
{
if
(
*
p
!=
ZIP_END
)
{
...
@@ -78,31 +88,44 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
...
@@ -78,31 +88,44 @@ unsigned char *ziplistPush(unsigned char *zl, unsigned char *entry, unsigned int
}
}
/* Write the entry */
/* Write the entry */
p
+=
zipEncodeLength
(
p
,
elen
);
p
+=
zipEncodeLength
(
p
,
encoding
,
elen
);
if
(
encoding
!=
ZIP_ENC_RAW
)
{
zipSaveInteger
(
p
,
value
,
encoding
);
}
else
{
memcpy
(
p
,
entry
,
elen
);
memcpy
(
p
,
entry
,
elen
);
}
ZIPLIST_INCR_LENGTH
(
zl
,
1
);
ZIPLIST_INCR_LENGTH
(
zl
,
1
);
return
zl
;
return
zl
;
}
}
unsigned
char
*
ziplistPop
(
unsigned
char
*
zl
,
sds
*
value
,
int
where
)
{
unsigned
char
*
ziplistPop
(
unsigned
char
*
zl
,
sds
*
target
,
int
where
)
{
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
len
,
rlen
;
unsigned
int
curlen
=
ZIPLIST_BYTES
(
zl
),
rawlen
;
unsigned
int
len
,
lensize
;
unsigned
char
*
p
;
unsigned
char
*
p
;
if
(
value
)
*
value
=
NULL
;
long
long
value
;
if
(
target
)
*
target
=
NULL
;
/* Get pointer to element to remove */
/* Get pointer to element to remove */
p
=
(
where
==
ZIPLIST_HEAD
)
?
ziplistHead
(
zl
)
:
ziplistTail
(
zl
);
p
=
(
where
==
ZIPLIST_HEAD
)
?
ziplistHead
(
zl
)
:
ziplistTail
(
zl
);
if
(
*
p
==
ZIP_END
)
return
zl
;
if
(
*
p
==
ZIP_END
)
return
zl
;
len
=
zipDecodeLength
(
p
);
len
=
zipDecodeLength
(
p
,
&
lensize
);
if
(
value
)
*
value
=
sdsnewlen
(
p
+
zipEncodeLength
(
NULL
,
len
),
len
);
if
(
target
)
{
if
(
ZIP_ENCODING
(
p
)
==
ZIP_ENC_RAW
)
{
*
target
=
sdsnewlen
(
p
+
lensize
,
len
);
}
else
{
value
=
zipLoadInteger
(
p
+
lensize
,
ZIP_ENCODING
(
p
));
*
target
=
sdscatprintf
(
sdsempty
(),
"%lld"
,
value
);
}
}
/* Move list to front when popping from the head */
/* Move list to front when popping from the head */
rlen
=
zipRawEntryLength
(
p
)
;
r
aw
len
=
lensize
+
len
;
if
(
where
==
ZIPLIST_HEAD
)
{
if
(
where
==
ZIPLIST_HEAD
)
{
memmove
(
p
,
p
+
rlen
,
curlen
-
ZIPLIST_HEADER_SIZE
-
len
);
memmove
(
p
,
p
+
r
aw
len
,
curlen
-
ZIPLIST_HEADER_SIZE
-
len
);
}
}
/* Resize and update length */
/* Resize and update length */
zl
=
ziplistResize
(
zl
,
curlen
-
rlen
);
zl
=
ziplistResize
(
zl
,
curlen
-
r
aw
len
);
ZIPLIST_INCR_LENGTH
(
zl
,
-
1
);
ZIPLIST_INCR_LENGTH
(
zl
,
-
1
);
return
zl
;
return
zl
;
}
}
...
@@ -121,10 +144,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
...
@@ -121,10 +144,11 @@ unsigned char *ziplistIndex(unsigned char *zl, unsigned int index) {
/* Store entry at current position in sds *value and return pointer
/* Store entry at current position in sds *value and return pointer
* to the next entry. */
* to the next entry. */
unsigned
char
*
ziplistNext
(
unsigned
char
*
p
,
unsigned
char
**
q
,
unsigned
char
**
entry
,
unsigned
int
*
elen
)
{
unsigned
char
*
ziplistNext
(
unsigned
char
*
p
,
unsigned
char
**
q
,
unsigned
char
**
entry
,
unsigned
int
*
elen
)
{
unsigned
int
lensize
;
if
(
*
p
==
ZIP_END
)
return
NULL
;
if
(
*
p
==
ZIP_END
)
return
NULL
;
if
(
entry
)
{
if
(
entry
)
{
*
elen
=
zipDecodeLength
(
p
);
*
elen
=
zipDecodeLength
(
p
,
&
lensize
);
*
entry
=
p
+
ZIP_LEN_BYTES
(
*
elen
)
;
*
entry
=
p
+
lensize
;
}
}
if
(
q
!=
NULL
)
*
q
=
p
;
if
(
q
!=
NULL
)
*
q
=
p
;
p
+=
zipRawEntryLength
(
p
);
p
+=
zipRawEntryLength
(
p
);
...
@@ -174,16 +198,22 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
...
@@ -174,16 +198,22 @@ unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p) {
}
}
void
ziplistRepr
(
unsigned
char
*
zl
)
{
void
ziplistRepr
(
unsigned
char
*
zl
)
{
unsigned
char
*
p
;
unsigned
char
*
p
,
encoding
;
unsigned
int
l
;
unsigned
int
l
,
lsize
;
long
long
value
;
printf
(
"{bytes %d} {length %u}
\n
"
,
ZIPLIST_BYTES
(
zl
),
ZIPLIST_LENGTH
(
zl
));
printf
(
"{
total
bytes %d} {length %u}
\n
"
,
ZIPLIST_BYTES
(
zl
),
ZIPLIST_LENGTH
(
zl
));
p
=
ziplistHead
(
zl
);
p
=
ziplistHead
(
zl
);
while
(
*
p
!=
ZIP_END
)
{
while
(
*
p
!=
ZIP_END
)
{
l
=
zipDecodeLength
(
p
);
l
=
zipDecodeLength
(
p
,
&
lsize
);
printf
(
"{key %u}"
,
l
);
printf
(
"{header %u, payload %u} "
,
lsize
,
l
);
p
+=
zipEncodeLength
(
NULL
,
l
);
encoding
=
ZIP_ENCODING
(
p
);
p
+=
lsize
;
if
(
encoding
==
ZIP_ENC_RAW
)
{
fwrite
(
p
,
l
,
1
,
stdout
);
fwrite
(
p
,
l
,
1
,
stdout
);
}
else
{
printf
(
"%lld"
,
zipLoadInteger
(
p
,
encoding
));
}
printf
(
"
\n
"
);
printf
(
"
\n
"
);
p
+=
l
;
p
+=
l
;
}
}
...
@@ -200,11 +230,33 @@ unsigned char *createList() {
...
@@ -200,11 +230,33 @@ unsigned char *createList() {
return
zl
;
return
zl
;
}
}
unsigned
char
*
createIntList
()
{
unsigned
char
*
zl
=
ziplistNew
();
char
buf
[
32
];
sprintf
(
buf
,
"100"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"128000"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"-100"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_HEAD
);
sprintf
(
buf
,
"4294967296"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_HEAD
);
sprintf
(
buf
,
"non integer"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
sprintf
(
buf
,
"much much longer non integer"
);
zl
=
ziplistPush
(
zl
,
buf
,
strlen
(
buf
),
ZIPLIST_TAIL
);
return
zl
;
}
int
main
(
int
argc
,
char
**
argv
)
{
int
main
(
int
argc
,
char
**
argv
)
{
unsigned
char
*
zl
,
*
p
,
*
q
,
*
entry
;
unsigned
char
*
zl
,
*
p
,
*
q
,
*
entry
;
unsigned
int
elen
;
unsigned
int
elen
;
sds
s
;
sds
s
;
zl
=
createIntList
();
ziplistRepr
(
zl
);
zl
=
createList
();
zl
=
createList
();
ziplistRepr
(
zl
);
ziplistRepr
(
zl
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment