Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
f4e1a4de
Commit
f4e1a4de
authored
Feb 14, 2018
by
antirez
Browse files
CG: RDB loading first implementation.
parent
db7a5f23
Changes
3
Hide whitespace changes
Inline
Side-by-side
src/rdb.c
View file @
f4e1a4de
...
@@ -75,6 +75,18 @@ static int rdbWriteRaw(rio *rdb, void *p, size_t len) {
...
@@ -75,6 +75,18 @@ static int rdbWriteRaw(rio *rdb, void *p, size_t len) {
return
len
;
return
len
;
}
}
/* This is just a wrapper for the low level function rioRead() that will
* automatically abort if it is not possible to read the specified amount
* of bytes. */
void
rdbLoadRaw
(
rio
*
rdb
,
void
*
buf
,
uint64_t
len
)
{
if
(
rioRead
(
rdb
,
buf
,
len
)
==
0
)
{
rdbExitReportCorruptRDB
(
"Impossible to read %llu bytes in rdbLoadRaw()"
,
(
unsigned
long
long
)
len
);
return
;
/* Not reached. */
}
}
int
rdbSaveType
(
rio
*
rdb
,
unsigned
char
type
)
{
int
rdbSaveType
(
rio
*
rdb
,
unsigned
char
type
)
{
return
rdbWriteRaw
(
rdb
,
&
type
,
1
);
return
rdbWriteRaw
(
rdb
,
&
type
,
1
);
}
}
...
@@ -90,7 +102,7 @@ int rdbLoadType(rio *rdb) {
...
@@ -90,7 +102,7 @@ int rdbLoadType(rio *rdb) {
time_t
rdbLoadTime
(
rio
*
rdb
)
{
time_t
rdbLoadTime
(
rio
*
rdb
)
{
int32_t
t32
;
int32_t
t32
;
if
(
rioRead
(
rdb
,
&
t32
,
4
)
==
0
)
return
-
1
;
rdbLoadRaw
(
rdb
,
&
t32
,
4
);
return
(
time_t
)
t32
;
return
(
time_t
)
t32
;
}
}
...
@@ -101,7 +113,7 @@ int rdbSaveMillisecondTime(rio *rdb, long long t) {
...
@@ -101,7 +113,7 @@ int rdbSaveMillisecondTime(rio *rdb, long long t) {
long
long
rdbLoadMillisecondTime
(
rio
*
rdb
)
{
long
long
rdbLoadMillisecondTime
(
rio
*
rdb
)
{
int64_t
t64
;
int64_t
t64
;
if
(
rioRead
(
rdb
,
&
t64
,
8
)
==
0
)
return
-
1
;
rdbLoadRaw
(
rdb
,
&
t64
,
8
);
return
(
long
long
)
t64
;
return
(
long
long
)
t64
;
}
}
...
@@ -906,7 +918,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
...
@@ -906,7 +918,7 @@ ssize_t rdbSaveObject(rio *rdb, robj *o) {
nwritten
+=
n
;
nwritten
+=
n
;
/* Save the consumers of this group. */
/* Save the consumers of this group. */
if
((
n
=
rdbSaveStream
PEL
(
rdb
,
cg
->
pel
,
1
))
==
-
1
)
return
-
1
;
if
((
n
=
rdbSaveStream
Consumers
(
rdb
,
cg
))
==
-
1
)
return
-
1
;
nwritten
+=
n
;
nwritten
+=
n
;
}
}
raxStop
(
&
ri
);
raxStop
(
&
ri
);
...
@@ -1599,6 +1611,72 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
...
@@ -1599,6 +1611,72 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) {
/* Load the last entry ID. */
/* Load the last entry ID. */
s
->
last_id
.
ms
=
rdbLoadLen
(
rdb
,
NULL
);
s
->
last_id
.
ms
=
rdbLoadLen
(
rdb
,
NULL
);
s
->
last_id
.
seq
=
rdbLoadLen
(
rdb
,
NULL
);
s
->
last_id
.
seq
=
rdbLoadLen
(
rdb
,
NULL
);
/* Consumer groups loading */
size_t
cgroups_count
=
rdbLoadLen
(
rdb
,
NULL
);
while
(
cgroups_count
--
)
{
/* Get the consumer group name and ID. We can then create the
* consumer group ASAP and populate its structure as
* we read more data. */
streamID
cg_id
;
sds
cgname
=
rdbGenericLoadStringObject
(
rdb
,
RDB_LOAD_SDS
,
NULL
);
cg_id
.
ms
=
rdbLoadLen
(
rdb
,
NULL
);
cg_id
.
seq
=
rdbLoadLen
(
rdb
,
NULL
);
streamCG
*
cgroup
=
streamCreateCG
(
s
,
cgname
,
sdslen
(
cgname
),
&
cg_id
);
if
(
cgroup
==
NULL
)
rdbExitReportCorruptRDB
(
"Duplicated consumer group name %s"
,
cgname
);
sdsfree
(
cgname
);
/* Load the global PEL for this consumer group, however we'll
* not yet populate the NACK structures with the message
* owner, since consumers for this group and their messages will
* be read as a next step. So for now leave them not resolved
* and later populate it. */
size_t
pel_size
=
rdbLoadLen
(
rdb
,
NULL
);
while
(
pel_size
--
)
{
unsigned
char
rawid
[
sizeof
(
streamID
)];
rdbLoadRaw
(
rdb
,
rawid
,
sizeof
(
rawid
));
streamNACK
*
nack
=
streamCreateNACK
(
NULL
);
nack
->
delivery_time
=
rdbLoadMillisecondTime
(
rdb
);
nack
->
delivery_count
=
rdbLoadLen
(
rdb
,
NULL
);
if
(
!
raxInsert
(
cgroup
->
pel
,
rawid
,
sizeof
(
rawid
),
nack
,
NULL
))
rdbExitReportCorruptRDB
(
"Duplicated gobal PEL entry "
"loading stream consumer group"
);
}
/* Now that we loaded our global PEL, we need to load the
* consumers and their local PELs. */
size_t
consumers_num
=
rdbLoadLen
(
rdb
,
NULL
);
while
(
consumers_num
--
)
{
sds
cname
=
rdbGenericLoadStringObject
(
rdb
,
RDB_LOAD_SDS
,
NULL
);
streamConsumer
*
consumer
=
streamLookupConsumer
(
cgroup
,
cname
,
1
);
sdsfree
(
cname
);
consumer
->
seen_time
=
rdbLoadMillisecondTime
(
rdb
);
/* Load the PEL about entries owned by this specific
* consumer. */
pel_size
=
rdbLoadLen
(
rdb
,
NULL
);
while
(
pel_size
--
)
{
unsigned
char
rawid
[
sizeof
(
streamID
)];
rdbLoadRaw
(
rdb
,
rawid
,
sizeof
(
rawid
));
streamNACK
*
nack
=
raxFind
(
cgroup
->
pel
,
rawid
,
sizeof
(
rawid
));
if
(
nack
==
NULL
)
rdbExitReportCorruptRDB
(
"Consumer entry not found in "
"group global PEL"
);
/* Set the NACK consumer, that was left to NULL when
* loading the global PEL. Then set the same shared
* NACK structure also in the consumer-specific PEL. */
nack
->
consumer
=
consumer
;
if
(
raxInsert
(
consumer
->
pel
,
rawid
,
sizeof
(
rawid
),
nack
,
NULL
))
rdbExitReportCorruptRDB
(
"Duplicated consumer PEL entry "
" loading a stream consumer "
"group"
);
}
}
}
}
else
if
(
rdbtype
==
RDB_TYPE_MODULE
||
rdbtype
==
RDB_TYPE_MODULE_2
)
{
}
else
if
(
rdbtype
==
RDB_TYPE_MODULE
||
rdbtype
==
RDB_TYPE_MODULE_2
)
{
uint64_t
moduleid
=
rdbLoadLen
(
rdb
,
NULL
);
uint64_t
moduleid
=
rdbLoadLen
(
rdb
,
NULL
);
moduleType
*
mt
=
moduleTypeLookupModuleByID
(
moduleid
);
moduleType
*
mt
=
moduleTypeLookupModuleByID
(
moduleid
);
...
@@ -1728,7 +1806,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi) {
...
@@ -1728,7 +1806,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi) {
/* EXPIRETIME: load an expire associated with the next key
/* EXPIRETIME: load an expire associated with the next key
* to load. Note that after loading an expire we need to
* to load. Note that after loading an expire we need to
* load the actual type, and continue. */
* load the actual type, and continue. */
if
((
expiretime
=
rdbLoadTime
(
rdb
)
)
==
-
1
)
goto
eoferr
;
expiretime
=
rdbLoadTime
(
rdb
);
/* We read the time so we need to read the object type again. */
/* We read the time so we need to read the object type again. */
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
/* the EXPIRETIME opcode specifies time in seconds, so convert
/* the EXPIRETIME opcode specifies time in seconds, so convert
...
@@ -1737,7 +1815,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi) {
...
@@ -1737,7 +1815,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi) {
}
else
if
(
type
==
RDB_OPCODE_EXPIRETIME_MS
)
{
}
else
if
(
type
==
RDB_OPCODE_EXPIRETIME_MS
)
{
/* EXPIRETIME_MS: milliseconds precision expire times introduced
/* EXPIRETIME_MS: milliseconds precision expire times introduced
* with RDB v3. Like EXPIRETIME but no with more precision. */
* with RDB v3. Like EXPIRETIME but no with more precision. */
if
((
expiretime
=
rdbLoadMillisecondTime
(
rdb
)
)
==
-
1
)
goto
eoferr
;
expiretime
=
rdbLoadMillisecondTime
(
rdb
);
/* We read the time so we need to read the object type again. */
/* We read the time so we need to read the object type again. */
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
if
((
type
=
rdbLoadType
(
rdb
))
==
-
1
)
goto
eoferr
;
}
else
if
(
type
==
RDB_OPCODE_EOF
)
{
}
else
if
(
type
==
RDB_OPCODE_EOF
)
{
...
...
src/stream.h
View file @
f4e1a4de
...
@@ -96,5 +96,7 @@ void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsign
...
@@ -96,5 +96,7 @@ void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsign
void
streamIteratorStop
(
streamIterator
*
si
);
void
streamIteratorStop
(
streamIterator
*
si
);
streamCG
*
streamLookupCG
(
stream
*
s
,
sds
groupname
);
streamCG
*
streamLookupCG
(
stream
*
s
,
sds
groupname
);
streamConsumer
*
streamLookupConsumer
(
streamCG
*
cg
,
sds
name
,
int
create
);
streamConsumer
*
streamLookupConsumer
(
streamCG
*
cg
,
sds
name
,
int
create
);
streamCG
*
streamCreateCG
(
stream
*
s
,
char
*
name
,
size_t
namelen
,
streamID
*
id
);
streamNACK
*
streamCreateNACK
(
streamConsumer
*
consumer
);
#endif
#endif
src/t_stream.c
View file @
f4e1a4de
...
@@ -41,7 +41,6 @@
...
@@ -41,7 +41,6 @@
#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1)
/* Same fields as master entry. */
#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1)
/* Same fields as master entry. */
void
streamFreeCG
(
streamCG
*
cg
);
void
streamFreeCG
(
streamCG
*
cg
);
streamNACK
*
streamCreateNACK
(
streamConsumer
*
consumer
);
size_t
streamReplyWithRangeFromConsumerPEL
(
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
,
streamCG
*
group
,
streamConsumer
*
consumer
);
size_t
streamReplyWithRangeFromConsumerPEL
(
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
,
streamCG
*
group
,
streamConsumer
*
consumer
);
/* -----------------------------------------------------------------------
/* -----------------------------------------------------------------------
...
@@ -1304,7 +1303,7 @@ void streamFreeConsumer(streamConsumer *sc) {
...
@@ -1304,7 +1303,7 @@ void streamFreeConsumer(streamConsumer *sc) {
* specified name and last server ID. If a consumer group with the same name
* specified name and last server ID. If a consumer group with the same name
* already existed NULL is returned, otherwise the pointer to the consumer
* already existed NULL is returned, otherwise the pointer to the consumer
* group is returned. */
* group is returned. */
streamCG
*
streamCreateCG
(
stream
*
s
,
char
*
name
,
size_t
namelen
,
streamID
id
)
{
streamCG
*
streamCreateCG
(
stream
*
s
,
char
*
name
,
size_t
namelen
,
streamID
*
id
)
{
if
(
s
->
cgroups
==
NULL
)
s
->
cgroups
=
raxNew
();
if
(
s
->
cgroups
==
NULL
)
s
->
cgroups
=
raxNew
();
if
(
raxFind
(
s
->
cgroups
,(
unsigned
char
*
)
name
,
namelen
)
!=
raxNotFound
)
if
(
raxFind
(
s
->
cgroups
,(
unsigned
char
*
)
name
,
namelen
)
!=
raxNotFound
)
return
NULL
;
return
NULL
;
...
@@ -1312,7 +1311,7 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID id) {
...
@@ -1312,7 +1311,7 @@ streamCG *streamCreateCG(stream *s, char *name, size_t namelen, streamID id) {
streamCG
*
cg
=
zmalloc
(
sizeof
(
*
cg
));
streamCG
*
cg
=
zmalloc
(
sizeof
(
*
cg
));
cg
->
pel
=
raxNew
();
cg
->
pel
=
raxNew
();
cg
->
consumers
=
raxNew
();
cg
->
consumers
=
raxNew
();
cg
->
last_id
=
id
;
cg
->
last_id
=
*
id
;
raxInsert
(
s
->
cgroups
,(
unsigned
char
*
)
name
,
namelen
,
cg
,
NULL
);
raxInsert
(
s
->
cgroups
,(
unsigned
char
*
)
name
,
namelen
,
cg
,
NULL
);
return
cg
;
return
cg
;
}
}
...
@@ -1388,7 +1387,7 @@ NULL
...
@@ -1388,7 +1387,7 @@ NULL
}
else
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
4
],
&
id
,
0
)
!=
C_OK
)
{
}
else
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
4
],
&
id
,
0
)
!=
C_OK
)
{
return
;
return
;
}
}
streamCG
*
cg
=
streamCreateCG
(
s
,
grpname
,
sdslen
(
grpname
),
id
);
streamCG
*
cg
=
streamCreateCG
(
s
,
grpname
,
sdslen
(
grpname
),
&
id
);
if
(
cg
)
{
if
(
cg
)
{
addReply
(
c
,
shared
.
ok
);
addReply
(
c
,
shared
.
ok
);
}
else
{
}
else
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment