Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
ee3490ec
Commit
ee3490ec
authored
Nov 17, 2017
by
antirez
Browse files
Streams: state machine for reverse iteration WIP 1.
parent
3c5d773f
Changes
4
Hide whitespace changes
Inline
Side-by-side
src/aof.c
View file @
ee3490ec
...
@@ -1035,7 +1035,7 @@ int rewriteHashObject(rio *r, robj *key, robj *o) {
...
@@ -1035,7 +1035,7 @@ int rewriteHashObject(rio *r, robj *key, robj *o) {
* The function returns 0 on error, 1 on success. */
* The function returns 0 on error, 1 on success. */
int
rewriteStreamObject
(
rio
*
r
,
robj
*
key
,
robj
*
o
)
{
int
rewriteStreamObject
(
rio
*
r
,
robj
*
key
,
robj
*
o
)
{
streamIterator
si
;
streamIterator
si
;
streamIteratorStart
(
&
si
,
o
->
ptr
,
NULL
,
NULL
);
streamIteratorStart
(
&
si
,
o
->
ptr
,
NULL
,
NULL
,
0
);
streamID
id
;
streamID
id
;
int64_t
numfields
;
int64_t
numfields
;
...
...
src/blocked.c
View file @
ee3490ec
...
@@ -326,7 +326,7 @@ void handleClientsBlockedOnKeys(void) {
...
@@ -326,7 +326,7 @@ void handleClientsBlockedOnKeys(void) {
addReplyMultiBulkLen
(
receiver
,
2
);
addReplyMultiBulkLen
(
receiver
,
2
);
addReplyBulk
(
receiver
,
rl
->
key
);
addReplyBulk
(
receiver
,
rl
->
key
);
streamReplyWithRange
(
receiver
,
s
,
&
start
,
NULL
,
streamReplyWithRange
(
receiver
,
s
,
&
start
,
NULL
,
receiver
->
bpop
.
xread_count
);
receiver
->
bpop
.
xread_count
,
0
);
}
}
}
}
}
}
...
...
src/stream.h
View file @
ee3490ec
...
@@ -28,9 +28,10 @@ typedef struct stream {
...
@@ -28,9 +28,10 @@ typedef struct stream {
typedef
struct
streamIterator
{
typedef
struct
streamIterator
{
streamID
master_id
;
/* ID of the master entry at listpack head. */
streamID
master_id
;
/* ID of the master entry at listpack head. */
uint64_t
master_fields_count
;
/* Master entries # of fields. */
uint64_t
master_fields_count
;
/* Master entries # of fields. */
unsigned
char
*
master_fields_start
;
/* Master entries start in list
a
pck. */
unsigned
char
*
master_fields_start
;
/* Master entries start in listp
a
ck. */
unsigned
char
*
master_fields_ptr
;
/* Master field to emit next. */
unsigned
char
*
master_fields_ptr
;
/* Master field to emit next. */
int
entry_flags
;
/* Flags of entry we are emitting. */
int
entry_flags
;
/* Flags of entry we are emitting. */
int
rev
;
/* True if iterating end to start (reverse). */
uint64_t
start_key
[
2
];
/* Start key as 128 bit big endian. */
uint64_t
start_key
[
2
];
/* Start key as 128 bit big endian. */
uint64_t
end_key
[
2
];
/* End key as 128 bit big endian. */
uint64_t
end_key
[
2
];
/* End key as 128 bit big endian. */
raxIterator
ri
;
/* Rax iterator. */
raxIterator
ri
;
/* Rax iterator. */
...
@@ -49,8 +50,8 @@ struct client;
...
@@ -49,8 +50,8 @@ struct client;
stream
*
streamNew
(
void
);
stream
*
streamNew
(
void
);
void
freeStream
(
stream
*
s
);
void
freeStream
(
stream
*
s
);
size_t
streamReplyWithRange
(
struct
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
);
size_t
streamReplyWithRange
(
struct
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
,
int
rev
);
void
streamIteratorStart
(
streamIterator
*
si
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
);
void
streamIteratorStart
(
streamIterator
*
si
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
int
rev
);
int
streamIteratorGetID
(
streamIterator
*
si
,
streamID
*
id
,
int64_t
*
numfields
);
int
streamIteratorGetID
(
streamIterator
*
si
,
streamID
*
id
,
int64_t
*
numfields
);
void
streamIteratorGetField
(
streamIterator
*
si
,
unsigned
char
**
fieldptr
,
unsigned
char
**
valueptr
,
int64_t
*
fieldlen
,
int64_t
*
valuelen
);
void
streamIteratorGetField
(
streamIterator
*
si
,
unsigned
char
**
fieldptr
,
unsigned
char
**
valueptr
,
int64_t
*
fieldlen
,
int64_t
*
valuelen
);
void
streamIteratorStop
(
streamIterator
*
si
);
void
streamIteratorStop
(
streamIterator
*
si
);
...
...
src/t_stream.c
View file @
ee3490ec
...
@@ -426,7 +426,9 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
...
@@ -426,7 +426,9 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
/* Initialize the stream iterator, so that we can call iterating functions
/* Initialize the stream iterator, so that we can call iterating functions
* to get the next items. This requires a corresponding streamIteratorStop()
* to get the next items. This requires a corresponding streamIteratorStop()
* at the end.
* at the end. The 'rev' parameter controls the direction. If it's zero the
* iteration is from the start to the end element (inclusive), otherwise
* if rev is non-zero, the iteration is reversed.
*
*
* Once the iterator is initalized, we iterate like this:
* Once the iterator is initalized, we iterate like this:
*
*
...
@@ -443,7 +445,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
...
@@ -443,7 +445,7 @@ int64_t streamTrimByLength(stream *s, size_t maxlen, int approx) {
* }
* }
* }
* }
* streamIteratorStop(&myiterator); */
* streamIteratorStop(&myiterator); */
void
streamIteratorStart
(
streamIterator
*
si
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
)
{
void
streamIteratorStart
(
streamIterator
*
si
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
int
rev
)
{
/* Intialize the iterator and translates the iteration start/stop
/* Intialize the iterator and translates the iteration start/stop
* elements into a 128 big big-endian number. */
* elements into a 128 big big-endian number. */
if
(
start
)
{
if
(
start
)
{
...
@@ -462,17 +464,26 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI
...
@@ -462,17 +464,26 @@ void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamI
/* Seek the correct node in the radix tree. */
/* Seek the correct node in the radix tree. */
raxStart
(
&
si
->
ri
,
s
->
rax
);
raxStart
(
&
si
->
ri
,
s
->
rax
);
if
(
start
&&
(
start
->
ms
||
start
->
seq
))
{
if
(
!
rev
)
{
raxSeek
(
&
si
->
ri
,
"<="
,(
unsigned
char
*
)
si
->
start_key
,
if
(
start
&&
(
start
->
ms
||
start
->
seq
))
{
sizeof
(
si
->
start_key
));
raxSeek
(
&
si
->
ri
,
"<="
,(
unsigned
char
*
)
si
->
start_key
,
if
(
raxEOF
(
&
si
->
ri
))
raxSeek
(
&
si
->
ri
,
">"
,(
unsigned
char
*
)
si
->
start_key
,
sizeof
(
si
->
start_key
));
sizeof
(
si
->
start_key
));
if
(
raxEOF
(
&
si
->
ri
))
raxSeek
(
&
si
->
ri
,
"^"
,
NULL
,
0
);
}
else
{
raxSeek
(
&
si
->
ri
,
"^"
,
NULL
,
0
);
}
}
else
{
}
else
{
raxSeek
(
&
si
->
ri
,
"^"
,
NULL
,
0
);
if
(
end
&&
(
end
->
ms
||
end
->
seq
))
{
raxSeek
(
&
si
->
ri
,
"<="
,(
unsigned
char
*
)
si
->
end_key
,
sizeof
(
si
->
end_key
));
if
(
raxEOF
(
&
si
->
ri
))
raxSeek
(
&
si
->
ri
,
"$"
,
NULL
,
0
);
}
else
{
raxSeek
(
&
si
->
ri
,
"$"
,
NULL
,
0
);
}
}
}
si
->
lp
=
NULL
;
/* There is no current listpack right now. */
si
->
lp
=
NULL
;
/* There is no current listpack right now. */
si
->
lp_ele
=
NULL
;
/* Current listpack cursor. */
si
->
lp_ele
=
NULL
;
/* Current listpack cursor. */
si
->
rev
=
rev
;
/* Direction, if non-zero reversed, from end to start. */
}
}
/* Return 1 and store the current item ID at 'id' if there are still
/* Return 1 and store the current item ID at 'id' if there are still
...
@@ -484,7 +495,8 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
...
@@ -484,7 +495,8 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
* iteration or the previous listpack was completely iterated.
* iteration or the previous listpack was completely iterated.
* Go to the next node. */
* Go to the next node. */
if
(
si
->
lp
==
NULL
||
si
->
lp_ele
==
NULL
)
{
if
(
si
->
lp
==
NULL
||
si
->
lp_ele
==
NULL
)
{
if
(
!
raxNext
(
&
si
->
ri
))
return
0
;
if
(
!
si
->
rev
&&
!
raxNext
(
&
si
->
ri
))
return
0
;
else
if
(
si
->
rev
&&
!
raxPrev
(
&
si
->
ri
))
return
0
;
serverAssert
(
si
->
ri
.
key_len
==
sizeof
(
streamID
));
serverAssert
(
si
->
ri
.
key_len
==
sizeof
(
streamID
));
/* Get the master ID. */
/* Get the master ID. */
streamDecodeID
(
si
->
ri
.
key
,
&
si
->
master_id
);
streamDecodeID
(
si
->
ri
.
key
,
&
si
->
master_id
);
...
@@ -499,16 +511,38 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
...
@@ -499,16 +511,38 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
/* Skip master fileds to seek the first entry. */
/* Skip master fileds to seek the first entry. */
for
(
uint64_t
i
=
0
;
i
<
si
->
master_fields_count
;
i
++
)
for
(
uint64_t
i
=
0
;
i
<
si
->
master_fields_count
;
i
++
)
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
/* We are now pointing the zero term of the master entry. */
/* We are now pointing the zero term of the master entry. If
* we are iterating in reverse order, we need to seek the
* end of the listpack. */
if
(
si
->
rev
)
si
->
lp_ele
=
lpLast
(
si
->
lp
);
}
else
if
(
si
->
rev
)
{
/* If we are itereating in the reverse order, and this is not
* the first entry emitted for this listpack, then we already
* emitted the current entry, and have to go back to the previous
* one. */
int
lp_count
=
lpGetInteger
(
si
->
lp_ele
);
while
(
lp_count
--
)
si
->
lp_ele
=
lpPrev
(
si
->
lp
,
si
->
lp_ele
);
/* Seek lp-count of prev entry. */
si
->
lp_ele
=
lpPrev
(
si
->
lp
,
si
->
lp_ele
);
}
}
/* For every radix tree node, iterate the corresponding listpack,
/* For every radix tree node, iterate the corresponding listpack,
* returning elements when they are within range. */
* returning elements when they are within range. */
while
(
1
)
{
while
(
1
)
{
/* Skip the previous entry lp-count field, or in case of the
if
(
!
si
->
rev
)
{
* master entry, the zero term field. */
/* If we are going forward, skip the previous entry
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
* lp-count field (or in case of the master entry, the zero
if
(
si
->
lp_ele
==
NULL
)
break
;
* term field) */
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
if
(
si
->
lp_ele
==
NULL
)
break
;
}
else
{
/* If we are going backward, read the number of elements this
* entry is composed of, and jump backward N times to seek
* its start. */
int
lp_count
=
lpGetInteger
(
si
->
lp_ele
);
if
(
lp_count
==
0
)
break
;
/* We reached the master entry. */
while
(
lp_count
--
)
si
->
lp_ele
=
lpPrev
(
si
->
lp
,
si
->
lp_ele
);
}
/* Get the flags entry. */
/* Get the flags entry. */
int
flags
=
lpGetInteger
(
si
->
lp_ele
);
int
flags
=
lpGetInteger
(
si
->
lp_ele
);
...
@@ -535,15 +569,28 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
...
@@ -535,15 +569,28 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
/* If current >= start, and the entry is not marked as
/* If current >= start, and the entry is not marked as
* deleted, emit it. */
* deleted, emit it. */
if
(
memcmp
(
buf
,
si
->
start_key
,
sizeof
(
streamID
))
>=
0
&&
if
(
!
si
->
rev
)
{
!
(
flags
&
STREAM_ITEM_FLAG_DELETED
))
if
(
memcmp
(
buf
,
si
->
start_key
,
sizeof
(
streamID
))
>=
0
&&
{
!
(
flags
&
STREAM_ITEM_FLAG_DELETED
))
if
(
memcmp
(
buf
,
si
->
end_key
,
sizeof
(
streamID
))
>
0
)
{
return
0
;
/* We are already out of range. */
if
(
memcmp
(
buf
,
si
->
end_key
,
sizeof
(
streamID
))
>
0
)
si
->
entry_flags
=
flags
;
return
0
;
/* We are already out of range. */
if
(
flags
&
STREAM_ITEM_FLAG_SAMEFIELDS
)
si
->
entry_flags
=
flags
;
si
->
master_fields_ptr
=
si
->
master_fields_start
;
if
(
flags
&
STREAM_ITEM_FLAG_SAMEFIELDS
)
return
1
;
/* Valid item returned. */
si
->
master_fields_ptr
=
si
->
master_fields_start
;
return
1
;
/* Valid item returned. */
}
}
else
{
if
(
memcmp
(
buf
,
si
->
end_key
,
sizeof
(
streamID
))
<=
0
&&
!
(
flags
&
STREAM_ITEM_FLAG_DELETED
))
{
if
(
memcmp
(
buf
,
si
->
start_key
,
sizeof
(
streamID
))
<
0
)
return
0
;
/* We are already out of range. */
si
->
entry_flags
=
flags
;
if
(
flags
&
STREAM_ITEM_FLAG_SAMEFIELDS
)
si
->
master_fields_ptr
=
si
->
master_fields_start
;
return
1
;
/* Valid item returned. */
}
}
}
/* If we do not emit, we have to discard. */
/* If we do not emit, we have to discard. */
...
@@ -553,7 +600,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
...
@@ -553,7 +600,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
si
->
lp_ele
=
lpNext
(
si
->
lp
,
si
->
lp_ele
);
}
}
/* End of listpack reached. Try the next radix tree node. */
/* End of listpack reached. Try the next
/prev
radix tree node. */
}
}
}
}
...
@@ -585,15 +632,16 @@ void streamIteratorStop(streamIterator *si) {
...
@@ -585,15 +632,16 @@ void streamIteratorStop(streamIterator *si) {
/* Send the specified range to the client 'c'. The range the client will
/* Send the specified range to the client 'c'. The range the client will
* receive is between start and end inclusive, if 'count' is non zero, no more
* receive is between start and end inclusive, if 'count' is non zero, no more
* than 'count' elemnets are sent. The 'end' pointer can be NULL to mean that
* than 'count' elemnets are sent. The 'end' pointer can be NULL to mean that
* we want all the elements from 'start' till the end of the stream. */
* we want all the elements from 'start' till the end of the stream. If 'rev'
size_t
streamReplyWithRange
(
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
)
{
* is non zero, elements are produced in reversed order from end to start. */
size_t
streamReplyWithRange
(
client
*
c
,
stream
*
s
,
streamID
*
start
,
streamID
*
end
,
size_t
count
,
int
rev
)
{
void
*
arraylen_ptr
=
addDeferredMultiBulkLength
(
c
);
void
*
arraylen_ptr
=
addDeferredMultiBulkLength
(
c
);
size_t
arraylen
=
0
;
size_t
arraylen
=
0
;
streamIterator
si
;
streamIterator
si
;
int64_t
numfields
;
int64_t
numfields
;
streamID
id
;
streamID
id
;
streamIteratorStart
(
&
si
,
s
,
start
,
end
);
streamIteratorStart
(
&
si
,
s
,
start
,
end
,
rev
);
while
(
streamIteratorGetID
(
&
si
,
&
id
,
&
numfields
))
{
while
(
streamIteratorGetID
(
&
si
,
&
id
,
&
numfields
))
{
/* Emit a two elements array for each item. The first is
/* Emit a two elements array for each item. The first is
* the ID, the second is an array of field-value pairs. */
* the ID, the second is an array of field-value pairs. */
...
@@ -797,25 +845,32 @@ void xaddCommand(client *c) {
...
@@ -797,25 +845,32 @@ void xaddCommand(client *c) {
signalKeyAsReady
(
c
->
db
,
c
->
argv
[
1
]);
signalKeyAsReady
(
c
->
db
,
c
->
argv
[
1
]);
}
}
/* XRANGE key start end [COUNT <n>] */
/* XRANGE key start end [COUNT <n>]
[REV]
*/
void
xrangeCommand
(
client
*
c
)
{
void
xrangeCommand
(
client
*
c
)
{
robj
*
o
;
robj
*
o
;
stream
*
s
;
stream
*
s
;
streamID
startid
,
endid
;
streamID
startid
,
endid
;
long
long
count
=
0
;
long
long
count
=
0
;
int
rev
=
0
;
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
2
],
&
startid
,
0
)
==
C_ERR
)
return
;
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
2
],
&
startid
,
0
)
==
C_ERR
)
return
;
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
3
],
&
endid
,
UINT64_MAX
)
==
C_ERR
)
return
;
if
(
streamParseIDOrReply
(
c
,
c
->
argv
[
3
],
&
endid
,
UINT64_MAX
)
==
C_ERR
)
return
;
/* Parse the COUNT option if any. */
/* Parse the COUNT option if any. */
if
(
c
->
argc
>
5
)
{
if
(
c
->
argc
>
4
)
{
if
(
strcasecmp
(
c
->
argv
[
4
]
->
ptr
,
"COUNT"
)
==
0
)
{
for
(
int
j
=
4
;
j
<
c
->
argc
;
j
++
)
{
if
(
getLongLongFromObjectOrReply
(
c
,
c
->
argv
[
5
],
&
count
,
NULL
)
!=
C_OK
)
int
additional
=
c
->
argc
-
j
-
1
;
if
(
strcasecmp
(
c
->
argv
[
j
]
->
ptr
,
"COUNT"
)
==
0
&&
additional
>=
1
)
{
if
(
getLongLongFromObjectOrReply
(
c
,
c
->
argv
[
j
+
1
],
&
count
,
NULL
)
!=
C_OK
)
return
;
if
(
count
<
0
)
count
=
0
;
j
++
;
/* Consume additional arg. */
}
else
if
(
strcasecmp
(
c
->
argv
[
j
]
->
ptr
,
"REV"
)
==
0
)
{
rev
=
1
;
}
else
{
addReply
(
c
,
shared
.
syntaxerr
);
return
;
return
;
if
(
count
<
0
)
count
=
0
;
}
}
else
{
addReply
(
c
,
shared
.
syntaxerr
);
return
;
}
}
}
}
...
@@ -823,7 +878,7 @@ void xrangeCommand(client *c) {
...
@@ -823,7 +878,7 @@ void xrangeCommand(client *c) {
if
((
o
=
lookupKeyReadOrReply
(
c
,
c
->
argv
[
1
],
shared
.
emptymultibulk
))
==
NULL
if
((
o
=
lookupKeyReadOrReply
(
c
,
c
->
argv
[
1
],
shared
.
emptymultibulk
))
==
NULL
||
checkType
(
c
,
o
,
OBJ_STREAM
))
return
;
||
checkType
(
c
,
o
,
OBJ_STREAM
))
return
;
s
=
o
->
ptr
;
s
=
o
->
ptr
;
streamReplyWithRange
(
c
,
s
,
&
startid
,
&
endid
,
count
);
streamReplyWithRange
(
c
,
s
,
&
startid
,
&
endid
,
count
,
rev
);
}
}
/* XLEN */
/* XLEN */
...
@@ -931,7 +986,7 @@ void xreadCommand(client *c) {
...
@@ -931,7 +986,7 @@ void xreadCommand(client *c) {
* of the stream and the data we extracted from it. */
* of the stream and the data we extracted from it. */
addReplyMultiBulkLen
(
c
,
2
);
addReplyMultiBulkLen
(
c
,
2
);
addReplyBulk
(
c
,
c
->
argv
[
i
+
streams_arg
]);
addReplyBulk
(
c
,
c
->
argv
[
i
+
streams_arg
]);
streamReplyWithRange
(
c
,
s
,
&
start
,
NULL
,
count
);
streamReplyWithRange
(
c
,
s
,
&
start
,
NULL
,
count
,
0
);
}
}
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment