Unverified Commit 814aad65 authored by guybe7's avatar guybe7 Committed by GitHub
Browse files

XADD and XTRIM, Trim by MINID, and new LIMIT argument (#8169)

This PR adds another trimming strategy to XADD and XTRIM named MINID
(complements the existing MAXLEN).
It also adds a new LIMIT argument that allows incremental trimming by repeated
calls (rather than all at once).

This provides the ability to trim all records older than a certain ID (which makes it
possible for the user to trim by age too).
Example:
XTRIM mystream MINID ~ 1608540753 will trim entries with id < 1608540753,
but might not trim all (because of the ~ modifier)

The purpose is to ease the use of streams. many users use streams as logs and
the common case is wanting a log
of the last X seconds rather than a log that contains maximum X entries (new
MINID vs existing MAXLEN)

The new LIMIT modifier is only supported when the trim strategy uses ~.
i.e. when the user asked for exact trimming, it all happens in one go (no
possibility for incremental trimming).
However, when ~ is provided, we trim full rax nodes, up to the limit number
of records.
The default limit is 100*stream_node_max_entries (used when LIMIT is not
provided).
I.e. this is a behavior change (even if the existing MAXLEN strategy is used).
An explicit limit of 0 means unlimited (but note that it's not the default).

Other changes:

Refactor arg parsing code for XADD and XTRIM to use common code.
parent 5843a45d
This diff is collapsed.
...@@ -94,6 +94,7 @@ start_server { ...@@ -94,6 +94,7 @@ start_server {
r XADD mystream MAXLEN 5 * yitem $j r XADD mystream MAXLEN 5 * yitem $j
} }
} }
assert {[r xlen mystream] == 5}
set res [r xrange mystream - +] set res [r xrange mystream - +]
set expected 995 set expected 995
foreach r $res { foreach r $res {
...@@ -138,6 +139,39 @@ start_server { ...@@ -138,6 +139,39 @@ start_server {
assert_equal [lindex $items 1 1] {item 2 value b} assert_equal [lindex $items 1 1] {item 2 value b}
} }
test {XADD with MINID option} {
r DEL mystream
for {set j 1} {$j < 1001} {incr j} {
set minid 1000
if {$j >= 5} {
set minid [expr {$j-5}]
}
if {rand() < 0.9} {
r XADD mystream MINID $minid $j xitem $j
} else {
r XADD mystream MINID $minid $j yitem $j
}
}
assert {[r xlen mystream] == 6}
set res [r xrange mystream - +]
set expected 995
foreach r $res {
assert {[lindex $r 1 1] == $expected}
incr expected
}
}
test {XTRIM with MINID option} {
r DEL mystream
r XADD mystream 1-0 f v
r XADD mystream 2-0 f v
r XADD mystream 3-0 f v
r XADD mystream 4-0 f v
r XADD mystream 5-0 f v
r XTRIM mystream MINID = 3-0
assert_equal [r XRANGE mystream - +] {{3-0 {f v}} {4-0 {f v}} {5-0 {f v}}}
}
test {XADD mass insertion and XLEN} { test {XADD mass insertion and XLEN} {
r DEL mystream r DEL mystream
r multi r multi
...@@ -448,7 +482,49 @@ start_server { ...@@ -448,7 +482,49 @@ start_server {
assert {[r XLEN mystream] == 400} assert {[r XLEN mystream] == 400}
} }
test {XADD with LIMIT consecutive calls} {
r del mystream
r config set stream-node-max-entries 10
for {set j 0} {$j < 100} {incr j} {
r XADD mystream * xitem v
}
r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
assert {[r xlen mystream] == 71}
r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
assert {[r xlen mystream] == 62}
r config set stream-node-max-entries 100
}
test {XTRIM with ~ is limited} {
r del mystream
r config set stream-node-max-entries 1
for {set j 0} {$j < 102} {incr j} {
r XADD mystream * xitem v
}
r XTRIM mystream MAXLEN ~ 1
assert {[r xlen mystream] == 2}
r config set stream-node-max-entries 100
}
test {XTRIM without ~ is not limited} {
r del mystream
r config set stream-node-max-entries 1
for {set j 0} {$j < 102} {incr j} {
r XADD mystream * xitem v
}
r XTRIM mystream MAXLEN 1
assert {[r xlen mystream] == 1}
r config set stream-node-max-entries 100
}
test {XTRIM without ~ and with LIMIT} {
r del mystream
r config set stream-node-max-entries 1
for {set j 0} {$j < 102} {incr j} {
r XADD mystream * xitem v
}
assert_error ERR* {r XTRIM mystream MAXLEN 1 LIMIT 30}
}
} }
start_server {tags {"stream"} overrides {appendonly yes}} { start_server {tags {"stream"} overrides {appendonly yes}} {
...@@ -466,6 +542,22 @@ start_server {tags {"stream"} overrides {appendonly yes}} { ...@@ -466,6 +542,22 @@ start_server {tags {"stream"} overrides {appendonly yes}} {
} }
} }
start_server {tags {"stream"} overrides {appendonly yes}} {
test {XADD with MINID > lastid can propagate correctly} {
for {set j 0} {$j < 100} {incr j} {
set id [expr {$j+1}]
r XADD mystream $id xitem v
}
r XADD mystream MINID 1 * xitem v
incr j
assert {[r xlen mystream] == $j}
r debug loadaof
r XADD mystream * xitem v
incr j
assert {[r xlen mystream] == $j}
}
}
start_server {tags {"stream"} overrides {appendonly yes}} { start_server {tags {"stream"} overrides {appendonly yes}} {
test {XADD with ~ MAXLEN can propagate correctly} { test {XADD with ~ MAXLEN can propagate correctly} {
for {set j 0} {$j < 100} {incr j} { for {set j 0} {$j < 100} {incr j} {
...@@ -482,6 +574,52 @@ start_server {tags {"stream"} overrides {appendonly yes}} { ...@@ -482,6 +574,52 @@ start_server {tags {"stream"} overrides {appendonly yes}} {
} }
} }
start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
test {XADD with ~ MAXLEN and LIMIT can propagate correctly} {
for {set j 0} {$j < 100} {incr j} {
r XADD mystream * xitem v
}
r XADD mystream MAXLEN ~ 55 LIMIT 30 * xitem v
assert {[r xlen mystream] == 71}
r config set stream-node-max-entries 1
r debug loadaof
r XADD mystream * xitem v
assert {[r xlen mystream] == 72}
}
}
start_server {tags {"stream"} overrides {appendonly yes}} {
test {XADD with ~ MINID can propagate correctly} {
for {set j 0} {$j < 100} {incr j} {
set id [expr {$j+1}]
r XADD mystream $id xitem v
}
r XADD mystream MINID ~ $j * xitem v
incr j
assert {[r xlen mystream] == $j}
r config set stream-node-max-entries 1
r debug loadaof
r XADD mystream * xitem v
incr j
assert {[r xlen mystream] == $j}
}
}
start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
test {XADD with ~ MINID and LIMIT can propagate correctly} {
for {set j 0} {$j < 100} {incr j} {
set id [expr {$j+1}]
r XADD mystream $id xitem v
}
r XADD mystream MINID ~ 55 LIMIT 30 * xitem v
assert {[r xlen mystream] == 71}
r config set stream-node-max-entries 1
r debug loadaof
r XADD mystream * xitem v
assert {[r xlen mystream] == 72}
}
}
start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} { start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} {
test {XTRIM with ~ MAXLEN can propagate correctly} { test {XTRIM with ~ MAXLEN can propagate correctly} {
for {set j 0} {$j < 100} {incr j} { for {set j 0} {$j < 100} {incr j} {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment