Unverified Commit 30eb6c32 authored by YaacovHazan's avatar YaacovHazan Committed by GitHub
Browse files

Merge unstable into 8.0 (#13573)

parents 5fe3e74a 80458d04
......@@ -62,7 +62,7 @@ jobs:
- uses: actions/checkout@v4
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
sudo apt-get update && sudo apt-get install libc6-dev-i386 gcc-multilib g++-multilib
make REDIS_CFLAGS='-Werror' 32bit
build-libc-malloc:
......@@ -79,7 +79,7 @@ jobs:
- uses: actions/checkout@v4
- name: make
run: |
dnf -y install which gcc make
dnf -y install which gcc gcc-c++ make
make REDIS_CFLAGS='-Werror'
build-old-chain-jemalloc:
......@@ -96,6 +96,7 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8
apt-get install -y make gcc-4.8 g++-4.8
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc REDIS_CFLAGS='-Werror'
......@@ -94,8 +94,9 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
apt-get update && apt-get install -y make gcc-13
apt-get update && apt-get install -y make gcc-13 g++-13
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100
make CC=gcc REDIS_CFLAGS='-Werror -DREDIS_TEST -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3'
- name: testprep
run: apt-get install -y tcl8.6 tclx procps
......@@ -211,7 +212,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
sudo apt-get update && sudo apt-get install libc6-dev-i386
sudo apt-get update && sudo apt-get install libc6-dev-i386 g++ gcc-multilib g++-multilib
make 32bit REDIS_CFLAGS='-Werror -DREDIS_TEST'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
......@@ -456,7 +457,7 @@ jobs:
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
sudo apt-get install tcl8.6 tclx valgrind g++ -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
......@@ -521,7 +522,7 @@ jobs:
- name: testprep
run: |
sudo apt-get update
sudo apt-get install tcl8.6 tclx valgrind -y
sudo apt-get install tcl8.6 tclx valgrind g++ -y
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --valgrind --no-latency --verbose --clients 1 --timeout 2400 --dump-logs ${{github.event.inputs.test_args}}
......@@ -678,7 +679,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make
dnf -y install which gcc make g++
make REDIS_CFLAGS='-Werror'
- name: testprep
run: |
......@@ -720,7 +721,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make openssl-devel openssl
dnf -y install which gcc make openssl-devel openssl g++
make BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
......@@ -767,7 +768,7 @@ jobs:
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: |
dnf -y install which gcc make openssl-devel openssl
dnf -y install which gcc make openssl-devel openssl g++
make BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
......@@ -907,6 +908,9 @@ jobs:
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'freebsd')
timeout-minutes: 14400
env:
CC: clang
CXX: clang++
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
......@@ -925,7 +929,7 @@ jobs:
version: 13.2
shell: bash
run: |
sudo pkg install -y bash gmake lang/tcl86 lang/tclx
sudo pkg install -y bash gmake lang/tcl86 lang/tclx gcc
gmake
./runtest --single unit/keyspace --single unit/auth --single unit/networking --single unit/protocol
......@@ -1080,8 +1084,9 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8
apt-get install -y make gcc-4.8 g++-4.8
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc REDIS_CFLAGS='-Werror'
- name: testprep
run: apt-get install -y tcl tcltls tclx
......@@ -1128,9 +1133,10 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8 openssl libssl-dev
apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
make CC=gcc BUILD_TLS=module REDIS_CFLAGS='-Werror'
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make CC=gcc CXX=g++ BUILD_TLS=module REDIS_CFLAGS='-Werror'
- name: testprep
run: |
apt-get install -y tcl tcltls tclx
......@@ -1182,8 +1188,9 @@ jobs:
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
apt-get update
apt-get install -y make gcc-4.8 openssl libssl-dev
apt-get install -y make gcc-4.8 g++-4.8 openssl libssl-dev
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 100
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 100
make BUILD_TLS=module CC=gcc REDIS_CFLAGS='-Werror'
- name: testprep
run: |
......
......@@ -30,6 +30,7 @@ deps/lua/src/luac
deps/lua/src/liblua.a
deps/hdr_histogram/libhdrhistogram.a
deps/fpconv/libfpconv.a
deps/fast_float/libfast_float.a
tests/tls/*
.make-*
.prerequisites
......
......@@ -42,6 +42,7 @@ distclean:
-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
-(cd fpconv && $(MAKE) clean) > /dev/null || true
-(cd fast_float && $(MAKE) clean) > /dev/null || true
-(rm -f .make-*)
.PHONY: distclean
......@@ -74,6 +75,12 @@ fpconv: .make-prerequisites
.PHONY: fpconv
fast_float: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
cd fast_float && $(MAKE) libfast_float
.PHONY: fast_float
ifeq ($(uname_S),SunOS)
# Make isinf() available
LUA_CFLAGS= -D__C99FEATURES__=1
......
# Fallback to gcc/g++ when $CC or $CXX is not in $PATH.
CC ?= gcc
CXX ?= g++
CFLAGS=-Wall -O3
# This avoids loosing the fastfloat specific compile flags when we override the CFLAGS via the main project
FASTFLOAT_CFLAGS=-std=c++11 -DFASTFLOAT_ALLOWS_LEADING_PLUS
LDFLAGS=
libfast_float: fast_float_strtod.o
$(AR) -r libfast_float.a fast_float_strtod.o
32bit: CFLAGS += -m32
32bit: LDFLAGS += -m32
32bit: libfast_float
fast_float_strtod.o: fast_float_strtod.cpp
$(CXX) $(CFLAGS) $(FASTFLOAT_CFLAGS) -c fast_float_strtod.cpp $(LDFLAGS)
clean:
rm -f *.o
rm -f *.a
rm -f *.h.gch
rm -rf *.dSYM
README for fast_float v6.1.4
----------------------------------------------
We're using the fast_float library[1] in our (compiled-in)
floating-point fast_float_strtod implementation for faster and more
portable parsing of 64 decimal strings.
The single file fast_float.h is an amalgamation of the entire library,
which can be (re)generated with the amalgamate.py script (from the
fast_float repository) via the command
```
git clone https://github.com/fastfloat/fast_float
cd fast_float
git checkout v6.1.4
python3 ./script/amalgamate.py --license=MIT \
> $REDIS_SRC/deps/fast_float/fast_float.h
```
[1]: https://github.com/fastfloat/fast_float
This diff is collapsed.
#include "fast_float.h"
#include <iostream>
#include <string>
#include <system_error>
#include <cerrno>
/* Convert NPTR to a double using the fast_float library.
*
* This function behaves similarly to the standard strtod function, converting
* the initial portion of the string pointed to by `nptr` to a `double` value,
* using the fast_float library for high performance. If the conversion fails,
* errno is set to EINVAL error code.
*
* @param nptr A pointer to the null-terminated byte string to be interpreted.
* @param endptr A pointer to a pointer to character. If `endptr` is not NULL,
* it will point to the character after the last character used
* in the conversion.
* @return The converted value as a double. If no valid conversion could
* be performed, returns 0.0.
* If ENDPTR is not NULL, a pointer to the character after the last one used
* in the number is put in *ENDPTR. */
extern "C" double fast_float_strtod(const char *nptr, char **endptr) {
double result = 0.0;
auto answer = fast_float::from_chars(nptr, nptr + strlen(nptr), result);
if (answer.ec != std::errc()) {
errno = EINVAL; // Fallback to for other errors
}
if (endptr != NULL) {
*endptr = (char *)answer.ptr;
}
return result;
}
#ifndef __FAST_FLOAT_STRTOD_H__
#define __FAST_FLOAT_STRTOD_H__
#if defined(__cplusplus)
extern "C"
{
#endif
double fast_float_strtod(const char *in, char **out);
#if defined(__cplusplus)
}
#endif
#endif /* __FAST_FLOAT_STRTOD_H__ */
......@@ -33,11 +33,29 @@ clean_environment: uninstall-rust
install-rust:
ifeq ($(INSTALL_RUST_TOOLCHAIN),yes)
@RUST_VERSION=1.80.1; \
case "$$(uname -m)" in \
'x86_64') RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-gnu"; RUST_SHA256="85e936d5d36970afb80756fa122edcc99bd72a88155f6bdd514f5d27e778e00a" ;; \
'aarch64') RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-gnu"; RUST_SHA256="2e89bad7857711a1c11d017ea28fbfeec54076317763901194f8f5decbac1850" ;; \
*) echo >&2 "Unsupported architecture: '$$(uname -m)'"; exit 1 ;; \
ARCH="$$(uname -m)"; \
if ldd --version 2>&1 | grep -q musl; then LIBC_TYPE="musl"; else LIBC_TYPE="gnu"; fi; \
echo "Detected architecture: $${ARCH} and libc: $${LIBC_TYPE}"; \
case "$${ARCH}" in \
'x86_64') \
if [ "$${LIBC_TYPE}" = "musl" ]; then \
RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-musl"; \
RUST_SHA256="37bbec6a7b9f55fef79c451260766d281a7a5b9d2e65c348bbc241127cf34c8d"; \
else \
RUST_INSTALLER="rust-$${RUST_VERSION}-x86_64-unknown-linux-gnu"; \
RUST_SHA256="85e936d5d36970afb80756fa122edcc99bd72a88155f6bdd514f5d27e778e00a"; \
fi ;; \
'aarch64') \
if [ "$${LIBC_TYPE}" = "musl" ]; then \
RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-musl"; \
RUST_SHA256="dd668c2d82f77c5458deb023932600fae633fff8d7f876330e01bc47e9976d17"; \
else \
RUST_INSTALLER="rust-$${RUST_VERSION}-aarch64-unknown-linux-gnu"; \
RUST_SHA256="2e89bad7857711a1c11d017ea28fbfeec54076317763901194f8f5decbac1850"; \
fi ;; \
*) echo >&2 "Unsupported architecture: '$${ARCH}'"; exit 1 ;; \
esac; \
echo "Downloading and installing Rust standalone installer: $${RUST_INSTALLER}"; \
wget --quiet -O $${RUST_INSTALLER}.tar.xz https://static.rust-lang.org/dist/$${RUST_INSTALLER}.tar.xz; \
echo "$${RUST_SHA256} $${RUST_INSTALLER}.tar.xz" | sha256sum -c --quiet || { echo "Rust standalone installer checksum failed!"; exit 1; }; \
tar -xf $${RUST_INSTALLER}.tar.xz; \
......
......@@ -34,7 +34,7 @@ endif
ifneq ($(OPTIMIZATION),-O0)
OPTIMIZATION+=-fno-omit-frame-pointer
endif
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv fast_float
NODEPS:=clean distclean
# Default settings
......@@ -127,7 +127,7 @@ endif
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
FINAL_LDFLAGS=$(LDFLAGS) $(OPT) $(REDIS_LDFLAGS) $(DEBUG)
FINAL_LIBS=-lm
FINAL_LIBS=-lm -lstdc++
DEBUG=-g -ggdb
# Linux ARM32 needs -latomic at linking time
......@@ -235,7 +235,7 @@ ifdef OPENSSL_PREFIX
endif
# Include paths to dependencies
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/fpconv -I../deps/fast_float
# Determine systemd support and/or build preference (defaulting to auto-detection)
BUILD_WITH_SYSTEMD=no
......@@ -409,7 +409,7 @@ endif
# redis-server
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a $(FINAL_LIBS)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/hdr_histogram/libhdrhistogram.a ../deps/fpconv/libfpconv.a ../deps/fast_float/libfast_float.a $(FINAL_LIBS)
# redis-sentinel
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
......
......@@ -10,7 +10,7 @@ const char *ascii_logo =
" _._ \n"
" _.-``__ ''-._ \n"
" _.-`` `. `_. ''-._ Redis Community Edition \n"
" .-`` .-```. ```\\/ _.,_ ''-._ %s (%s/%d) %s bit\n"
" .-`` .-```. ```\\/ _.,_ ''-._ %s (%s/%d) %s bit\n"
" ( ' , .-` | `, ) Running in %s mode\n"
" |`-._`-...-` __...-.``-._|'` _.-'| Port: %d\n"
" | `-._ `._ / _.-' | PID: %ld\n"
......
......@@ -93,6 +93,7 @@ int auxTlsPortPresent(clusterNode *n);
static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen);
void freeClusterLink(clusterLink *link);
int verifyClusterNodeId(const char *name, int length);
static void updateShardId(clusterNode *node, const char *shard_id);
int getNodeDefaultClientPort(clusterNode *n) {
return server.tls_cluster ? n->tls_port : n->tcp_port;
......@@ -203,12 +204,11 @@ int auxShardIdSetter(clusterNode *n, void *value, int length) {
return C_ERR;
}
memcpy(n->shard_id, value, CLUSTER_NAMELEN);
/* if n already has replicas, make sure they all agree
* on the shard id */
/* if n already has replicas, make sure they all use
* the primary shard id */
for (int i = 0; i < n->numslaves; i++) {
if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0) {
return C_ERR;
}
if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
updateShardId(n->slaves[i], n->shard_id);
}
clusterAddNodeToShard(value, n);
return C_OK;
......@@ -550,18 +550,12 @@ int clusterLoadConfig(char *filename) {
clusterAddNode(master);
}
/* shard_id can be absent if we are loading a nodes.conf generated
* by an older version of Redis; we should follow the primary's
* shard_id in this case */
if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(master->shard_id, n);
} else if (clusterGetNodesInMyShard(master) != NULL &&
memcmp(master->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)
{
/* If the primary has been added to a shard, make sure this
* node has the same persisted shard id as the primary. */
goto fmterr;
}
* by an older version of Redis;
* ignore replica's shard_id in the file, only use the primary's.
* If replica precedes primary in file, it will be corrected
* later by the auxShardIdSetter */
memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(master->shard_id, n);
n->slaveof = master;
clusterNodeAddSlave(master,n);
} else if (auxFieldHandlers[af_shard_id].isPresent(n) == 0) {
......@@ -908,22 +902,39 @@ static void updateAnnouncedHumanNodename(clusterNode *node, char *new) {
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
static void assignShardIdToNode(clusterNode *node, const char *shard_id, int flag) {
clusterRemoveNodeFromShard(node);
memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, node);
clusterDoBeforeSleep(flag);
}
static void updateShardId(clusterNode *node, const char *shard_id) {
if (shard_id && memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
clusterRemoveNodeFromShard(node);
memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, node);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);
}
if (shard_id && myself != node && myself->slaveof == node) {
if (memcmp(myself->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {
/* shard-id can diverge right after a rolling upgrade
* from pre-7.2 releases */
clusterRemoveNodeFromShard(myself);
memcpy(myself->shard_id, shard_id, CLUSTER_NAMELEN);
clusterAddNodeToShard(shard_id, myself);
clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);
/* If the replica or master does not support shard-id (old version),
* we still need to make our best effort to keep their shard-id consistent.
*
* 1. Master supports but the replica does not.
* We might first update the replica's shard-id to the master's randomly
* generated shard-id. Then, when the master's shard-id arrives, we must
* also update all its replicas.
* 2. If the master does not support but the replica does.
* We also need to synchronize the master's shard-id with the replica.
* 3. If neither of master and replica supports it.
* The master will have a randomly generated shard-id and will update
* the replica to match the master's shard-id. */
if (node->slaveof == NULL) {
for (int i = 0; i < clusterNodeNumSlaves(node); i++) {
clusterNode *slavenode = clusterNodeGetSlave(node, i);
if (memcmp(slavenode->shard_id, shard_id, CLUSTER_NAMELEN) != 0)
assignShardIdToNode(slavenode, shard_id, CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
}
} else {
clusterNode *masternode = node->slaveof;
if (memcmp(masternode->shard_id, shard_id, CLUSTER_NAMELEN) != 0)
assignShardIdToNode(masternode, shard_id, CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);
}
}
}
......
......@@ -24,6 +24,7 @@
/* Flags for expireIfNeeded */
#define EXPIRE_FORCE_DELETE_EXPIRED 1
#define EXPIRE_AVOID_DELETE_EXPIRED 2
#define EXPIRE_ALLOW_ACCESS_EXPIRED 4
/* Return values for expireIfNeeded */
typedef enum {
......@@ -94,6 +95,8 @@ robj *lookupKey(redisDb *db, robj *key, int flags, dictEntry **deref) {
expire_flags |= EXPIRE_FORCE_DELETE_EXPIRED;
if (flags & LOOKUP_NOEXPIRE)
expire_flags |= EXPIRE_AVOID_DELETE_EXPIRED;
if (flags & LOOKUP_ACCESS_EXPIRED)
expire_flags |= EXPIRE_ALLOW_ACCESS_EXPIRED;
if (expireIfNeeded(db, key, expire_flags) != KEY_VALID) {
/* The key is no longer valid. */
val = NULL;
......@@ -105,7 +108,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags, dictEntry **deref) {
* Don't do it if we have a saving child, as this will trigger
* a copy on write madness. */
if (server.current_client && server.current_client->flags & CLIENT_NO_TOUCH &&
server.current_client->cmd->proc != touchCommand)
server.executing_client->cmd->proc != touchCommand)
flags |= LOOKUP_NOTOUCH;
if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){
if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) {
......@@ -1223,50 +1226,95 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
}
} while (cursor && maxiterations-- && data.sampled < count);
} else if (o->type == OBJ_SET) {
unsigned long array_reply_len = 0;
void *replylen = NULL;
listRelease(keys);
char *str;
char buf[LONG_STR_SIZE];
size_t len;
int64_t llele;
/* Reply to the client. */
addReplyArrayLen(c, 2);
/* Cursor is always 0 given we iterate over all set */
addReplyBulkLongLong(c,0);
/* If there is no pattern the length is the entire set size, otherwise we defer the reply size */
if (use_pattern)
replylen = addReplyDeferredLen(c);
else {
array_reply_len = setTypeSize(o);
addReplyArrayLen(c, array_reply_len);
}
setTypeIterator *si = setTypeInitIterator(o);
unsigned long cur_length = 0;
while (setTypeNext(si, &str, &len, &llele) != -1) {
if (str == NULL) {
len = ll2string(buf, sizeof(buf), llele);
}
char *key = str ? str : buf;
if (use_pattern && !stringmatchlen(pat, sdslen(pat), key, len, 0)) {
if (use_pattern && !stringmatchlen(pat, patlen, key, len, 0)) {
continue;
}
listAddNodeTail(keys, sdsnewlen(key, len));
addReplyBulkCBuffer(c, key, len);
cur_length++;
}
setTypeReleaseIterator(si);
cursor = 0;
if (use_pattern)
setDeferredArrayLen(c,replylen,cur_length);
else
serverAssert(cur_length == array_reply_len); /* fail on corrupt data */
return;
} else if ((o->type == OBJ_HASH || o->type == OBJ_ZSET) &&
o->encoding == OBJ_ENCODING_LISTPACK)
{
unsigned char *p = lpFirst(o->ptr);
unsigned char *str;
int64_t len;
unsigned long array_reply_len = 0;
unsigned char intbuf[LP_INTBUF_SIZE];
void *replylen = NULL;
listRelease(keys);
/* Reply to the client. */
addReplyArrayLen(c, 2);
/* Cursor is always 0 given we iterate over all set */
addReplyBulkLongLong(c,0);
/* If there is no pattern the length is the entire set size, otherwise we defer the reply size */
if (use_pattern)
replylen = addReplyDeferredLen(c);
else {
array_reply_len = o->type == OBJ_HASH ? hashTypeLength(o, 0) : zsetLength(o);
if (!no_values) {
array_reply_len *= 2;
}
addReplyArrayLen(c, array_reply_len);
}
unsigned long cur_length = 0;
while(p) {
str = lpGet(p, &len, intbuf);
/* point to the value */
p = lpNext(o->ptr, p);
if (use_pattern && !stringmatchlen(pat, sdslen(pat), (char *)str, len, 0)) {
if (use_pattern && !stringmatchlen(pat, patlen, (char *)str, len, 0)) {
/* jump to the next key/val pair */
p = lpNext(o->ptr, p);
continue;
}
/* add key object */
listAddNodeTail(keys, sdsnewlen(str, len));
addReplyBulkCBuffer(c, str, len);
cur_length++;
/* add value object */
if (!no_values) {
str = lpGet(p, &len, intbuf);
listAddNodeTail(keys, sdsnewlen(str, len));
addReplyBulkCBuffer(c, str, len);
cur_length++;
}
p = lpNext(o->ptr, p);
}
cursor = 0;
if (use_pattern)
setDeferredArrayLen(c,replylen,cur_length);
else
serverAssert(cur_length == array_reply_len); /* fail on corrupt data */
return;
} else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_LISTPACK_EX) {
int64_t len;
long long expire_at;
......@@ -1274,6 +1322,16 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
unsigned char *p = lpFirst(lp);
unsigned char *str, *val;
unsigned char intbuf[LP_INTBUF_SIZE];
void *replylen = NULL;
listRelease(keys);
/* Reply to the client. */
addReplyArrayLen(c, 2);
/* Cursor is always 0 given we iterate over all set */
addReplyBulkLongLong(c,0);
/* In the case of OBJ_ENCODING_LISTPACK_EX we always defer the reply size given some fields might be expired */
replylen = addReplyDeferredLen(c);
unsigned long cur_length = 0;
while (p) {
str = lpGet(p, &len, intbuf);
......@@ -1284,7 +1342,7 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
serverAssert(p && lpGetIntegerValue(p, &expire_at));
if (hashTypeIsExpired(o, expire_at) ||
(use_pattern && !stringmatchlen(pat, sdslen(pat), (char *)str, len, 0)))
(use_pattern && !stringmatchlen(pat, patlen, (char *)str, len, 0)))
{
/* jump to the next key/val pair */
p = lpNext(lp, p);
......@@ -1292,15 +1350,18 @@ void scanGenericCommand(client *c, robj *o, unsigned long long cursor) {
}
/* add key object */
listAddNodeTail(keys, sdsnewlen(str, len));
addReplyBulkCBuffer(c, str, len);
cur_length++;
/* add value object */
if (!no_values) {
str = lpGet(val, &len, intbuf);
listAddNodeTail(keys, sdsnewlen(str, len));
addReplyBulkCBuffer(c, str, len);
cur_length++;
}
p = lpNext(lp, p);
}
cursor = 0;
setDeferredArrayLen(c,replylen,cur_length);
return;
} else {
serverPanic("Not handled encoding in SCAN.");
}
......@@ -1993,14 +2054,17 @@ int keyIsExpired(redisDb *db, robj *key) {
*
* On the other hand, if you just want expiration check, but need to avoid
* the actual key deletion and propagation of the deletion, use the
* EXPIRE_AVOID_DELETE_EXPIRED flag.
* EXPIRE_AVOID_DELETE_EXPIRED flag. If also needed to read expired key (that
* hasn't being deleted yet) then use EXPIRE_ALLOW_ACCESS_EXPIRED.
*
* The return value of the function is KEY_VALID if the key is still valid.
* The function returns KEY_EXPIRED if the key is expired BUT not deleted,
* or returns KEY_DELETED if the key is expired and deleted. */
keyStatus expireIfNeeded(redisDb *db, robj *key, int flags) {
if (server.lazy_expire_disabled) return KEY_VALID;
if (!keyIsExpired(db,key)) return KEY_VALID;
if ((!keyIsExpired(db,key)) ||
(server.lazy_expire_disabled) ||
(flags & EXPIRE_ALLOW_ACCESS_EXPIRED))
return KEY_VALID;
/* If we are running in the context of a replica, instead of
* evicting the expired key from the database, we return ASAP:
......
......@@ -15,6 +15,7 @@
#include "bio.h"
#include "quicklist.h"
#include "fpconv_dtoa.h"
#include "fast_float_strtod.h"
#include "cluster.h"
#include "threads_mngr.h"
#include "script.h"
......@@ -833,7 +834,7 @@ NULL
addReplyError(c,"Wrong protocol type name. Please use one of the following: string|integer|double|bignum|null|array|set|map|attrib|push|verbatim|true|false");
}
} else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) {
double dtime = strtod(c->argv[2]->ptr,NULL);
double dtime = fast_float_strtod(c->argv[2]->ptr,NULL);
long long utime = dtime*1000000;
struct timespec tv;
......
......@@ -1876,6 +1876,105 @@ uint64_t ebGetExpireTime(EbucketsType *type, eItem item) {
return ebGetMetaExpTime(meta);
}
/* Init ebuckets iterator
*
* This is a non-safe iterator. Any modification to ebuckets will invalidate the
* iterator. Calling this function takes care to reference the first item
* in ebuckets with minimal expiration time. If no items to iterate, then
* iter->currItem will be NULL and iter->itemsCurrBucket will be set to 0.
*/
void ebStart(EbucketsIterator *iter, ebuckets eb, EbucketsType *type) {
iter->eb = eb;
iter->type = type;
iter->isRax = 0;
if (ebIsEmpty(eb)) {
iter->currItem = NULL;
iter->itemsCurrBucket = 0;
} else if (ebIsList(eb)) {
iter->currItem = ebGetListPtr(type, eb);
iter->itemsCurrBucket = type->getExpireMeta(iter->currItem)->numItems;
} else {
rax *rax = ebGetRaxPtr(eb);
raxStart(&iter->raxIter, rax);
raxSeek(&iter->raxIter, "^", NULL, 0);
raxNext(&iter->raxIter);
FirstSegHdr *firstSegHdr = iter->raxIter.data;
iter->itemsCurrBucket = firstSegHdr->totalItems;
iter->currItem = firstSegHdr->head;
iter->isRax = 1;
}
}
/* Advance iterator to the next item
*
* Returns:
* - 0 if the end of ebuckets has been reached, setting `iter->currItem`
* to NULL.
* - 1 otherwise, updating `iter->currItem` to the next item.
*/
int ebNext(EbucketsIterator *iter) {
if (iter->currItem == NULL)
return 0;
eItem item = iter->currItem;
ExpireMeta *meta = iter->type->getExpireMeta(item);
if (iter->isRax) {
if (meta->lastItemBucket) {
if (raxNext(&iter->raxIter)) {
FirstSegHdr *firstSegHdr = iter->raxIter.data;
iter->currItem = firstSegHdr->head;
iter->itemsCurrBucket = firstSegHdr->totalItems;
} else {
iter->currItem = NULL;
}
} else if (meta->lastInSegment) {
NextSegHdr *nextSegHdr = meta->next;
iter->currItem = nextSegHdr->head;
} else {
iter->currItem = meta->next;
}
} else {
iter->currItem = meta->next;
}
if (iter->currItem == NULL) {
iter->itemsCurrBucket = 0;
return 0;
}
return 1;
}
/* Advance the iterator to the next bucket
*
* Returns:
* - 0 if no more ebuckets are available, setting `iter->currItem` to NULL
* and `iter->itemsCurrBucket` to 0.
* - 1 otherwise, updating `iter->currItem` and `iter->itemsCurrBucket` for the
* next ebucket.
*/
int ebNextBucket(EbucketsIterator *iter) {
if (iter->currItem == NULL)
return 0;
if ((iter->isRax) && (raxNext(&iter->raxIter))) {
FirstSegHdr *currSegHdr = iter->raxIter.data;
iter->currItem = currSegHdr->head;
iter->itemsCurrBucket = currSegHdr->totalItems;
} else {
iter->currItem = NULL;
iter->itemsCurrBucket = 0;
}
return 1;
}
/* Stop and cleanup the ebuckets iterator */
void ebStop(EbucketsIterator *iter) {
if (iter->isRax)
raxStop(&iter->raxIter);
}
/*** Unit tests ***/
#ifdef REDIS_TEST
......@@ -2117,6 +2216,50 @@ int ebucketsTest(int argc, char **argv, int flags) {
}
#endif
TEST("basic iterator test") {
MyItem *items[100];
for (uint32_t numItems = 0 ; numItems < ARRAY_SIZE(items) ; ++numItems) {
ebuckets eb = NULL;
EbucketsIterator iter;
/* Create and add items to ebuckets */
for (uint32_t i = 0; i < numItems; i++) {
items[i] = zmalloc(sizeof(MyItem));
ebAdd(&eb, &myEbucketsType, items[i], i);
}
/* iterate items */
ebStart(&iter, eb, &myEbucketsType);
for (uint32_t i = 0; i < numItems; i++) {
assert(iter.currItem == items[i]);
int res = ebNext(&iter);
if (i+1<numItems) {
assert(res == 1);
assert(iter.currItem != NULL);
} else {
assert(res == 0);
assert(iter.currItem == NULL);
}
}
ebStop(&iter);
/* iterate buckets */
ebStart(&iter, eb, &myEbucketsType);
uint32_t countItems = 0;
uint32_t countBuckets = 0;
while (1) {
countItems += iter.itemsCurrBucket;
if (!ebNextBucket(&iter)) break;
countBuckets++;
}
ebStop(&iter);
assert(countItems == numItems);
if (numItems>=8) assert(numItems/8 >= countBuckets);
ebDestroy(&eb, &myEbucketsType, NULL);
}
}
TEST("list - Create a single item, get TTL, and remove") {
MyItem *singleItem = zmalloc(sizeof(MyItem));
ebuckets eb = NULL;
......@@ -2146,9 +2289,8 @@ int ebucketsTest(int argc, char **argv, int flags) {
assert(ebRemove(&eb, &myEbucketsType, items[i]));
}
for (int i = 0 ; i < EB_LIST_MAX_ITEMS ; i++) {
for (int i = 0 ; i < EB_LIST_MAX_ITEMS ; i++)
zfree(items[i]);
}
ebDestroy(&eb, &myEbucketsType, NULL);
}
......
......@@ -255,6 +255,20 @@ typedef struct ExpireInfo {
EB_EXPIRE_TIME_INVALID if none left. */
} ExpireInfo;
/* Iterator to traverse ebuckets items */
typedef struct EbucketsIterator {
/* private data of iterator */
ebuckets eb;
EbucketsType *type;
raxIterator raxIter;
int isRax;
/* public read only */
eItem currItem; /* Current item ref. Use ebGetMetaExpTime()
on `currItem` to get expiration time.*/
uint64_t itemsCurrBucket; /* Number of items in current bucket. */
} EbucketsIterator;
/* ebuckets API */
static inline ebuckets ebCreate(void) { return NULL; } /* Empty ebuckets */
......@@ -281,6 +295,14 @@ int ebAdd(ebuckets *eb, EbucketsType *type, eItem item, uint64_t expireTime);
uint64_t ebGetExpireTime(EbucketsType *type, eItem item);
void ebStart(EbucketsIterator *iter, ebuckets eb, EbucketsType *type);
void ebStop(EbucketsIterator *iter);
int ebNext(EbucketsIterator *iter);
int ebNextBucket(EbucketsIterator *iter);
typedef eItem (ebDefragFunction)(const eItem item);
eItem ebDefragItem(ebuckets *eb, EbucketsType *type, eItem item, ebDefragFunction *fn);
......
......@@ -4070,7 +4070,8 @@ static void moduleInitKeyTypeSpecific(RedisModuleKey *key) {
* * REDISMODULE_OPEN_KEY_NONOTIFY - Don't trigger keyspace event on key misses.
* * REDISMODULE_OPEN_KEY_NOSTATS - Don't update keyspace hits/misses counters.
* * REDISMODULE_OPEN_KEY_NOEXPIRE - Avoid deleting lazy expired keys.
* * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key. */
* * REDISMODULE_OPEN_KEY_NOEFFECTS - Avoid any effects from fetching the key.
* * REDISMODULE_OPEN_KEY_ACCESS_EXPIRED - Access expired keys that have not yet been deleted */
RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
RedisModuleKey *kp;
robj *value;
......@@ -4080,6 +4081,7 @@ RedisModuleKey *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
flags |= (mode & REDISMODULE_OPEN_KEY_NOSTATS? LOOKUP_NOSTATS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEXPIRE? LOOKUP_NOEXPIRE: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_NOEFFECTS? LOOKUP_NOEFFECTS: 0);
flags |= (mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED ? (LOOKUP_ACCESS_EXPIRED) : 0);
 
if (mode & REDISMODULE_WRITE) {
value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags);
......@@ -5378,6 +5380,9 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
va_list ap;
if (key->value && key->value->type != OBJ_HASH) return REDISMODULE_ERR;
 
if (key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)
hfeFlags = HFE_LAZY_ACCESS_EXPIRED; /* allow read also expired fields */
va_start(ap, flags);
while(1) {
RedisModuleString *field, **valueptr;
......@@ -11087,8 +11092,9 @@ static void moduleScanKeyCallback(void *privdata, const dictEntry *de) {
} else if (o->type == OBJ_HASH) {
sds val = dictGetVal(de);
 
/* If field is expired, then ignore */
if (hfieldIsExpired(key))
/* If field is expired and not indicated to access expired, then ignore */
if ((!(data->key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hfieldIsExpired(key)))
return;
 
field = createStringObject(key, hfieldlen(key));
......@@ -11224,7 +11230,8 @@ int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleSc
p = lpNext(lp, p);
 
/* Skip expired fields */
if (hashTypeIsExpired(o, vllExpire))
if ((!(key->mode & REDISMODULE_OPEN_KEY_ACCESS_EXPIRED)) &&
(hashTypeIsExpired(o, vllExpire)))
continue;
}
 
......
......@@ -931,6 +931,33 @@ void addReplyHumanLongDouble(client *c, long double d) {
}
}
static inline void _addReplyLongLongSharedHdr(client *c, long long ll, char prefix,
robj *shared_hdr[OBJ_SHARED_BULKHDR_LEN])
{
char buf[128];
int len;
const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0;
if (opt_hdr) {
_addReplyToBufferOrList(c, shared_hdr[ll]->ptr, OBJ_SHARED_HDR_STRLEN(ll));
return;
}
buf[0] = prefix;
len = ll2string(buf + 1, sizeof(buf) - 1, ll);
buf[len + 1] = '\r';
buf[len + 2] = '\n';
_addReplyToBufferOrList(c, buf, len + 3);
}
static inline void _addReplyLongLongBulk(client *c, long long ll) {
_addReplyLongLongSharedHdr(c, ll, '$', shared.bulkhdr);
}
static inline void _addReplyLongLongMBulk(client *c, long long ll) {
_addReplyLongLongSharedHdr(c, ll, '*', shared.mbulkhdr);
}
/* Add a long long as integer reply or bulk len / multi bulk count.
* Basically this is used to output <prefix><long long><crlf>. */
static void _addReplyLongLongWithPrefix(client *c, long long ll, char prefix) {
......@@ -987,7 +1014,9 @@ void addReplyAggregateLen(client *c, long length, int prefix) {
}
void addReplyArrayLen(client *c, long length) {
addReplyAggregateLen(c,length,'*');
serverAssert(length >= 0);
if (prepareClientToWrite(c) != C_OK) return;
_addReplyLongLongMBulk(c, length);
}
void addReplyMapLen(client *c, long length) {
......@@ -1044,7 +1073,7 @@ void addReplyNullArray(client *c) {
void addReplyBulkLen(client *c, robj *obj) {
size_t len = stringObjectLen(obj);
if (prepareClientToWrite(c) != C_OK) return;
_addReplyLongLongWithPrefix(c, len, '$');
_addReplyLongLongBulk(c, len);
}
/* Add a Redis Object as a bulk reply */
......@@ -1057,7 +1086,7 @@ void addReplyBulk(client *c, robj *obj) {
/* Add a C buffer as bulk reply */
void addReplyBulkCBuffer(client *c, const void *p, size_t len) {
if (prepareClientToWrite(c) != C_OK) return;
_addReplyLongLongWithPrefix(c, len, '$');
_addReplyLongLongBulk(c, len);
_addReplyToBufferOrList(c, p, len);
_addReplyToBufferOrList(c, "\r\n", 2);
}
......
......@@ -3695,7 +3695,7 @@ static int evalMode(int argc, char **argv) {
int eval_ldb = config.eval_ldb; /* Save it, may be reverted. */
retval = issueCommand(argc+3-got_comma, argv2);
for (j = 0; j < argc+3-got_comma; j++) sdsfree(argv2[j]);
free(argv2);
zfree(argv2);
if (eval_ldb) {
if (!config.eval_ldb) {
/* If the debugging session ended immediately, there was an
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment