Commit 72935b9d authored by Vitaly Arbuzov's avatar Vitaly Arbuzov
Browse files

Merge branch 'unstable' into dict-split-by-slot

parents 6baf20af 6948daca
codespell==2.2.2 codespell==2.2.4
...@@ -75,3 +75,4 @@ jobs: ...@@ -75,3 +75,4 @@ jobs:
run: | run: |
yum -y install gcc make yum -y install gcc make
make REDIS_CFLAGS='-Werror' make REDIS_CFLAGS='-Werror'
...@@ -11,7 +11,7 @@ on: ...@@ -11,7 +11,7 @@ on:
inputs: inputs:
skipjobs: skipjobs:
description: 'jobs to skip (delete the ones you wanna keep, do not leave empty)' description: 'jobs to skip (delete the ones you wanna keep, do not leave empty)'
default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,centos,malloc,specific' default: 'valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,centos,malloc,specific,reply-schema'
skiptests: skiptests:
description: 'tests to skip (delete the ones you wanna keep, do not leave empty)' description: 'tests to skip (delete the ones you wanna keep, do not leave empty)'
default: 'redis,modules,sentinel,cluster,unittest' default: 'redis,modules,sentinel,cluster,unittest'
...@@ -898,3 +898,44 @@ jobs: ...@@ -898,3 +898,44 @@ jobs:
- name: cluster tests - name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster') if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}} run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
reply-schemas-validator:
runs-on: ubuntu-latest
timeout-minutes: 14400
if: |
(github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
!contains(github.event.inputs.skipjobs, 'reply-schema')
steps:
- name: prep
if: github.event_name == 'workflow_dispatch'
run: |
echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
echo "skipping: ${{github.event.inputs.skipjobs}} and ${{github.event.inputs.skiptests}}"
- uses: actions/checkout@v3
with:
repository: ${{ env.GITHUB_REPOSITORY }}
ref: ${{ env.GITHUB_HEAD_REF }}
- name: make
run: make REDIS_CFLAGS='-Werror -DLOG_REQ_RES'
- name: testprep
run: sudo apt-get install tcl8.6 tclx
- name: test
if: true && !contains(github.event.inputs.skiptests, 'redis')
run: ./runtest --log-req-res --dont-clean --force-resp3 --tags -slow --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: module api test
if: true && !contains(github.event.inputs.skiptests, 'modules')
run: ./runtest-moduleapi --log-req-res --dont-clean --force-resp3 --dont-pre-clean --verbose --dump-logs ${{github.event.inputs.test_args}}
- name: sentinel tests
if: true && !contains(github.event.inputs.skiptests, 'sentinel')
run: ./runtest-sentinel --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}}
- name: cluster tests
if: true && !contains(github.event.inputs.skiptests, 'cluster')
run: ./runtest-cluster --log-req-res --dont-clean --force-resp3 ${{github.event.inputs.cluster_test_args}}
- name: Install Python dependencies
uses: py-actions/py-dependency-install@v4
with:
path: "./utils/req-res-validator/requirements.txt"
- name: validator
run: ./utils/req-res-log-validator.py --verbose --fail-missing-reply-schemas --fail-commands-not-all-hit
name: Reply-schemas linter
on:
push:
paths:
- 'src/commands/*.json'
pull_request:
paths:
- 'src/commands/*.json'
jobs:
reply-schemas-linter:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup nodejs
uses: actions/setup-node@v3
- name: Install packages
run: npm install ajv
- name: linter
run: node ./utils/reply_schema_linter.js
...@@ -163,6 +163,7 @@ enum KEY_ACTION{ ...@@ -163,6 +163,7 @@ enum KEY_ACTION{
CTRL_F = 6, /* Ctrl-f */ CTRL_F = 6, /* Ctrl-f */
CTRL_H = 8, /* Ctrl-h */ CTRL_H = 8, /* Ctrl-h */
TAB = 9, /* Tab */ TAB = 9, /* Tab */
NL = 10, /* Enter typed before raw mode was enabled */
CTRL_K = 11, /* Ctrl+k */ CTRL_K = 11, /* Ctrl+k */
CTRL_L = 12, /* Ctrl+l */ CTRL_L = 12, /* Ctrl+l */
ENTER = 13, /* Enter */ ENTER = 13, /* Enter */
...@@ -256,8 +257,8 @@ static int enableRawMode(int fd) { ...@@ -256,8 +257,8 @@ static int enableRawMode(int fd) {
* We want read to return every single byte, without timeout. */ * We want read to return every single byte, without timeout. */
raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ raw.c_cc[VMIN] = 1; raw.c_cc[VTIME] = 0; /* 1 byte, no timer */
/* put terminal in raw mode after flushing */ /* put terminal in raw mode */
if (tcsetattr(fd,TCSAFLUSH,&raw) < 0) goto fatal; if (tcsetattr(fd,TCSANOW,&raw) < 0) goto fatal;
rawmode = 1; rawmode = 1;
return 0; return 0;
...@@ -268,7 +269,7 @@ fatal: ...@@ -268,7 +269,7 @@ fatal:
static void disableRawMode(int fd) { static void disableRawMode(int fd) {
/* Don't even check the return value as it's too late. */ /* Don't even check the return value as it's too late. */
if (rawmode && tcsetattr(fd,TCSAFLUSH,&orig_termios) != -1) if (rawmode && tcsetattr(fd,TCSANOW,&orig_termios) != -1)
rawmode = 0; rawmode = 0;
} }
...@@ -840,6 +841,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, ...@@ -840,6 +841,8 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen,
} }
switch(c) { switch(c) {
case NL: /* enter, typed before raw mode was enabled */
break;
case ENTER: /* enter */ case ENTER: /* enter */
history_len--; history_len--;
free(history[history_len]); free(history[history_len]);
......
...@@ -1972,7 +1972,7 @@ zset-max-listpack-entries 128 ...@@ -1972,7 +1972,7 @@ zset-max-listpack-entries 128
zset-max-listpack-value 64 zset-max-listpack-value 64
# HyperLogLog sparse representation bytes limit. The limit includes the # HyperLogLog sparse representation bytes limit. The limit includes the
# 16 bytes header. When an HyperLogLog using the sparse representation crosses # 16 bytes header. When a HyperLogLog using the sparse representation crosses
# this limit, it is converted into the dense representation. # this limit, it is converted into the dense representation.
# #
# A value greater than 16000 is totally useless, since at that point the # A value greater than 16000 is totally useless, since at that point the
......
...@@ -52,4 +52,6 @@ $TCLSH tests/test_helper.tcl \ ...@@ -52,4 +52,6 @@ $TCLSH tests/test_helper.tcl \
--single unit/moduleapi/publish \ --single unit/moduleapi/publish \
--single unit/moduleapi/usercall \ --single unit/moduleapi/usercall \
--single unit/moduleapi/postnotifications \ --single unit/moduleapi/postnotifications \
--single unit/moduleapi/async_rm_call \
--single unit/moduleapi/moduleauth \
"${@}" "${@}"
...@@ -17,7 +17,7 @@ uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') ...@@ -17,7 +17,7 @@ uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
OPTIMIZATION?=-O3 OPTIMIZATION?=-O3
ifeq ($(OPTIMIZATION),-O3) ifeq ($(OPTIMIZATION),-O3)
REDIS_CFLAGS+=-flto REDIS_CFLAGS+=-flto=auto
REDIS_LDFLAGS+=-flto REDIS_LDFLAGS+=-flto
endif endif
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram fpconv
...@@ -330,9 +330,17 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN ...@@ -330,9 +330,17 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN
QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2;
endif endif
ifneq (, $(findstring LOG_REQ_RES, $(REDIS_CFLAGS)))
COMMANDS_FILENAME=commands_with_reply_schema
GEN_COMMANDS_FLAGS=--with-reply-schema
else
COMMANDS_FILENAME=commands
GEN_COMMANDS_FLAGS=
endif
REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX) REDIS_SERVER_NAME=redis-server$(PROG_SUFFIX)
REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX) REDIS_SENTINEL_NAME=redis-sentinel$(PROG_SUFFIX)
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o $(COMMANDS_FILENAME).o strl.o connection.o unix.o logreqres.o
REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX) REDIS_CLI_NAME=redis-cli$(PROG_SUFFIX)
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o ae.o redisassert.o crcspeed.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o
REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX) REDIS_BENCHMARK_NAME=redis-benchmark$(PROG_SUFFIX)
...@@ -425,8 +433,8 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ ...@@ -425,8 +433,8 @@ DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ
# The file commands.c is checked in and doesn't normally need to be rebuilt. It # The file commands.c is checked in and doesn't normally need to be rebuilt. It
# is built only if python is available and its prereqs are modified. # is built only if python is available and its prereqs are modified.
ifneq (,$(PYTHON)) ifneq (,$(PYTHON))
commands.c: commands/*.json ../utils/generate-command-code.py $(COMMANDS_FILENAME).c: commands/*.json ../utils/generate-command-code.py
$(QUIET_GEN)$(PYTHON) ../utils/generate-command-code.py $(QUIET_GEN)$(PYTHON) ../utils/generate-command-code.py $(GEN_COMMANDS_FLAGS)
endif endif
clean: clean:
......
...@@ -629,7 +629,6 @@ void ACLSetSelectorCommandBitsForCategory(dict *commands, aclSelector *selector, ...@@ -629,7 +629,6 @@ void ACLSetSelectorCommandBitsForCategory(dict *commands, aclSelector *selector,
dictEntry *de; dictEntry *de;
while ((de = dictNext(di)) != NULL) { while ((de = dictNext(di)) != NULL) {
struct redisCommand *cmd = dictGetVal(de); struct redisCommand *cmd = dictGetVal(de);
if (cmd->flags & CMD_MODULE) continue; /* Ignore modules commands. */
if (cmd->acl_categories & cflag) { if (cmd->acl_categories & cflag) {
ACLChangeSelectorPerm(selector,cmd,value); ACLChangeSelectorPerm(selector,cmd,value);
} }
...@@ -640,6 +639,44 @@ void ACLSetSelectorCommandBitsForCategory(dict *commands, aclSelector *selector, ...@@ -640,6 +639,44 @@ void ACLSetSelectorCommandBitsForCategory(dict *commands, aclSelector *selector,
dictReleaseIterator(di); dictReleaseIterator(di);
} }
/* This function is responsible for recomputing the command bits for all selectors of the existing users.
* It uses the 'command_rules', a string representation of the ordered categories and commands,
* to recompute the command bits. */
void ACLRecomputeCommandBitsFromCommandRulesAllUsers() {
raxIterator ri;
raxStart(&ri,Users);
raxSeek(&ri,"^",NULL,0);
while(raxNext(&ri)) {
user *u = ri.data;
listIter li;
listNode *ln;
listRewind(u->selectors,&li);
while((ln = listNext(&li))) {
aclSelector *selector = (aclSelector *) listNodeValue(ln);
int argc = 0;
sds *argv = sdssplitargs(selector->command_rules, &argc);
serverAssert(argv != NULL);
/* Checking selector's permissions for all commands to start with a clean state. */
if (ACLSelectorCanExecuteFutureCommands(selector)) {
int res = ACLSetSelector(selector,"+@all",-1);
serverAssert(res == C_OK);
} else {
int res = ACLSetSelector(selector,"-@all",-1);
serverAssert(res == C_OK);
}
/* Apply all of the commands and categories to this selector. */
for(int i = 0; i < argc; i++) {
int res = ACLSetSelector(selector, argv[i], sdslen(argv[i]));
serverAssert(res == C_OK);
}
sdsfreesplitres(argv, argc);
}
}
raxStop(&ri);
}
int ACLSetSelectorCategory(aclSelector *selector, const char *category, int allow) { int ACLSetSelectorCategory(aclSelector *selector, const char *category, int allow) {
uint64_t cflag = ACLGetCommandCategoryFlagByName(category + 1); uint64_t cflag = ACLGetCommandCategoryFlagByName(category + 1);
if (!cflag) return C_ERR; if (!cflag) return C_ERR;
...@@ -1406,24 +1443,50 @@ int ACLCheckUserCredentials(robj *username, robj *password) { ...@@ -1406,24 +1443,50 @@ int ACLCheckUserCredentials(robj *username, robj *password) {
return C_ERR; return C_ERR;
} }
/* If `err` is provided, this is added as an error reply to the client.
* Otherwise, the standard Auth error is added as a reply. */
void addAuthErrReply(client *c, robj *err) {
if (clientHasPendingReplies(c)) return;
if (!err) {
addReplyError(c, "-WRONGPASS invalid username-password pair or user is disabled.");
return;
}
addReplyError(c, err->ptr);
}
/* This is like ACLCheckUserCredentials(), however if the user/pass /* This is like ACLCheckUserCredentials(), however if the user/pass
* are correct, the connection is put in authenticated state and the * are correct, the connection is put in authenticated state and the
* connection user reference is populated. * connection user reference is populated.
* *
* The return value is C_OK or C_ERR with the same meaning as * The return value is AUTH_OK on success (valid username / password pair) & AUTH_ERR otherwise. */
* ACLCheckUserCredentials(). */ int checkPasswordBasedAuth(client *c, robj *username, robj *password) {
int ACLAuthenticateUser(client *c, robj *username, robj *password) {
if (ACLCheckUserCredentials(username,password) == C_OK) { if (ACLCheckUserCredentials(username,password) == C_OK) {
c->authenticated = 1; c->authenticated = 1;
c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr)); c->user = ACLGetUserByName(username->ptr,sdslen(username->ptr));
moduleNotifyUserChanged(c); moduleNotifyUserChanged(c);
return C_OK; return AUTH_OK;
} else { } else {
addACLLogEntry(c,ACL_DENIED_AUTH,(c->flags & CLIENT_MULTI) ? ACL_LOG_CTX_MULTI : ACL_LOG_CTX_TOPLEVEL,0,username->ptr,NULL); addACLLogEntry(c,ACL_DENIED_AUTH,(c->flags & CLIENT_MULTI) ? ACL_LOG_CTX_MULTI : ACL_LOG_CTX_TOPLEVEL,0,username->ptr,NULL);
return C_ERR; return AUTH_ERR;
} }
} }
/* Attempt authenticating the user - first through module based authentication,
* and then, if needed, with normal password based authentication.
* Returns one of the following codes:
* AUTH_OK - Indicates that authentication succeeded.
* AUTH_ERR - Indicates that authentication failed.
* AUTH_BLOCKED - Indicates module authentication is in progress through a blocking implementation.
*/
int ACLAuthenticateUser(client *c, robj *username, robj *password, robj **err) {
int result = checkModuleAuthentication(c, username, password, err);
/* If authentication was not handled by any Module, attempt normal password based auth. */
if (result == AUTH_NOT_HANDLED) {
result = checkPasswordBasedAuth(c, username, password);
}
return result;
}
/* For ACL purposes, every user has a bitmap with the commands that such /* For ACL purposes, every user has a bitmap with the commands that such
* user is allowed to execute. In order to populate the bitmap, every command * user is allowed to execute. In order to populate the bitmap, every command
* should have an assigned ID (that is used to index the bitmap). This function * should have an assigned ID (that is used to index the bitmap). This function
...@@ -3046,11 +3109,14 @@ void authCommand(client *c) { ...@@ -3046,11 +3109,14 @@ void authCommand(client *c) {
redactClientCommandArgument(c, 2); redactClientCommandArgument(c, 2);
} }
if (ACLAuthenticateUser(c,username,password) == C_OK) { robj *err = NULL;
addReply(c,shared.ok); int result = ACLAuthenticateUser(c, username, password, &err);
} else { if (result == AUTH_OK) {
addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); addReply(c, shared.ok);
} else if (result == AUTH_ERR) {
addAuthErrReply(c, err);
} }
if (err) decrRefCount(err);
} }
/* Set the password for the "default" ACL user. This implements supports for /* Set the password for the "default" ACL user. This implements supports for
......
...@@ -920,12 +920,12 @@ int aofFsyncInProgress(void) { ...@@ -920,12 +920,12 @@ int aofFsyncInProgress(void) {
/* Starts a background task that performs fsync() against the specified /* Starts a background task that performs fsync() against the specified
* file descriptor (the one of the AOF file) in another thread. */ * file descriptor (the one of the AOF file) in another thread. */
void aof_background_fsync(int fd) { void aof_background_fsync(int fd) {
bioCreateFsyncJob(fd); bioCreateFsyncJob(fd, server.master_repl_offset, 1);
} }
/* Close the fd on the basis of aof_background_fsync. */ /* Close the fd on the basis of aof_background_fsync. */
void aof_background_fsync_and_close(int fd) { void aof_background_fsync_and_close(int fd) {
bioCreateCloseJob(fd, 1, 1); bioCreateCloseAofJob(fd, server.master_repl_offset, 1);
} }
/* Kills an AOFRW child process if exists */ /* Kills an AOFRW child process if exists */
...@@ -962,6 +962,8 @@ void stopAppendOnly(void) { ...@@ -962,6 +962,8 @@ void stopAppendOnly(void) {
server.aof_state = AOF_OFF; server.aof_state = AOF_OFF;
server.aof_rewrite_scheduled = 0; server.aof_rewrite_scheduled = 0;
server.aof_last_incr_size = 0; server.aof_last_incr_size = 0;
server.fsynced_reploff = -1;
atomicSet(server.fsynced_reploff_pending, 0);
killAppendOnlyChild(); killAppendOnlyChild();
sdsfree(server.aof_buf); sdsfree(server.aof_buf);
server.aof_buf = sdsempty(); server.aof_buf = sdsempty();
...@@ -972,6 +974,18 @@ void stopAppendOnly(void) { ...@@ -972,6 +974,18 @@ void stopAppendOnly(void) {
int startAppendOnly(void) { int startAppendOnly(void) {
serverAssert(server.aof_state == AOF_OFF); serverAssert(server.aof_state == AOF_OFF);
/* Wait for all bio jobs related to AOF to drain. This prevents a race
* between updates to `fsynced_reploff_pending` of the worker thread, belonging
* to the previous AOF, and the new one. This concern is specific for a full
* sync scenario where we don't wanna risk the ACKed replication offset
* jumping backwards or forward when switching to a different master. */
bioDrainWorker(BIO_AOF_FSYNC);
/* Set the initial repl_offset, which will be applied to fsynced_reploff
* when AOFRW finishes (after possibly being updated by a bio thread) */
atomicSet(server.fsynced_reploff_pending, server.master_repl_offset);
server.fsynced_reploff = 0;
server.aof_state = AOF_WAIT_REWRITE; server.aof_state = AOF_WAIT_REWRITE;
if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) { if (hasActiveChildProcess() && server.child_type != CHILD_TYPE_AOF) {
server.aof_rewrite_scheduled = 1; server.aof_rewrite_scheduled = 1;
...@@ -1241,6 +1255,7 @@ try_fsync: ...@@ -1241,6 +1255,7 @@ try_fsync:
latencyAddSampleIfNeeded("aof-fsync-always",latency); latencyAddSampleIfNeeded("aof-fsync-always",latency);
server.aof_fsync_offset = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size;
server.aof_last_fsync = server.unixtime; server.aof_last_fsync = server.unixtime;
atomicSet(server.fsynced_reploff_pending, server.master_repl_offset);
} else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC && } else if ((server.aof_fsync == AOF_FSYNC_EVERYSEC &&
server.unixtime > server.aof_last_fsync)) { server.unixtime > server.aof_last_fsync)) {
if (!sync_in_progress) { if (!sync_in_progress) {
...@@ -2671,9 +2686,17 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { ...@@ -2671,9 +2686,17 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
serverLog(LL_NOTICE, "Background AOF rewrite finished successfully"); serverLog(LL_NOTICE, "Background AOF rewrite finished successfully");
/* Change state from WAIT_REWRITE to ON if needed */ /* Change state from WAIT_REWRITE to ON if needed */
if (server.aof_state == AOF_WAIT_REWRITE) if (server.aof_state == AOF_WAIT_REWRITE) {
server.aof_state = AOF_ON; server.aof_state = AOF_ON;
/* Update the fsynced replication offset that just now become valid.
* This could either be the one we took in startAppendOnly, or a
* newer one set by the bio thread. */
long long fsynced_reploff_pending;
atomicGet(server.fsynced_reploff_pending, fsynced_reploff_pending);
server.fsynced_reploff = fsynced_reploff_pending;
}
serverLog(LL_VERBOSE, serverLog(LL_VERBOSE,
"Background AOF rewrite signal handler took %lldus", ustime()-now); "Background AOF rewrite signal handler took %lldus", ustime()-now);
} else if (!bysignal && exitcode != 0) { } else if (!bysignal && exitcode != 0) {
......
...@@ -15,13 +15,15 @@ ...@@ -15,13 +15,15 @@
* DESIGN * DESIGN
* ------ * ------
* *
* The design is trivial, we have a structure representing a job to perform * The design is simple: We have a structure representing a job to perform,
* and a different thread and job queue for every job type. * and several worker threads and job queues. Every job type is assigned to
* Every thread waits for new jobs in its queue, and process every job * a specific worker thread, and a single worker may handle several different
* job types.
* Every thread waits for new jobs in its queue, and processes every job
* sequentially. * sequentially.
* *
* Jobs of the same type are guaranteed to be processed from the least * Jobs handled by the same worker are guaranteed to be processed from the
* recently inserted to the most recently inserted (older jobs processed * least-recently-inserted to the most-recently-inserted (older jobs processed
* first). * first).
* *
* Currently there is no way for the creator of the job to be notified about * Currently there is no way for the creator of the job to be notified about
...@@ -61,17 +63,39 @@ ...@@ -61,17 +63,39 @@
#include "server.h" #include "server.h"
#include "bio.h" #include "bio.h"
static pthread_t bio_threads[BIO_NUM_OPS]; static char* bio_worker_title[] = {
static pthread_mutex_t bio_mutex[BIO_NUM_OPS]; "bio_close_file",
static pthread_cond_t bio_newjob_cond[BIO_NUM_OPS]; "bio_aof",
static list *bio_jobs[BIO_NUM_OPS]; "bio_lazy_free",
};
#define BIO_WORKER_NUM (sizeof(bio_worker_title) / sizeof(*bio_worker_title))
static unsigned int bio_job_to_worker[] = {
[BIO_CLOSE_FILE] = 0,
[BIO_AOF_FSYNC] = 1,
[BIO_CLOSE_AOF] = 1,
[BIO_LAZY_FREE] = 2,
};
static pthread_t bio_threads[BIO_WORKER_NUM];
static pthread_mutex_t bio_mutex[BIO_WORKER_NUM];
static pthread_cond_t bio_newjob_cond[BIO_WORKER_NUM];
static list *bio_jobs[BIO_WORKER_NUM];
static unsigned long bio_jobs_counter[BIO_NUM_OPS] = {0};
/* This structure represents a background Job. It is only used locally to this /* This structure represents a background Job. It is only used locally to this
* file as the API does not expose the internals at all. */ * file as the API does not expose the internals at all. */
typedef union bio_job { typedef union bio_job {
struct {
int type; /* Job-type tag. This needs to appear as the first element in all union members. */
} header;
/* Job specific arguments.*/ /* Job specific arguments.*/
struct { struct {
int type;
int fd; /* Fd for file based background jobs */ int fd; /* Fd for file based background jobs */
long long offset; /* A job-specific offset, if applicable */
unsigned need_fsync:1; /* A flag to indicate that a fsync is required before unsigned need_fsync:1; /* A flag to indicate that a fsync is required before
* the file is closed. */ * the file is closed. */
unsigned need_reclaim_cache:1; /* A flag to indicate that reclaim cache is required before unsigned need_reclaim_cache:1; /* A flag to indicate that reclaim cache is required before
...@@ -79,6 +103,7 @@ typedef union bio_job { ...@@ -79,6 +103,7 @@ typedef union bio_job {
} fd_args; } fd_args;
struct { struct {
int type;
lazy_free_fn *free_fn; /* Function that will free the provided arguments */ lazy_free_fn *free_fn; /* Function that will free the provided arguments */
void *free_args[]; /* List of arguments to be passed to the free function */ void *free_args[]; /* List of arguments to be passed to the free function */
} free_args; } free_args;
...@@ -95,10 +120,10 @@ void bioInit(void) { ...@@ -95,10 +120,10 @@ void bioInit(void) {
pthread_attr_t attr; pthread_attr_t attr;
pthread_t thread; pthread_t thread;
size_t stacksize; size_t stacksize;
int j; unsigned long j;
/* Initialization of state vars and objects */ /* Initialization of state vars and objects */
for (j = 0; j < BIO_NUM_OPS; j++) { for (j = 0; j < BIO_WORKER_NUM; j++) {
pthread_mutex_init(&bio_mutex[j],NULL); pthread_mutex_init(&bio_mutex[j],NULL);
pthread_cond_init(&bio_newjob_cond[j],NULL); pthread_cond_init(&bio_newjob_cond[j],NULL);
bio_jobs[j] = listCreate(); bio_jobs[j] = listCreate();
...@@ -113,8 +138,8 @@ void bioInit(void) { ...@@ -113,8 +138,8 @@ void bioInit(void) {
/* Ready to spawn our threads. We use the single argument the thread /* Ready to spawn our threads. We use the single argument the thread
* function accepts in order to pass the job ID the thread is * function accepts in order to pass the job ID the thread is
* responsible of. */ * responsible for. */
for (j = 0; j < BIO_NUM_OPS; j++) { for (j = 0; j < BIO_WORKER_NUM; j++) {
void *arg = (void*)(unsigned long) j; void *arg = (void*)(unsigned long) j;
if (pthread_create(&thread,&attr,bioProcessBackgroundJobs,arg) != 0) { if (pthread_create(&thread,&attr,bioProcessBackgroundJobs,arg) != 0) {
serverLog(LL_WARNING,"Fatal: Can't initialize Background Jobs."); serverLog(LL_WARNING,"Fatal: Can't initialize Background Jobs.");
...@@ -125,10 +150,13 @@ void bioInit(void) { ...@@ -125,10 +150,13 @@ void bioInit(void) {
} }
void bioSubmitJob(int type, bio_job *job) { void bioSubmitJob(int type, bio_job *job) {
pthread_mutex_lock(&bio_mutex[type]); job->header.type = type;
listAddNodeTail(bio_jobs[type],job); unsigned long worker = bio_job_to_worker[type];
pthread_cond_signal(&bio_newjob_cond[type]); pthread_mutex_lock(&bio_mutex[worker]);
pthread_mutex_unlock(&bio_mutex[type]); listAddNodeTail(bio_jobs[worker],job);
bio_jobs_counter[type]++;
pthread_cond_signal(&bio_newjob_cond[worker]);
pthread_mutex_unlock(&bio_mutex[worker]);
} }
void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) { void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...) {
...@@ -155,42 +183,40 @@ void bioCreateCloseJob(int fd, int need_fsync, int need_reclaim_cache) { ...@@ -155,42 +183,40 @@ void bioCreateCloseJob(int fd, int need_fsync, int need_reclaim_cache) {
bioSubmitJob(BIO_CLOSE_FILE, job); bioSubmitJob(BIO_CLOSE_FILE, job);
} }
void bioCreateFsyncJob(int fd) { void bioCreateCloseAofJob(int fd, long long offset, int need_reclaim_cache) {
bio_job *job = zmalloc(sizeof(*job)); bio_job *job = zmalloc(sizeof(*job));
job->fd_args.fd = fd; job->fd_args.fd = fd;
job->fd_args.offset = offset;
job->fd_args.need_fsync = 1;
job->fd_args.need_reclaim_cache = need_reclaim_cache;
bioSubmitJob(BIO_CLOSE_AOF, job);
}
void bioCreateFsyncJob(int fd, long long offset, int need_reclaim_cache) {
bio_job *job = zmalloc(sizeof(*job));
job->fd_args.fd = fd;
job->fd_args.offset = offset;
job->fd_args.need_reclaim_cache = need_reclaim_cache;
bioSubmitJob(BIO_AOF_FSYNC, job); bioSubmitJob(BIO_AOF_FSYNC, job);
} }
void *bioProcessBackgroundJobs(void *arg) { void *bioProcessBackgroundJobs(void *arg) {
bio_job *job; bio_job *job;
unsigned long type = (unsigned long) arg; unsigned long worker = (unsigned long) arg;
sigset_t sigset; sigset_t sigset;
/* Check that the type is within the right interval. */ /* Check that the worker is within the right interval. */
if (type >= BIO_NUM_OPS) { serverAssert(worker < BIO_WORKER_NUM);
serverLog(LL_WARNING,
"Warning: bio thread started with wrong type %lu",type);
return NULL;
}
switch (type) { redis_set_thread_title(bio_worker_title[worker]);
case BIO_CLOSE_FILE:
redis_set_thread_title("bio_close_file");
break;
case BIO_AOF_FSYNC:
redis_set_thread_title("bio_aof_fsync");
break;
case BIO_LAZY_FREE:
redis_set_thread_title("bio_lazy_free");
break;
}
redisSetCpuAffinity(server.bio_cpulist); redisSetCpuAffinity(server.bio_cpulist);
makeThreadKillable(); makeThreadKillable();
pthread_mutex_lock(&bio_mutex[type]); pthread_mutex_lock(&bio_mutex[worker]);
/* Block SIGALRM so we are sure that only the main thread will /* Block SIGALRM so we are sure that only the main thread will
* receive the watchdog signal. */ * receive the watchdog signal. */
sigemptyset(&sigset); sigemptyset(&sigset);
...@@ -203,21 +229,26 @@ void *bioProcessBackgroundJobs(void *arg) { ...@@ -203,21 +229,26 @@ void *bioProcessBackgroundJobs(void *arg) {
listNode *ln; listNode *ln;
/* The loop always starts with the lock hold. */ /* The loop always starts with the lock hold. */
if (listLength(bio_jobs[type]) == 0) { if (listLength(bio_jobs[worker]) == 0) {
pthread_cond_wait(&bio_newjob_cond[type],&bio_mutex[type]); pthread_cond_wait(&bio_newjob_cond[worker], &bio_mutex[worker]);
continue; continue;
} }
/* Pop the job from the queue. */ /* Get the job from the queue. */
ln = listFirst(bio_jobs[type]); ln = listFirst(bio_jobs[worker]);
job = ln->value; job = ln->value;
/* It is now possible to unlock the background system as we know have /* It is now possible to unlock the background system as we know have
* a stand alone job structure to process.*/ * a stand alone job structure to process.*/
pthread_mutex_unlock(&bio_mutex[type]); pthread_mutex_unlock(&bio_mutex[worker]);
/* Process the job accordingly to its type. */ /* Process the job accordingly to its type. */
if (type == BIO_CLOSE_FILE) { int job_type = job->header.type;
if (job->fd_args.need_fsync) {
redis_fsync(job->fd_args.fd); if (job_type == BIO_CLOSE_FILE) {
if (job->fd_args.need_fsync &&
redis_fsync(job->fd_args.fd) == -1 &&
errno != EBADF && errno != EINVAL)
{
serverLog(LL_WARNING, "Fail to fsync the AOF file: %s",strerror(errno));
} }
if (job->fd_args.need_reclaim_cache) { if (job->fd_args.need_reclaim_cache) {
if (reclaimFilePageCache(job->fd_args.fd, 0, 0) == -1) { if (reclaimFilePageCache(job->fd_args.fd, 0, 0) == -1) {
...@@ -225,7 +256,7 @@ void *bioProcessBackgroundJobs(void *arg) { ...@@ -225,7 +256,7 @@ void *bioProcessBackgroundJobs(void *arg) {
} }
} }
close(job->fd_args.fd); close(job->fd_args.fd);
} else if (type == BIO_AOF_FSYNC) { } else if (job_type == BIO_AOF_FSYNC || job_type == BIO_CLOSE_AOF) {
/* The fd may be closed by main thread and reused for another /* The fd may be closed by main thread and reused for another
* socket, pipe, or file. We just ignore these errno because * socket, pipe, or file. We just ignore these errno because
* aof fsync did not really fail. */ * aof fsync did not really fail. */
...@@ -242,8 +273,17 @@ void *bioProcessBackgroundJobs(void *arg) { ...@@ -242,8 +273,17 @@ void *bioProcessBackgroundJobs(void *arg) {
} }
} else { } else {
atomicSet(server.aof_bio_fsync_status,C_OK); atomicSet(server.aof_bio_fsync_status,C_OK);
atomicSet(server.fsynced_reploff_pending, job->fd_args.offset);
} }
} else if (type == BIO_LAZY_FREE) {
if (job->fd_args.need_reclaim_cache) {
if (reclaimFilePageCache(job->fd_args.fd, 0, 0) == -1) {
serverLog(LL_NOTICE,"Unable to reclaim page cache: %s", strerror(errno));
}
}
if (job_type == BIO_CLOSE_AOF)
close(job->fd_args.fd);
} else if (job_type == BIO_LAZY_FREE) {
job->free_args.free_fn(job->free_args.free_args); job->free_args.free_fn(job->free_args.free_args);
} else { } else {
serverPanic("Wrong job type in bioProcessBackgroundJobs()."); serverPanic("Wrong job type in bioProcessBackgroundJobs().");
...@@ -252,37 +292,53 @@ void *bioProcessBackgroundJobs(void *arg) { ...@@ -252,37 +292,53 @@ void *bioProcessBackgroundJobs(void *arg) {
/* Lock again before reiterating the loop, if there are no longer /* Lock again before reiterating the loop, if there are no longer
* jobs to process we'll block again in pthread_cond_wait(). */ * jobs to process we'll block again in pthread_cond_wait(). */
pthread_mutex_lock(&bio_mutex[type]); pthread_mutex_lock(&bio_mutex[worker]);
listDelNode(bio_jobs[type],ln); listDelNode(bio_jobs[worker], ln);
bio_jobs_counter[job_type]--;
pthread_cond_signal(&bio_newjob_cond[worker]);
} }
} }
/* Return the number of pending jobs of the specified type. */ /* Return the number of pending jobs of the specified type. */
unsigned long bioPendingJobsOfType(int type) { unsigned long bioPendingJobsOfType(int type) {
unsigned long long val; unsigned int worker = bio_job_to_worker[type];
pthread_mutex_lock(&bio_mutex[type]);
val = listLength(bio_jobs[type]); pthread_mutex_lock(&bio_mutex[worker]);
pthread_mutex_unlock(&bio_mutex[type]); unsigned long val = bio_jobs_counter[type];
pthread_mutex_unlock(&bio_mutex[worker]);
return val; return val;
} }
/* Wait for the job queue of the worker for jobs of specified type to become empty. */
void bioDrainWorker(int job_type) {
unsigned long worker = bio_job_to_worker[job_type];
pthread_mutex_lock(&bio_mutex[worker]);
while (listLength(bio_jobs[worker]) > 0) {
pthread_cond_wait(&bio_newjob_cond[worker], &bio_mutex[worker]);
}
pthread_mutex_unlock(&bio_mutex[worker]);
}
/* Kill the running bio threads in an unclean way. This function should be /* Kill the running bio threads in an unclean way. This function should be
* used only when it's critical to stop the threads for some reason. * used only when it's critical to stop the threads for some reason.
* Currently Redis does this only on crash (for instance on SIGSEGV) in order * Currently Redis does this only on crash (for instance on SIGSEGV) in order
* to perform a fast memory check without other threads messing with memory. */ * to perform a fast memory check without other threads messing with memory. */
void bioKillThreads(void) { void bioKillThreads(void) {
int err, j; int err;
unsigned long j;
for (j = 0; j < BIO_NUM_OPS; j++) { for (j = 0; j < BIO_WORKER_NUM; j++) {
if (bio_threads[j] == pthread_self()) continue; if (bio_threads[j] == pthread_self()) continue;
if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) { if (bio_threads[j] && pthread_cancel(bio_threads[j]) == 0) {
if ((err = pthread_join(bio_threads[j],NULL)) != 0) { if ((err = pthread_join(bio_threads[j],NULL)) != 0) {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Bio thread for job type #%d can not be joined: %s", "Bio worker thread #%lu can not be joined: %s",
j, strerror(err)); j, strerror(err));
} else { } else {
serverLog(LL_WARNING, serverLog(LL_WARNING,
"Bio thread for job type #%d terminated",j); "Bio worker thread #%lu terminated",j);
} }
} }
} }
......
...@@ -35,15 +35,20 @@ typedef void lazy_free_fn(void *args[]); ...@@ -35,15 +35,20 @@ typedef void lazy_free_fn(void *args[]);
/* Exported API */ /* Exported API */
void bioInit(void); void bioInit(void);
unsigned long bioPendingJobsOfType(int type); unsigned long bioPendingJobsOfType(int type);
void bioDrainWorker(int job_type);
void bioKillThreads(void); void bioKillThreads(void);
void bioCreateCloseJob(int fd, int need_fsync, int need_reclaim_cache); void bioCreateCloseJob(int fd, int need_fsync, int need_reclaim_cache);
void bioCreateFsyncJob(int fd); void bioCreateCloseAofJob(int fd, long long offset, int need_reclaim_cache);
void bioCreateFsyncJob(int fd, long long offset, int need_reclaim_cache);
void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...); void bioCreateLazyFreeJob(lazy_free_fn free_fn, int arg_count, ...);
/* Background job opcodes */ /* Background job opcodes */
#define BIO_CLOSE_FILE 0 /* Deferred close(2) syscall. */ enum {
#define BIO_AOF_FSYNC 1 /* Deferred AOF fsync. */ BIO_CLOSE_FILE = 0, /* Deferred close(2) syscall. */
#define BIO_LAZY_FREE 2 /* Deferred objects freeing. */ BIO_AOF_FSYNC, /* Deferred AOF fsync. */
#define BIO_NUM_OPS 3 BIO_LAZY_FREE, /* Deferred objects freeing. */
BIO_CLOSE_AOF, /* Deferred close for AOF files. */
BIO_NUM_OPS
};
#endif #endif
...@@ -79,6 +79,7 @@ void initClientBlockingState(client *c) { ...@@ -79,6 +79,7 @@ void initClientBlockingState(client *c) {
c->bstate.numreplicas = 0; c->bstate.numreplicas = 0;
c->bstate.reploffset = 0; c->bstate.reploffset = 0;
c->bstate.unblock_on_nokey = 0; c->bstate.unblock_on_nokey = 0;
c->bstate.async_rm_call_handle = NULL;
} }
/* Block a client for the specific operation type. Once the CLIENT_BLOCKED /* Block a client for the specific operation type. Once the CLIENT_BLOCKED
...@@ -92,7 +93,7 @@ void blockClient(client *c, int btype) { ...@@ -92,7 +93,7 @@ void blockClient(client *c, int btype) {
c->flags |= CLIENT_BLOCKED; c->flags |= CLIENT_BLOCKED;
c->bstate.btype = btype; c->bstate.btype = btype;
server.blocked_clients++; if (!(c->flags & CLIENT_MODULE)) server.blocked_clients++; /* We count blocked client stats on regular clients and not on module clients */
server.blocked_clients_by_type[btype]++; server.blocked_clients_by_type[btype]++;
addClientToTimeoutTable(c); addClientToTimeoutTable(c);
} }
...@@ -131,6 +132,13 @@ void processUnblockedClients(void) { ...@@ -131,6 +132,13 @@ void processUnblockedClients(void) {
listDelNode(server.unblocked_clients,ln); listDelNode(server.unblocked_clients,ln);
c->flags &= ~CLIENT_UNBLOCKED; c->flags &= ~CLIENT_UNBLOCKED;
if (c->flags & CLIENT_MODULE) {
if (!(c->flags & CLIENT_BLOCKED)) {
moduleCallCommandUnblockedHandler(c);
}
continue;
}
/* Process remaining data in the input buffer, unless the client /* Process remaining data in the input buffer, unless the client
* is blocked again. Actually processInputBuffer() checks that the * is blocked again. Actually processInputBuffer() checks that the
* client is not blocked before to proceed, but things may change and * client is not blocked before to proceed, but things may change and
...@@ -172,12 +180,12 @@ void queueClientForReprocessing(client *c) { ...@@ -172,12 +180,12 @@ void queueClientForReprocessing(client *c) {
/* Unblock a client calling the right function depending on the kind /* Unblock a client calling the right function depending on the kind
* of operation the client is blocking for. */ * of operation the client is blocking for. */
void unblockClient(client *c) { void unblockClient(client *c, int queue_for_reprocessing) {
if (c->bstate.btype == BLOCKED_LIST || if (c->bstate.btype == BLOCKED_LIST ||
c->bstate.btype == BLOCKED_ZSET || c->bstate.btype == BLOCKED_ZSET ||
c->bstate.btype == BLOCKED_STREAM) { c->bstate.btype == BLOCKED_STREAM) {
unblockClientWaitingData(c); unblockClientWaitingData(c);
} else if (c->bstate.btype == BLOCKED_WAIT) { } else if (c->bstate.btype == BLOCKED_WAIT || c->bstate.btype == BLOCKED_WAITAOF) {
unblockClientWaitingReplicas(c); unblockClientWaitingReplicas(c);
} else if (c->bstate.btype == BLOCKED_MODULE) { } else if (c->bstate.btype == BLOCKED_MODULE) {
if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c);
...@@ -195,18 +203,23 @@ void unblockClient(client *c) { ...@@ -195,18 +203,23 @@ void unblockClient(client *c) {
* or in case a shutdown operation was canceled and we are still in the processCommand sequence */ * or in case a shutdown operation was canceled and we are still in the processCommand sequence */
if (!(c->flags & CLIENT_PENDING_COMMAND) && c->bstate.btype != BLOCKED_SHUTDOWN) { if (!(c->flags & CLIENT_PENDING_COMMAND) && c->bstate.btype != BLOCKED_SHUTDOWN) {
freeClientOriginalArgv(c); freeClientOriginalArgv(c);
/* Clients that are not blocked on keys are not reprocessed so we must
* call reqresAppendResponse here (for clients blocked on key,
* unblockClientOnKey is called, which eventually calls processCommand,
* which calls reqresAppendResponse) */
reqresAppendResponse(c);
resetClient(c); resetClient(c);
} }
/* Clear the flags, and put the client in the unblocked list so that /* Clear the flags, and put the client in the unblocked list so that
* we'll process new commands in its query buffer ASAP. */ * we'll process new commands in its query buffer ASAP. */
server.blocked_clients--; if (!(c->flags & CLIENT_MODULE)) server.blocked_clients--; /* We count blocked client stats on regular clients and not on module clients */
server.blocked_clients_by_type[c->bstate.btype]--; server.blocked_clients_by_type[c->bstate.btype]--;
c->flags &= ~CLIENT_BLOCKED; c->flags &= ~CLIENT_BLOCKED;
c->bstate.btype = BLOCKED_NONE; c->bstate.btype = BLOCKED_NONE;
c->bstate.unblock_on_nokey = 0; c->bstate.unblock_on_nokey = 0;
removeClientFromTimeoutTable(c); removeClientFromTimeoutTable(c);
queueClientForReprocessing(c); if (queue_for_reprocessing) queueClientForReprocessing(c);
} }
/* This function gets called when a blocked client timed out in order to /* This function gets called when a blocked client timed out in order to
...@@ -220,6 +233,10 @@ void replyToBlockedClientTimedOut(client *c) { ...@@ -220,6 +233,10 @@ void replyToBlockedClientTimedOut(client *c) {
updateStatsOnUnblock(c, 0, 0, 0); updateStatsOnUnblock(c, 0, 0, 0);
} else if (c->bstate.btype == BLOCKED_WAIT) { } else if (c->bstate.btype == BLOCKED_WAIT) {
addReplyLongLong(c,replicationCountAcksByOffset(c->bstate.reploffset)); addReplyLongLong(c,replicationCountAcksByOffset(c->bstate.reploffset));
} else if (c->bstate.btype == BLOCKED_WAITAOF) {
addReplyArrayLen(c,2);
addReplyLongLong(c,server.fsynced_reploff >= c->bstate.reploffset);
addReplyLongLong(c,replicationCountAOFAcksByOffset(c->bstate.reploffset));
} else if (c->bstate.btype == BLOCKED_MODULE) { } else if (c->bstate.btype == BLOCKED_MODULE) {
moduleBlockedClientTimedOut(c); moduleBlockedClientTimedOut(c);
} else { } else {
...@@ -238,7 +255,7 @@ void replyToClientsBlockedOnShutdown(void) { ...@@ -238,7 +255,7 @@ void replyToClientsBlockedOnShutdown(void) {
client *c = listNodeValue(ln); client *c = listNodeValue(ln);
if (c->flags & CLIENT_BLOCKED && c->bstate.btype == BLOCKED_SHUTDOWN) { if (c->flags & CLIENT_BLOCKED && c->bstate.btype == BLOCKED_SHUTDOWN) {
addReplyError(c, "Errors trying to SHUTDOWN. Check logs."); addReplyError(c, "Errors trying to SHUTDOWN. Check logs.");
unblockClient(c); unblockClient(c, 1);
} }
} }
} }
...@@ -385,7 +402,11 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo ...@@ -385,7 +402,11 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo
} }
} }
c->bstate.unblock_on_nokey = unblock_on_nokey; c->bstate.unblock_on_nokey = unblock_on_nokey;
c->flags |= CLIENT_PENDING_COMMAND; /* Currently we assume key blocking will require reprocessing the command.
* However in case of modules, they have a different way to handle the reprocessing
* which does not require setting the pending command flag */
if (btype != BLOCKED_MODULE)
c->flags |= CLIENT_PENDING_COMMAND;
blockClient(c,btype); blockClient(c,btype);
} }
...@@ -574,6 +595,16 @@ void blockForReplication(client *c, mstime_t timeout, long long offset, long num ...@@ -574,6 +595,16 @@ void blockForReplication(client *c, mstime_t timeout, long long offset, long num
blockClient(c,BLOCKED_WAIT); blockClient(c,BLOCKED_WAIT);
} }
/* block a client due to waitaof command */
void blockForAofFsync(client *c, mstime_t timeout, long long offset, int numlocal, long numreplicas) {
c->bstate.timeout = timeout;
c->bstate.reploffset = offset;
c->bstate.numreplicas = numreplicas;
c->bstate.numlocal = numlocal;
listAddNodeHead(server.clients_waiting_acks,c);
blockClient(c,BLOCKED_WAITAOF);
}
/* Postpone client from executing a command. For example the server might be busy /* Postpone client from executing a command. For example the server might be busy
* requesting to avoid processing clients commands which will be processed later * requesting to avoid processing clients commands which will be processed later
* when the it is ready to accept them. */ * when the it is ready to accept them. */
...@@ -589,7 +620,6 @@ void blockPostponeClient(client *c) { ...@@ -589,7 +620,6 @@ void blockPostponeClient(client *c) {
/* Block client due to shutdown command */ /* Block client due to shutdown command */
void blockClientShutdown(client *c) { void blockClientShutdown(client *c) {
blockClient(c, BLOCKED_SHUTDOWN); blockClient(c, BLOCKED_SHUTDOWN);
/* Mark this client to execute its command */
} }
/* Unblock a client once a specific key became available for it. /* Unblock a client once a specific key became available for it.
...@@ -608,12 +638,32 @@ static void unblockClientOnKey(client *c, robj *key) { ...@@ -608,12 +638,32 @@ static void unblockClientOnKey(client *c, robj *key) {
c->bstate.btype == BLOCKED_LIST || c->bstate.btype == BLOCKED_LIST ||
c->bstate.btype == BLOCKED_ZSET); c->bstate.btype == BLOCKED_ZSET);
unblockClient(c); /* We need to unblock the client before calling processCommandAndResetClient
* because it checks the CLIENT_BLOCKED flag */
unblockClient(c, 0);
/* In case this client was blocked on keys during command /* In case this client was blocked on keys during command
* we need to re process the command again */ * we need to re process the command again */
if (c->flags & CLIENT_PENDING_COMMAND) { if (c->flags & CLIENT_PENDING_COMMAND) {
c->flags &= ~CLIENT_PENDING_COMMAND; c->flags &= ~CLIENT_PENDING_COMMAND;
/* We want the command processing and the unblock handler (see RM_Call 'K' option)
* to run atomically, this is why we must enter the execution unit here before
* running the command, and exit the execution unit after calling the unblock handler (if exists).
* Notice that we also must set the current client so it will be available
* when we will try to send the the client side caching notification (done on 'afterCommand'). */
client *old_client = server.current_client;
server.current_client = c;
enterExecutionUnit(1, 0);
processCommandAndResetClient(c); processCommandAndResetClient(c);
if (!(c->flags & CLIENT_BLOCKED)) {
if (c->flags & CLIENT_MODULE) {
moduleCallCommandUnblockedHandler(c);
} else {
queueClientForReprocessing(c);
}
}
exitExecutionUnit();
afterCommand(c);
server.current_client = old_client;
} }
} }
...@@ -630,8 +680,6 @@ static void moduleUnblockClientOnKey(client *c, robj *key) { ...@@ -630,8 +680,6 @@ static void moduleUnblockClientOnKey(client *c, robj *key) {
if (moduleTryServeClientBlockedOnKey(c, key)) { if (moduleTryServeClientBlockedOnKey(c, key)) {
updateStatsOnUnblock(c, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies); updateStatsOnUnblock(c, 0, elapsedUs(replyTimer), server.stat_total_error_replies != prev_error_replies);
if (c->flags & CLIENT_PENDING_COMMAND)
c->flags &= ~CLIENT_PENDING_COMMAND;
moduleUnblockClient(c); moduleUnblockClient(c);
} }
/* We need to call afterCommand even if the client was not unblocked /* We need to call afterCommand even if the client was not unblocked
...@@ -651,7 +699,7 @@ void unblockClientOnTimeout(client *c) { ...@@ -651,7 +699,7 @@ void unblockClientOnTimeout(client *c) {
replyToBlockedClientTimedOut(c); replyToBlockedClientTimedOut(c);
if (c->flags & CLIENT_PENDING_COMMAND) if (c->flags & CLIENT_PENDING_COMMAND)
c->flags &= ~CLIENT_PENDING_COMMAND; c->flags &= ~CLIENT_PENDING_COMMAND;
unblockClient(c); unblockClient(c, 1);
} }
/* Unblock a client which is currently Blocked with error. /* Unblock a client which is currently Blocked with error.
...@@ -662,7 +710,7 @@ void unblockClientOnError(client *c, const char *err_str) { ...@@ -662,7 +710,7 @@ void unblockClientOnError(client *c, const char *err_str) {
updateStatsOnUnblock(c, 0, 0, 1); updateStatsOnUnblock(c, 0, 0, 1);
if (c->flags & CLIENT_PENDING_COMMAND) if (c->flags & CLIENT_PENDING_COMMAND)
c->flags &= ~CLIENT_PENDING_COMMAND; c->flags &= ~CLIENT_PENDING_COMMAND;
unblockClient(c); unblockClient(c, 1);
} }
/* sets blocking_keys to the total number of keys which has at least one client blocked on them /* sets blocking_keys to the total number of keys which has at least one client blocked on them
......
...@@ -234,6 +234,10 @@ void freeCallReply(CallReply *rep) { ...@@ -234,6 +234,10 @@ void freeCallReply(CallReply *rep) {
return; return;
} }
if (rep->flags & REPLY_FLAG_PARSED) { if (rep->flags & REPLY_FLAG_PARSED) {
if (rep->type == REDISMODULE_REPLY_PROMISE) {
zfree(rep);
return;
}
freeCallReplyInternal(rep); freeCallReplyInternal(rep);
} }
sdsfree(rep->original_proto); sdsfree(rep->original_proto);
...@@ -242,6 +246,17 @@ void freeCallReply(CallReply *rep) { ...@@ -242,6 +246,17 @@ void freeCallReply(CallReply *rep) {
zfree(rep); zfree(rep);
} }
CallReply *callReplyCreatePromise(void *private_data) {
CallReply *res = zmalloc(sizeof(*res));
res->type = REDISMODULE_REPLY_PROMISE;
/* Mark the reply as parsed so there will be not attempt to parse
* it when calling reply API such as freeCallReply.
* Also mark the reply as root so freeCallReply will not ignore it. */
res->flags |= REPLY_FLAG_PARSED | REPLY_FLAG_ROOT;
res->private_data = private_data;
return res;
}
static const ReplyParserCallbacks DefaultParserCallbacks = { static const ReplyParserCallbacks DefaultParserCallbacks = {
.null_callback = callReplyNull, .null_callback = callReplyNull,
.bulk_string_callback = callReplyBulkString, .bulk_string_callback = callReplyBulkString,
......
...@@ -33,6 +33,7 @@ ...@@ -33,6 +33,7 @@
#include "resp_parser.h" #include "resp_parser.h"
typedef struct CallReply CallReply; typedef struct CallReply CallReply;
typedef void (*RedisModuleOnUnblocked)(void *ctx, CallReply *reply, void *private_data);
CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data); CallReply *callReplyCreate(sds reply, list *deferred_error_list, void *private_data);
CallReply *callReplyCreateError(sds reply, void *private_data); CallReply *callReplyCreateError(sds reply, void *private_data);
...@@ -54,5 +55,6 @@ void *callReplyGetPrivateData(CallReply *rep); ...@@ -54,5 +55,6 @@ void *callReplyGetPrivateData(CallReply *rep);
int callReplyIsResp3(CallReply *rep); int callReplyIsResp3(CallReply *rep);
list *callReplyDeferredErrorList(CallReply *rep); list *callReplyDeferredErrorList(CallReply *rep);
void freeCallReply(CallReply *rep); void freeCallReply(CallReply *rep);
CallReply *callReplyCreatePromise(void *private_data);
#endif /* SRC_CALL_REPLY_H_ */ #endif /* SRC_CALL_REPLY_H_ */
...@@ -1837,6 +1837,26 @@ struct redisCommandArg WAIT_Args[] = { ...@@ -1837,6 +1837,26 @@ struct redisCommandArg WAIT_Args[] = {
{0} {0}
}; };
/********** WAITAOF ********************/
/* WAITAOF history */
#define WAITAOF_History NULL
/* WAITAOF tips */
const char *WAITAOF_tips[] = {
"request_policy:all_shards",
"response_policy:agg_min",
NULL
};
/* WAITAOF argument table */
struct redisCommandArg WAITAOF_Args[] = {
{"numlocal",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE},
{"numreplicas",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE},
{"timeout",ARG_TYPE_INTEGER,-1,NULL,NULL,NULL,CMD_ARG_NONE},
{0}
};
/********** GEOADD ********************/ /********** GEOADD ********************/
/* GEOADD history */ /* GEOADD history */
...@@ -7243,7 +7263,8 @@ struct redisCommand redisCommandTable[] = { ...@@ -7243,7 +7263,8 @@ struct redisCommand redisCommandTable[] = {
{"ttl","Get the time to live for a key in seconds","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,TTL_History,TTL_tips,ttlCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=TTL_Args}, {"ttl","Get the time to live for a key in seconds","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,TTL_History,TTL_tips,ttlCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=TTL_Args},
{"type","Determine the type stored at key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,TYPE_History,TYPE_tips,typeCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=TYPE_Args}, {"type","Determine the type stored at key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,TYPE_History,TYPE_tips,typeCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=TYPE_Args},
{"unlink","Delete a key asynchronously in another thread. Otherwise it is just as DEL, but non blocking.","O(1) for each key removed regardless of its size. Then the command does O(N) work in a different thread in order to reclaim memory, where N is the number of allocations the deleted objects where composed of.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,UNLINK_History,UNLINK_tips,unlinkCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RM|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=UNLINK_Args}, {"unlink","Delete a key asynchronously in another thread. Otherwise it is just as DEL, but non blocking.","O(1) for each key removed regardless of its size. Then the command does O(N) work in a different thread in order to reclaim memory, where N is the number of allocations the deleted objects where composed of.","4.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,UNLINK_History,UNLINK_tips,unlinkCommand,-2,CMD_WRITE|CMD_FAST,ACL_CATEGORY_KEYSPACE,{{NULL,CMD_KEY_RM|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=UNLINK_Args},
{"wait","Wait for the synchronous replication of all the write commands sent in the context of the current connection","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,WAIT_History,WAIT_tips,waitCommand,3,CMD_NOSCRIPT,ACL_CATEGORY_CONNECTION,.args=WAIT_Args}, {"wait","Wait for the synchronous replication of all the write commands sent in the context of the current connection","O(1)","3.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,WAIT_History,WAIT_tips,waitCommand,3,0,ACL_CATEGORY_CONNECTION,.args=WAIT_Args},
{"waitaof","Wait for all write commands sent in the context of the current connection to be synced to AOF of local host and/or replicas","O(1)","7.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GENERIC,WAITAOF_History,WAITAOF_tips,waitaofCommand,4,CMD_NOSCRIPT,ACL_CATEGORY_CONNECTION,.args=WAITAOF_Args},
/* geo */ /* geo */
{"geoadd","Add one or more geospatial items in the geospatial index represented using a sorted set","O(log(N)) for each item added, where N is the number of elements in the sorted set.","3.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GEO,GEOADD_History,GEOADD_tips,geoaddCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=GEOADD_Args}, {"geoadd","Add one or more geospatial items in the geospatial index represented using a sorted set","O(log(N)) for each item added, where N is the number of elements in the sorted set.","3.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GEO,GEOADD_History,GEOADD_tips,geoaddCommand,-5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_GEO,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=GEOADD_Args},
{"geodist","Returns the distance between two members of a geospatial index","O(log(N))","3.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GEO,GEODIST_History,GEODIST_tips,geodistCommand,-4,CMD_READONLY,ACL_CATEGORY_GEO,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=GEODIST_Args}, {"geodist","Returns the distance between two members of a geospatial index","O(log(N))","3.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_GEO,GEODIST_History,GEODIST_tips,geodistCommand,-4,CMD_READONLY,ACL_CATEGORY_GEO,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=GEODIST_Args},
...@@ -7279,11 +7300,11 @@ struct redisCommand redisCommandTable[] = { ...@@ -7279,11 +7300,11 @@ struct redisCommand redisCommandTable[] = {
{"pfmerge","Merge N different HyperLogLogs into a single one.","O(N) to merge N HyperLogLogs, but with high constant times.","2.8.9",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_HYPERLOGLOG,PFMERGE_History,PFMERGE_tips,pfmergeCommand,-2,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_HYPERLOGLOG,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=PFMERGE_Args}, {"pfmerge","Merge N different HyperLogLogs into a single one.","O(N) to merge N HyperLogLogs, but with high constant times.","2.8.9",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_HYPERLOGLOG,PFMERGE_History,PFMERGE_tips,pfmergeCommand,-2,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_HYPERLOGLOG,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=PFMERGE_Args},
{"pfselftest","An internal command for testing HyperLogLog values","N/A","2.8.9",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_HYPERLOGLOG,PFSELFTEST_History,PFSELFTEST_tips,pfselftestCommand,1,CMD_ADMIN,ACL_CATEGORY_HYPERLOGLOG}, {"pfselftest","An internal command for testing HyperLogLog values","N/A","2.8.9",CMD_DOC_SYSCMD,NULL,NULL,COMMAND_GROUP_HYPERLOGLOG,PFSELFTEST_History,PFSELFTEST_tips,pfselftestCommand,1,CMD_ADMIN,ACL_CATEGORY_HYPERLOGLOG},
/* list */ /* list */
{"blmove","Pop an element from a list, push it to another list and return it; or block until one is available","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLMOVE_History,BLMOVE_tips,blmoveCommand,6,CMD_WRITE|CMD_DENYOOM|CMD_NOSCRIPT|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=BLMOVE_Args}, {"blmove","Pop an element from a list, push it to another list and return it; or block until one is available","O(1)","6.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLMOVE_History,BLMOVE_tips,blmoveCommand,6,CMD_WRITE|CMD_DENYOOM|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=BLMOVE_Args},
{"blmpop","Pop elements from a list, or block until one is available","O(N+M) where N is the number of provided keys and M is the number of elements returned.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLMPOP_History,BLMPOP_tips,blmpopCommand,-5,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},blmpopGetKeys,.args=BLMPOP_Args}, {"blmpop","Pop elements from a list, or block until one is available","O(N+M) where N is the number of provided keys and M is the number of elements returned.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLMPOP_History,BLMPOP_tips,blmpopCommand,-5,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},blmpopGetKeys,.args=BLMPOP_Args},
{"blpop","Remove and get the first element in a list, or block until one is available","O(N) where N is the number of provided keys.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLPOP_History,BLPOP_tips,blpopCommand,-3,CMD_WRITE|CMD_NOSCRIPT|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BLPOP_Args}, {"blpop","Remove and get the first element in a list, or block until one is available","O(N) where N is the number of provided keys.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BLPOP_History,BLPOP_tips,blpopCommand,-3,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BLPOP_Args},
{"brpop","Remove and get the last element in a list, or block until one is available","O(N) where N is the number of provided keys.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BRPOP_History,BRPOP_tips,brpopCommand,-3,CMD_WRITE|CMD_NOSCRIPT|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BRPOP_Args}, {"brpop","Remove and get the last element in a list, or block until one is available","O(N) where N is the number of provided keys.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,BRPOP_History,BRPOP_tips,brpopCommand,-3,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BRPOP_Args},
{"brpoplpush","Pop an element from a list, push it to another list and return it; or block until one is available","O(1)","2.2.0",CMD_DOC_DEPRECATED,"`BLMOVE` with the `RIGHT` and `LEFT` arguments","6.2.0",COMMAND_GROUP_LIST,BRPOPLPUSH_History,BRPOPLPUSH_tips,brpoplpushCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_NOSCRIPT|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=BRPOPLPUSH_Args}, {"brpoplpush","Pop an element from a list, push it to another list and return it; or block until one is available","O(1)","2.2.0",CMD_DOC_DEPRECATED,"`BLMOVE` with the `RIGHT` and `LEFT` arguments","6.2.0",COMMAND_GROUP_LIST,BRPOPLPUSH_History,BRPOPLPUSH_tips,brpoplpushCommand,4,CMD_WRITE|CMD_DENYOOM|CMD_BLOCKING,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=BRPOPLPUSH_Args},
{"lindex","Get an element from a list by its index","O(N) where N is the number of elements to traverse to get to the element at index. This makes asking for the first or the last element of the list O(1).","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LINDEX_History,LINDEX_tips,lindexCommand,3,CMD_READONLY,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LINDEX_Args}, {"lindex","Get an element from a list by its index","O(N) where N is the number of elements to traverse to get to the element at index. This makes asking for the first or the last element of the list O(1).","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LINDEX_History,LINDEX_tips,lindexCommand,3,CMD_READONLY,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LINDEX_Args},
{"linsert","Insert an element before or after another element in a list","O(N) where N is the number of elements to traverse before seeing the value pivot. This means that inserting somewhere on the left end on the list (head) can be considered O(1) and inserting somewhere on the right end (tail) is O(N).","2.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LINSERT_History,LINSERT_tips,linsertCommand,5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LINSERT_Args}, {"linsert","Insert an element before or after another element in a list","O(N) where N is the number of elements to traverse before seeing the value pivot. This means that inserting somewhere on the left end on the list (head) can be considered O(1) and inserting somewhere on the right end (tail) is O(N).","2.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LINSERT_History,LINSERT_tips,linsertCommand,5,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RW|CMD_KEY_INSERT,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LINSERT_Args},
{"llen","Get the length of a list","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LLEN_History,LLEN_tips,llenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LLEN_Args}, {"llen","Get the length of a list","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_LIST,LLEN_History,LLEN_tips,llenCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_LIST,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=LLEN_Args},
...@@ -7372,8 +7393,8 @@ struct redisCommand redisCommandTable[] = { ...@@ -7372,8 +7393,8 @@ struct redisCommand redisCommandTable[] = {
{"sunionstore","Add multiple sets and store the resulting set in a key","O(N) where N is the total number of elements in all given sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SET,SUNIONSTORE_History,SUNIONSTORE_tips,sunionstoreCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SET,{{NULL,CMD_KEY_OW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNIONSTORE_Args}, {"sunionstore","Add multiple sets and store the resulting set in a key","O(N) where N is the total number of elements in all given sets.","1.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SET,SUNIONSTORE_History,SUNIONSTORE_tips,sunionstoreCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_SET,{{NULL,CMD_KEY_OW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}},{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_RANGE,.fk.range={-1,1,0}}},.args=SUNIONSTORE_Args},
/* sorted_set */ /* sorted_set */
{"bzmpop","Remove and return members with scores in a sorted set or block until one is available","O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZMPOP_History,BZMPOP_tips,bzmpopCommand,-5,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},blmpopGetKeys,.args=BZMPOP_Args}, {"bzmpop","Remove and return members with scores in a sorted set or block until one is available","O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.","7.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZMPOP_History,BZMPOP_tips,bzmpopCommand,-5,CMD_WRITE|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={2},KSPEC_FK_KEYNUM,.fk.keynum={0,1,1}}},blmpopGetKeys,.args=BZMPOP_Args},
{"bzpopmax","Remove and return the member with the highest score from one or more sorted sets, or block until one is available","O(log(N)) with N being the number of elements in the sorted set.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZPOPMAX_History,BZPOPMAX_tips,bzpopmaxCommand,-3,CMD_WRITE|CMD_NOSCRIPT|CMD_FAST|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BZPOPMAX_Args}, {"bzpopmax","Remove and return the member with the highest score from one or more sorted sets, or block until one is available","O(log(N)) with N being the number of elements in the sorted set.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZPOPMAX_History,BZPOPMAX_tips,bzpopmaxCommand,-3,CMD_WRITE|CMD_FAST|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BZPOPMAX_Args},
{"bzpopmin","Remove and return the member with the lowest score from one or more sorted sets, or block until one is available","O(log(N)) with N being the number of elements in the sorted set.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZPOPMIN_History,BZPOPMIN_tips,bzpopminCommand,-3,CMD_WRITE|CMD_NOSCRIPT|CMD_FAST|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BZPOPMIN_Args}, {"bzpopmin","Remove and return the member with the lowest score from one or more sorted sets, or block until one is available","O(log(N)) with N being the number of elements in the sorted set.","5.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,BZPOPMIN_History,BZPOPMIN_tips,bzpopminCommand,-3,CMD_WRITE|CMD_FAST|CMD_BLOCKING,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_ACCESS|CMD_KEY_DELETE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={-2,1,0}}},.args=BZPOPMIN_Args},
{"zadd","Add one or more members to a sorted set, or update its score if it already exists","O(log(N)) for each item added, where N is the number of elements in the sorted set.","1.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZADD_History,ZADD_tips,zaddCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZADD_Args}, {"zadd","Add one or more members to a sorted set, or update its score if it already exists","O(log(N)) for each item added, where N is the number of elements in the sorted set.","1.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZADD_History,ZADD_tips,zaddCommand,-4,CMD_WRITE|CMD_DENYOOM|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RW|CMD_KEY_UPDATE,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZADD_Args},
{"zcard","Get the number of members in a sorted set","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZCARD_History,ZCARD_tips,zcardCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZCARD_Args}, {"zcard","Get the number of members in a sorted set","O(1)","1.2.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZCARD_History,ZCARD_tips,zcardCommand,2,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RO,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZCARD_Args},
{"zcount","Count the members in a sorted set with scores within the given values","O(log(N)) with N being the number of elements in the sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZCOUNT_History,ZCOUNT_tips,zcountCommand,4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZCOUNT_Args}, {"zcount","Count the members in a sorted set with scores within the given values","O(log(N)) with N being the number of elements in the sorted set.","2.0.0",CMD_DOC_NONE,NULL,NULL,COMMAND_GROUP_SORTED_SET,ZCOUNT_History,ZCOUNT_tips,zcountCommand,4,CMD_READONLY|CMD_FAST,ACL_CATEGORY_SORTEDSET,{{NULL,CMD_KEY_RO|CMD_KEY_ACCESS,KSPEC_BS_INDEX,.bs.index={1},KSPEC_FK_RANGE,.fk.range={0,1,0}}},.args=ZCOUNT_Args},
......
...@@ -13,6 +13,24 @@ ...@@ -13,6 +13,24 @@
"STALE", "STALE",
"SENTINEL" "SENTINEL"
], ],
"reply_schema": {
"anyOf": [
{
"type": "array",
"description": "In case `categoryname` was not given, a list of existing ACL categories",
"items": {
"type": "string"
}
},
{
"type": "array",
"description": "In case `categoryname` was given, list of commands that fall under the provided ACL category",
"items": {
"type": "string"
}
}
]
},
"arguments": [ "arguments": [
{ {
"name": "categoryname", "name": "categoryname",
......
...@@ -14,6 +14,10 @@ ...@@ -14,6 +14,10 @@
"STALE", "STALE",
"SENTINEL" "SENTINEL"
], ],
"reply_schema": {
"type": "integer",
"description": "The number of users that were deleted"
},
"arguments": [ "arguments": [
{ {
"name": "username", "name": "username",
......
...@@ -15,6 +15,18 @@ ...@@ -15,6 +15,18 @@
"STALE", "STALE",
"SENTINEL" "SENTINEL"
], ],
"reply_schema": {
"anyOf": [
{
"const": "OK",
"description": "The given user may successfully execute the given command."
},
{
"type": "string",
"description": "The description of the problem, in case the user is not allowed to run the given command."
}
]
},
"arguments": [ "arguments": [
{ {
"name": "username", "name": "username",
......
...@@ -13,6 +13,10 @@ ...@@ -13,6 +13,10 @@
"STALE", "STALE",
"SENTINEL" "SENTINEL"
], ],
"reply_schema": {
"type": "string",
"description": "Pseudorandom data. By default it contains 64 bytes, representing 256 bits of data. If `bits` was given, the output string length is the number of specified bits (rounded to the next multiple of 4) divided by 4."
},
"arguments": [ "arguments": [
{ {
"name": "bits", "name": "bits",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment