Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ruanhaishen
redis
Commits
697af434
Commit
697af434
authored
Dec 28, 2010
by
antirez
Browse files
initial changes needed to turn the current VM code into a cache system. Tons of work to do still.
parent
33388d43
Changes
4
Show whitespace changes
Inline
Side-by-side
src/diskstore.c
View file @
697af434
...
@@ -16,8 +16,17 @@
...
@@ -16,8 +16,17 @@
* directory will contain in the average 15258 entires, that is ok with
* directory will contain in the average 15258 entires, that is ok with
* most filesystems implementation.
* most filesystems implementation.
*
*
* The actaul implementation of this disk store is highly related to the
* Note that since Redis supports multiple databases, the actual key name
* filesystem implementation. This implementation may be replaced by
* is:
*
* /0b/ee/<dbid>_0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
*
* so for instance if the key is inside DB 0:
*
* /0b/ee/0_0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
*
* The actaul implementation of this disk store is highly dependant to the
* filesystem implementation itself. This implementation may be replaced by
* a B+TREE implementation in future implementations.
* a B+TREE implementation in future implementations.
*
*
* Data ok every key is serialized using the same format used for .rdb
* Data ok every key is serialized using the same format used for .rdb
...
@@ -68,7 +77,7 @@
...
@@ -68,7 +77,7 @@
int
dsOpen
(
void
)
{
int
dsOpen
(
void
)
{
struct
stat
sb
;
struct
stat
sb
;
int
retval
;
int
retval
;
char
*
path
=
server
.
d
iskstore
_path
;
char
*
path
=
server
.
d
s
_path
;
if
((
retval
=
stat
(
path
,
&
sb
)
==
-
1
)
&&
errno
!=
ENOENT
)
{
if
((
retval
=
stat
(
path
,
&
sb
)
==
-
1
)
&&
errno
!=
ENOENT
)
{
redisLog
(
REDIS_WARNING
,
"Error opening disk store at %s: %s"
,
redisLog
(
REDIS_WARNING
,
"Error opening disk store at %s: %s"
,
...
...
src/networking.c
View file @
697af434
...
@@ -167,7 +167,7 @@ void _addReplyStringToList(redisClient *c, char *s, size_t len) {
...
@@ -167,7 +167,7 @@ void _addReplyStringToList(redisClient *c, char *s, size_t len) {
void
addReply
(
redisClient
*
c
,
robj
*
obj
)
{
void
addReply
(
redisClient
*
c
,
robj
*
obj
)
{
if
(
_installWriteEvent
(
c
)
!=
REDIS_OK
)
return
;
if
(
_installWriteEvent
(
c
)
!=
REDIS_OK
)
return
;
redisAssert
(
!
server
.
vm
_enabled
||
obj
->
storage
==
REDIS_VM_MEMORY
);
redisAssert
(
!
server
.
ds
_enabled
||
obj
->
storage
==
REDIS_VM_MEMORY
);
/* This is an important place where we can avoid copy-on-write
/* This is an important place where we can avoid copy-on-write
* when there is a saving child running, avoiding touching the
* when there is a saving child running, avoiding touching the
...
@@ -460,7 +460,7 @@ void freeClient(redisClient *c) {
...
@@ -460,7 +460,7 @@ void freeClient(redisClient *c) {
/* Remove from the list of clients waiting for swapped keys, or ready
/* Remove from the list of clients waiting for swapped keys, or ready
* to be restarted, but not yet woken up again. */
* to be restarted, but not yet woken up again. */
if
(
c
->
flags
&
REDIS_IO_WAIT
)
{
if
(
c
->
flags
&
REDIS_IO_WAIT
)
{
redisAssert
(
server
.
vm
_enabled
);
redisAssert
(
server
.
ds
_enabled
);
if
(
listLength
(
c
->
io_keys
)
==
0
)
{
if
(
listLength
(
c
->
io_keys
)
==
0
)
{
ln
=
listSearchKey
(
server
.
io_ready_clients
,
c
);
ln
=
listSearchKey
(
server
.
io_ready_clients
,
c
);
...
@@ -474,7 +474,7 @@ void freeClient(redisClient *c) {
...
@@ -474,7 +474,7 @@ void freeClient(redisClient *c) {
dontWaitForSwappedKey
(
c
,
ln
->
value
);
dontWaitForSwappedKey
(
c
,
ln
->
value
);
}
}
}
}
server
.
vm
_blocked_clients
--
;
server
.
cache
_blocked_clients
--
;
}
}
listRelease
(
c
->
io_keys
);
listRelease
(
c
->
io_keys
);
/* Master/slave cleanup.
/* Master/slave cleanup.
...
...
src/redis.c
View file @
697af434
...
@@ -618,27 +618,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
...
@@ -618,27 +618,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
* in order to guarantee a strict consistency. */
* in order to guarantee a strict consistency. */
if
(
server
.
masterhost
==
NULL
)
activeExpireCycle
();
if
(
server
.
masterhost
==
NULL
)
activeExpireCycle
();
/* Swap a few keys on disk if we are over the memory limit and VM
/* Remove a few cached objects from memory if we are over the
* is enbled. Try to free objects from the free list first. */
* configured memory limit */
if
(
vmCanSwapOut
())
{
while
(
server
.
ds_enabled
&&
zmalloc_used_memory
()
>
while
(
server
.
vm_enabled
&&
zmalloc_used_memory
()
>
server
.
cache_max_memory
)
server
.
vm_max_memory
)
{
{
int
retval
=
(
server
.
vm_max_threads
==
0
)
?
cacheFreeOneEntry
();
vmSwapOneObjectBlocking
()
:
vmSwapOneObjectThreaded
();
if
(
retval
==
REDIS_ERR
&&
!
(
loops
%
300
)
&&
zmalloc_used_memory
()
>
(
server
.
vm_max_memory
+
server
.
vm_max_memory
/
10
))
{
redisLog
(
REDIS_WARNING
,
"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!"
);
}
/* Note that when using threade I/O we free just one object,
* because anyway when the I/O thread in charge to swap this
* object out will finish, the handler of completed jobs
* will try to swap more objects if we are still out of memory. */
if
(
retval
==
REDIS_ERR
||
server
.
vm_max_threads
>
0
)
break
;
}
}
}
/* Replication cron function -- used to reconnect to master and
/* Replication cron function -- used to reconnect to master and
...
@@ -656,8 +641,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
...
@@ -656,8 +641,8 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
listNode
*
ln
;
listNode
*
ln
;
redisClient
*
c
;
redisClient
*
c
;
/* Awake clients that got all the
swapped
keys they requested */
/* Awake clients that got all the
on disk
keys they requested */
if
(
server
.
vm
_enabled
&&
listLength
(
server
.
io_ready_clients
))
{
if
(
server
.
ds
_enabled
&&
listLength
(
server
.
io_ready_clients
))
{
listIter
li
;
listIter
li
;
listRewind
(
server
.
io_ready_clients
,
&
li
);
listRewind
(
server
.
io_ready_clients
,
&
li
);
...
@@ -668,7 +653,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
...
@@ -668,7 +653,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
/* Resume the client. */
/* Resume the client. */
listDelNode
(
server
.
io_ready_clients
,
ln
);
listDelNode
(
server
.
io_ready_clients
,
ln
);
c
->
flags
&=
(
~
REDIS_IO_WAIT
);
c
->
flags
&=
(
~
REDIS_IO_WAIT
);
server
.
vm
_blocked_clients
--
;
server
.
cache
_blocked_clients
--
;
aeCreateFileEvent
(
server
.
el
,
c
->
fd
,
AE_READABLE
,
aeCreateFileEvent
(
server
.
el
,
c
->
fd
,
AE_READABLE
,
readQueryFromClient
,
c
);
readQueryFromClient
,
c
);
cmd
=
lookupCommand
(
c
->
argv
[
0
]
->
ptr
);
cmd
=
lookupCommand
(
c
->
argv
[
0
]
->
ptr
);
...
@@ -787,13 +772,10 @@ void initServerConfig() {
...
@@ -787,13 +772,10 @@ void initServerConfig() {
server
.
maxmemory
=
0
;
server
.
maxmemory
=
0
;
server
.
maxmemory_policy
=
REDIS_MAXMEMORY_VOLATILE_LRU
;
server
.
maxmemory_policy
=
REDIS_MAXMEMORY_VOLATILE_LRU
;
server
.
maxmemory_samples
=
3
;
server
.
maxmemory_samples
=
3
;
server
.
vm_enabled
=
0
;
server
.
ds_enabled
=
0
;
server
.
vm_swap_file
=
zstrdup
(
"/tmp/redis-%p.vm"
);
server
.
ds_path
=
zstrdup
(
"/tmp/redis.ds"
);
server
.
vm_page_size
=
256
;
/* 256 bytes per page */
server
.
cache_max_memory
=
64LL
*
1024
*
1024
;
/* 64 MB of RAM */
server
.
vm_pages
=
1024
*
1024
*
100
;
/* 104 millions of pages */
server
.
cache_blocked_clients
=
0
;
server
.
vm_max_memory
=
1024LL
*
1024
*
1024
*
1
;
/* 1 GB of RAM */
server
.
vm_max_threads
=
4
;
server
.
vm_blocked_clients
=
0
;
server
.
hash_max_zipmap_entries
=
REDIS_HASH_MAX_ZIPMAP_ENTRIES
;
server
.
hash_max_zipmap_entries
=
REDIS_HASH_MAX_ZIPMAP_ENTRIES
;
server
.
hash_max_zipmap_value
=
REDIS_HASH_MAX_ZIPMAP_VALUE
;
server
.
hash_max_zipmap_value
=
REDIS_HASH_MAX_ZIPMAP_VALUE
;
server
.
list_max_ziplist_entries
=
REDIS_LIST_MAX_ZIPLIST_ENTRIES
;
server
.
list_max_ziplist_entries
=
REDIS_LIST_MAX_ZIPLIST_ENTRIES
;
...
@@ -873,7 +855,7 @@ void initServer() {
...
@@ -873,7 +855,7 @@ void initServer() {
server
.
db
[
j
].
expires
=
dictCreate
(
&
keyptrDictType
,
NULL
);
server
.
db
[
j
].
expires
=
dictCreate
(
&
keyptrDictType
,
NULL
);
server
.
db
[
j
].
blocking_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
server
.
db
[
j
].
blocking_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
server
.
db
[
j
].
watched_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
server
.
db
[
j
].
watched_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
if
(
server
.
vm
_enabled
)
if
(
server
.
ds
_enabled
)
server
.
db
[
j
].
io_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
server
.
db
[
j
].
io_keys
=
dictCreate
(
&
keylistDictType
,
NULL
);
server
.
db
[
j
].
id
=
j
;
server
.
db
[
j
].
id
=
j
;
}
}
...
@@ -911,7 +893,7 @@ void initServer() {
...
@@ -911,7 +893,7 @@ void initServer() {
}
}
}
}
if
(
server
.
vm
_enabled
)
vm
Init
();
if
(
server
.
ds
_enabled
)
ds
Init
();
}
}
/* Populates the Redis Command Table starting from the hard coded list
/* Populates the Redis Command Table starting from the hard coded list
...
@@ -1050,8 +1032,8 @@ int processCommand(redisClient *c) {
...
@@ -1050,8 +1032,8 @@ int processCommand(redisClient *c) {
queueMultiCommand
(
c
,
cmd
);
queueMultiCommand
(
c
,
cmd
);
addReply
(
c
,
shared
.
queued
);
addReply
(
c
,
shared
.
queued
);
}
else
{
}
else
{
if
(
server
.
vm
_enabled
&&
server
.
vm_max_threads
>
0
&&
if
(
server
.
ds
_enabled
&&
blockClientOnSwappedKeys
(
c
,
cmd
))
blockClientOnSwappedKeys
(
c
,
cmd
))
return
REDIS_ERR
;
return
REDIS_ERR
;
call
(
c
,
cmd
);
call
(
c
,
cmd
);
}
}
return
REDIS_OK
;
return
REDIS_OK
;
...
@@ -1072,7 +1054,6 @@ int prepareForShutdown() {
...
@@ -1072,7 +1054,6 @@ int prepareForShutdown() {
if
(
server
.
appendonly
)
{
if
(
server
.
appendonly
)
{
/* Append only file: fsync() the AOF and exit */
/* Append only file: fsync() the AOF and exit */
aof_fsync
(
server
.
appendfd
);
aof_fsync
(
server
.
appendfd
);
if
(
server
.
vm_enabled
)
unlink
(
server
.
vm_swap_file
);
}
else
if
(
server
.
saveparamslen
>
0
)
{
}
else
if
(
server
.
saveparamslen
>
0
)
{
/* Snapshotting. Perform a SYNC SAVE and exit */
/* Snapshotting. Perform a SYNC SAVE and exit */
if
(
rdbSave
(
server
.
dbfilename
)
!=
REDIS_OK
)
{
if
(
rdbSave
(
server
.
dbfilename
)
!=
REDIS_OK
)
{
...
@@ -1185,7 +1166,7 @@ sds genRedisInfoString(void) {
...
@@ -1185,7 +1166,7 @@ sds genRedisInfoString(void) {
"hash_max_zipmap_value:%zu
\r\n
"
"hash_max_zipmap_value:%zu
\r\n
"
"pubsub_channels:%ld
\r\n
"
"pubsub_channels:%ld
\r\n
"
"pubsub_patterns:%u
\r\n
"
"pubsub_patterns:%u
\r\n
"
"
vm
_enabled:%d
\r\n
"
"
ds
_enabled:%d
\r\n
"
"role:%s
\r\n
"
"role:%s
\r\n
"
,
REDIS_VERSION
,
,
REDIS_VERSION
,
redisGitSHA1
(),
redisGitSHA1
(),
...
@@ -1228,7 +1209,7 @@ sds genRedisInfoString(void) {
...
@@ -1228,7 +1209,7 @@ sds genRedisInfoString(void) {
server
.
hash_max_zipmap_value
,
server
.
hash_max_zipmap_value
,
dictSize
(
server
.
pubsub_channels
),
dictSize
(
server
.
pubsub_channels
),
listLength
(
server
.
pubsub_patterns
),
listLength
(
server
.
pubsub_patterns
),
server
.
vm
_enabled
!=
0
,
server
.
ds
_enabled
!=
0
,
server
.
masterhost
==
NULL
?
"master"
:
"slave"
server
.
masterhost
==
NULL
?
"master"
:
"slave"
);
);
if
(
server
.
masterhost
)
{
if
(
server
.
masterhost
)
{
...
@@ -1255,33 +1236,13 @@ sds genRedisInfoString(void) {
...
@@ -1255,33 +1236,13 @@ sds genRedisInfoString(void) {
);
);
}
}
}
}
if
(
server
.
vm
_enabled
)
{
if
(
server
.
ds
_enabled
)
{
lockThreadedIO
();
lockThreadedIO
();
info
=
sdscatprintf
(
info
,
info
=
sdscatprintf
(
info
,
"vm_conf_max_memory:%llu
\r\n
"
"cache_max_memory:%llu
\r\n
"
"vm_conf_page_size:%llu
\r\n
"
"cache_blocked_clients:%lu
\r\n
"
"vm_conf_pages:%llu
\r\n
"
,(
unsigned
long
long
)
server
.
cache_max_memory
,
"vm_stats_used_pages:%llu
\r\n
"
(
unsigned
long
)
server
.
cache_blocked_clients
"vm_stats_swapped_objects:%llu
\r\n
"
"vm_stats_swappin_count:%llu
\r\n
"
"vm_stats_swappout_count:%llu
\r\n
"
"vm_stats_io_newjobs_len:%lu
\r\n
"
"vm_stats_io_processing_len:%lu
\r\n
"
"vm_stats_io_processed_len:%lu
\r\n
"
"vm_stats_io_active_threads:%lu
\r\n
"
"vm_stats_blocked_clients:%lu
\r\n
"
,(
unsigned
long
long
)
server
.
vm_max_memory
,
(
unsigned
long
long
)
server
.
vm_page_size
,
(
unsigned
long
long
)
server
.
vm_pages
,
(
unsigned
long
long
)
server
.
vm_stats_used_pages
,
(
unsigned
long
long
)
server
.
vm_stats_swapped_objects
,
(
unsigned
long
long
)
server
.
vm_stats_swapins
,
(
unsigned
long
long
)
server
.
vm_stats_swapouts
,
(
unsigned
long
)
listLength
(
server
.
io_newjobs
),
(
unsigned
long
)
listLength
(
server
.
io_processing
),
(
unsigned
long
)
listLength
(
server
.
io_processed
),
(
unsigned
long
)
server
.
io_active_threads
,
(
unsigned
long
)
server
.
vm_blocked_clients
);
);
unlockThreadedIO
();
unlockThreadedIO
();
}
}
...
...
src/redis.h
View file @
697af434
...
@@ -440,7 +440,7 @@ struct redisServer {
...
@@ -440,7 +440,7 @@ struct redisServer {
int
maxmemory_samples
;
int
maxmemory_samples
;
/* Blocked clients */
/* Blocked clients */
unsigned
int
bpop_blocked_clients
;
unsigned
int
bpop_blocked_clients
;
unsigned
int
vm
_blocked_clients
;
unsigned
int
cache
_blocked_clients
;
list
*
unblocked_clients
;
list
*
unblocked_clients
;
/* Sort parameters - qsort_r() is only available under BSD so we
/* Sort parameters - qsort_r() is only available under BSD so we
* have to take this state global, in order to pass it to sortCompare() */
* have to take this state global, in order to pass it to sortCompare() */
...
@@ -448,11 +448,9 @@ struct redisServer {
...
@@ -448,11 +448,9 @@ struct redisServer {
int
sort_alpha
;
int
sort_alpha
;
int
sort_bypattern
;
int
sort_bypattern
;
/* Virtual memory configuration */
/* Virtual memory configuration */
int
vm_enabled
;
int
ds_enabled
;
/* backend disk in redis.conf */
char
*
vm_swap_file
;
char
*
ds_path
;
/* location of the disk store on disk */
off_t
vm_page_size
;
unsigned
long
long
cache_max_memory
;
off_t
vm_pages
;
unsigned
long
long
vm_max_memory
;
/* Zip structure config */
/* Zip structure config */
size_t
hash_max_zipmap_entries
;
size_t
hash_max_zipmap_entries
;
size_t
hash_max_zipmap_value
;
size_t
hash_max_zipmap_value
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment