github
/
lua-nginx-module
зеркало из https://github.com/github/lua-nginx-module.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

the shared_dict get and set are now complete and usable :D

This commit is contained in:
agentzh (章亦春) 2011-10-24 22:27:09 +08:00
Родитель b71a0b52a6
Коммит f5a4f7243e
6 изменённых файлов: 340 добавлений и 128 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
src/ngx_http_lua_common.h
Просмотреть файл

@ -69,9 +69,7 @@ typedef struct {
ngx_int_t regex_cache_entries;
ngx_int_t regex_cache_max_entries;
/* FIXME we should define an ngx_hash_t object here because we can have
* multiple shared_dict zones */
ngx_shm_zone_t *shm_zone;
ngx_array_t *shm_zones; /* of ngx_shm_zone_t* */
unsigned postponed_to_rewrite_phase_end:1;
unsigned postponed_to_access_phase_end:1;

2
src/ngx_http_lua_conf.c
Просмотреть файл

@ -25,7 +25,7 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf)
* lmcf->lua_path = { 0, NULL };
* lmcf->lua_cpath = { 0, NULL };
* lmcf->regex_cache_entries = 0;
* lmcf->shm_zone = NULL;
* lmcf->shm_zones = NULL;
*/
lmcf->pool = cf->pool;

27
src/ngx_http_lua_directive.c
Просмотреть файл

@ -34,12 +34,18 @@ ngx_http_lua_shared_dict(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
ngx_http_lua_main_conf_t *lmcf = conf;
ngx_str_t *value, name;
ngx_shm_zone_t *shm_zone;
ngx_shm_zone_t *zone;
ngx_shm_zone_t **zp;
ngx_http_lua_shdict_ctx_t *ctx;
ssize_t size;
if (lmcf->shm_zone != NULL) {
return "is duplicate";
if (lmcf->shm_zones == NULL) {
lmcf->shm_zones = ngx_palloc(cf->pool, sizeof(ngx_array_t));
if (lmcf->shm_zones == NULL) {
return NGX_CONF_ERROR;
}
ngx_array_init(lmcf->shm_zones, cf->pool, 2, sizeof(ngx_shm_zone_t *));
}
value = cf->args->elts;
@ -69,14 +75,14 @@ ngx_http_lua_shared_dict(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
ctx->name = name;
shm_zone = ngx_shared_memory_add(cf, &name, (size_t) size,
zone = ngx_shared_memory_add(cf, &name, (size_t) size,
&ngx_http_lua_module);
if (shm_zone == NULL) {
if (zone == NULL) {
return NGX_CONF_ERROR;
}
if (shm_zone->data) {
ctx = shm_zone->data;
if (zone->data) {
ctx = zone->data;
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"lua_shared_dict \"%V\" is already defined as \"%V\"",
@ -84,10 +90,11 @@ ngx_http_lua_shared_dict(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
return NGX_CONF_ERROR;
}
shm_zone->init = ngx_http_lua_shdict_init_zone;
shm_zone->data = ctx;
zone->init = ngx_http_lua_shdict_init_zone;
zone->data = ctx;
lmcf->shm_zone = shm_zone;
zp = ngx_array_push(lmcf->shm_zones);
*zp = zone;
return NGX_CONF_OK;
}

223
src/ngx_http_lua_shdict.c
Просмотреть файл

@ -231,15 +231,16 @@ ngx_http_lua_inject_shdict_api(ngx_conf_t *cf, lua_State *L)
{
ngx_http_lua_main_conf_t *lmcf;
ngx_http_lua_shdict_ctx_t *ctx;
ngx_uint_t i;
ngx_shm_zone_t **zone;
lmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_lua_module);
if (lmcf->shm_zone != NULL) {
ctx = lmcf->shm_zone->data;
if (lmcf->shm_zones != NULL) {
lua_createtable(L, 0, 1 /* nrec */); /* ngx.shared */
lua_createtable(L, 0, 2 /* nrec */); /* shared mt */
lua_pushcfunction(L, ngx_http_lua_shdict_get);
lua_setfield(L, -2, "get");
lua_pushcfunction(L, ngx_http_lua_shdict_set);
@ -247,17 +248,20 @@ ngx_http_lua_inject_shdict_api(ngx_conf_t *cf, lua_State *L)
lua_pushvalue(L, -1); /* shared mt mt */
lua_setfield(L, -2, "__index"); /* shared mt */
/* FIXME we should iterate through all the shared_dict objects here */
zone = lmcf->shm_zones->elts;
lua_pushlstring(L, (char *) ctx->name.data, ctx->name.len);
/* shared mt key */
for (i = 0; i < lmcf->shm_zones->nelts; i++) {
ctx = zone[i]->data;
lua_pushlstring(L, (char *) ctx->name.data, ctx->name.len);
/* shared mt key */
lua_pushlightuserdata(L, zone[i]); /* shared mt key ud */
lua_pushvalue(L, -3); /* shared mt key ud mt */
lua_setmetatable(L, -2); /* shared mt key ud */
lua_rawset(L, -4); /* shared mt */
}
lua_createtable(L, 0, 1 /* nrec */); /* shared mt key tb */
lua_pushvalue(L, -2); /* shared mt key tb key */
lua_setfield(L, -2, "name"); /* shared mt key tb */
lua_pushvalue(L, -3); /* shared mt key tb mt */
lua_setmetatable(L, -2); /* shared mt key tb */
lua_rawset(L, -4); /* shared mt */
lua_pop(L, 1); /* shared */
} else {
@ -271,8 +275,6 @@ ngx_http_lua_inject_shdict_api(ngx_conf_t *cf, lua_State *L)
static int
ngx_http_lua_shdict_get(lua_State *L)
{
ngx_http_lua_main_conf_t *lmcf;
ngx_http_request_t *r;
int n;
ngx_str_t name;
ngx_str_t key;
@ -285,6 +287,7 @@ ngx_http_lua_shdict_get(lua_State *L)
int value_type;
lua_Number num;
u_char c;
ngx_shm_zone_t *zone;
n = lua_gettop(L);
@ -293,31 +296,16 @@ ngx_http_lua_shdict_get(lua_State *L)
"but only seen %d", n);
}
if (!lua_istable(L, 1)) {
return luaL_error(L, "the first argument must be the "
"shared_dict object");
luaL_checktype(L, 1, LUA_TLIGHTUSERDATA);
zone = lua_touserdata(L, 1);
if (zone == NULL) {
return luaL_error(L, "bad user data for the ngx_shm_zone_t pointer");
}
lua_getfield(L, 1, "name");
ctx = zone->data;
if (!lua_isstring(L, -1)) {
return luaL_error(L, "the shared_dict object must take a non-empty "
"\"name\" attribute");
}
name.data = (u_char *) lua_tolstring(L, -1, &name.len);
lua_getglobal(L, GLOBALS_SYMBOL_REQUEST);
r = lua_touserdata(L, -1);
lua_pop(L, 2);
lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module);
if (lmcf->shm_zone == NULL) {
return luaL_error(L, "cannot find shm zone for shared_dict %s",
name.data);
}
name = ctx->name;
key.data = (u_char *) luaL_checklstring(L, 2, &key.len);
@ -334,17 +322,15 @@ ngx_http_lua_shdict_get(lua_State *L)
hash = ngx_crc32_short(key.data, key.len);
ctx = lmcf->shm_zone->data;
dd("looking up key %s in shared dict %s", key.data, name.data);
ngx_shmtx_lock(&ctx->shpool->mutex);
#if 0
#if 1
ngx_http_lua_shdict_expire(ctx, 1);
#endif
rc = ngx_http_lua_shdict_lookup(lmcf->shm_zone, hash, key.data, key.len,
rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len,
&sd);
ngx_shmtx_unlock(&ctx->shpool->mutex);
@ -403,7 +389,6 @@ ngx_http_lua_shdict_get(lua_State *L)
static int
ngx_http_lua_shdict_set(lua_State *L)
{
ngx_http_lua_main_conf_t *lmcf;
ngx_http_request_t *r;
int n;
ngx_str_t name;
@ -420,6 +405,10 @@ ngx_http_lua_shdict_set(lua_State *L)
u_char *p;
ngx_rbtree_node_t *node;
ngx_time_t *tp;
ngx_shm_zone_t *zone;
int override = 0;
/* indicates whether to override other valid entries */
n = lua_gettop(L);
@ -428,31 +417,20 @@ ngx_http_lua_shdict_set(lua_State *L)
"but only seen %d", n);
}
if (!lua_istable(L, 1)) {
return luaL_error(L, "the first argument must be the "
"shared_dict object");
luaL_checktype(L, 1, LUA_TLIGHTUSERDATA);
zone = lua_touserdata(L, 1);
if (zone == NULL) {
return luaL_error(L, "bad user data for the ngx_shm_zone_t pointer");
}
lua_getfield(L, 1, "name");
ctx = zone->data;
if (!lua_isstring(L, -1)) {
return luaL_error(L, "the shared_dict object must take a non-empty "
"\"name\" attribute");
}
name.data = (u_char *) lua_tolstring(L, -1, &name.len);
name = ctx->name;
lua_getglobal(L, GLOBALS_SYMBOL_REQUEST);
r = lua_touserdata(L, -1);
lua_pop(L, 2);
lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module);
if (lmcf->shm_zone == NULL) {
return luaL_error(L, "cannot find shm zone for shared_dict %s",
name.data);
}
lua_pop(L, 1);
key.data = (u_char *) luaL_checklstring(L, 2, &key.len);
@ -501,70 +479,94 @@ ngx_http_lua_shdict_set(lua_State *L)
}
}
ctx = lmcf->shm_zone->data;
dd("looking up key %s in shared dict %s", key.data, name.data);
ngx_shmtx_lock(&ctx->shpool->mutex);
#if 0
#if 1
ngx_http_lua_shdict_expire(ctx, 1);
#endif
rc = ngx_http_lua_shdict_lookup(lmcf->shm_zone, hash, key.data, key.len,
rc = ngx_http_lua_shdict_lookup(zone, hash, key.data, key.len,
&sd);
if (rc == NGX_DECLINED) {
if (rc == NGX_OK || rc == NGX_DONE) {
n = offsetof(ngx_rbtree_node_t, color)
+ offsetof(ngx_http_lua_shdict_node_t, data)
+ key.len;
if (value.len == sd->value_len) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"lua shared dict set: found old entry and value size matched, "
"reusing it");
ngx_queue_remove(&sd->queue);
ngx_queue_insert_head(&ctx->sh->queue, &sd->queue);
sd->key_len = key.len;
tp = ngx_timeofday();
sd->expires = (ngx_msec_t) (tp->sec * 1000 + tp->msec)
+ (ngx_msec_t) (exptime * 1000);
sd->value_len = value.len;
sd->value_type = value_type;
p = ngx_copy(sd->data, key.data, key.len);
ngx_memcpy(p, value.data, value.len);
ngx_shmtx_unlock(&ctx->shpool->mutex);
lua_pushboolean(L, override);
return 1;
}
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"lua shared dict set: found old entry bug value size NOT matched, "
"removing it first");
ngx_queue_remove(&sd->queue);
node = (ngx_rbtree_node_t *)
((u_char *) sd - offsetof(ngx_rbtree_node_t, color));
ngx_rbtree_delete(&ctx->sh->rbtree, node);
ngx_slab_free_locked(ctx->shpool, node);
}
/* rc == NGX_DECLINED or value size unmatch */
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"lua shared dict set: creating a new entry");
n = offsetof(ngx_rbtree_node_t, color)
+ offsetof(ngx_http_lua_shdict_node_t, data)
+ key.len
+ value.len;
node = ngx_slab_alloc_locked(ctx->shpool, n);
if (node == NULL) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"lua shared dict set: overriding non-expired items due to memory "
"shortage for entry \"%V\"", &name);
override = 1;
ngx_http_lua_shdict_expire(ctx, 0);
node = ngx_slab_alloc_locked(ctx->shpool, n);
if (node == NULL) {
ngx_shmtx_unlock(&ctx->shpool->mutex);
ngx_http_lua_shdict_expire(ctx, 0);
node = ngx_slab_alloc_locked(ctx->shpool, n);
if (node == NULL) {
ngx_shmtx_unlock(&ctx->shpool->mutex);
return luaL_error(L, "failed to allocate memory for "
"shared_dict %s (maybe the total capacity is too small?)",
name.data);
}
return luaL_error(L, "failed to allocate memory for "
"shared_dict %s (maybe the total capacity is too small?)",
name.data);
}
sd = (ngx_http_lua_shdict_node_t *) &node->color;
node->key = hash;
sd->key_len = key.len;
tp = ngx_timeofday();
sd->expires = (ngx_msec_t) (tp->sec * 1000 + tp->msec)
+ (ngx_msec_t) (exptime * 1000);
sd->value_len = value.len;
sd->value_type = value_type;
p = ngx_copy(sd->data, key.data, key.len);
ngx_memcpy(p, value.data, value.len);
ngx_rbtree_insert(&ctx->sh->rbtree, node);
ngx_queue_insert_head(&ctx->sh->queue, &sd->queue);
ngx_shmtx_unlock(&ctx->shpool->mutex);
return 0;
}
/* rc == NGX_OK || rc == NGX_DONE */
ngx_shmtx_unlock(&ctx->shpool->mutex);
ngx_queue_remove(&sd->queue);
ngx_queue_insert_head(&ctx->sh->queue, &sd->queue);
sd = (ngx_http_lua_shdict_node_t *) &node->color;
node->key = hash;
sd->key_len = key.len;
tp = ngx_timeofday();
@ -577,6 +579,13 @@ ngx_http_lua_shdict_set(lua_State *L)
p = ngx_copy(sd->data, key.data, key.len);
ngx_memcpy(p, value.data, value.len);
return 0;
ngx_rbtree_insert(&ctx->sh->rbtree, node);
ngx_queue_insert_head(&ctx->sh->queue, &sd->queue);
ngx_shmtx_unlock(&ctx->shpool->mutex);
lua_pushboolean(L, override);
return 1;
}

206
t/043-shdict.t
Просмотреть файл

@ -4,11 +4,11 @@ use Test::Nginx::Socket;
#worker_connections(1014);
#master_process_enabled(1);
log_level('warn');
#log_level('warn');
repeat_each(2);
#repeat_each(2);
plan tests => repeat_each() * (blocks() * 2);
plan tests => repeat_each() * (blocks() * 2 + 2);
#no_diff();
#no_long_string();
@ -18,7 +18,7 @@ run_tests();
__DATA__
=== TEST 1: short sanity
=== TEST 1: string key, int value
--- http_config
lua_shared_dict dogs 1m;
--- config
@ -26,11 +26,209 @@ __DATA__
content_by_lua '
local dogs = ngx.shared.dogs
dogs:set("foo", 32)
dogs:set("bah", 10502)
ngx.say(dogs:get("foo"))
ngx.say(dogs:get("bah"))
';
}
--- request
GET /test
--- response_body
32
10502
=== TEST 2: string key, floating-point value
--- http_config
lua_shared_dict cats 1m;
--- config
location = /test {
content_by_lua '
local cats = ngx.shared.cats
cats:set("foo", 3.14159)
cats:set("baz", 1.28)
cats:set("baz", 3.96)
ngx.say(cats:get("foo"))
ngx.say(cats:get("baz"))
';
}
--- request
GET /test
--- response_body
3.14159
3.96
=== TEST 3: string key, boolean value
--- http_config
lua_shared_dict cats 1m;
--- config
location = /test {
content_by_lua '
local cats = ngx.shared.cats
cats:set("foo", true)
cats:set("bar", false)
ngx.say(cats:get("foo"))
ngx.say(cats:get("bar"))
';
}
--- request
GET /test
--- response_body
true
false
=== TEST 4: number keys, string values
--- http_config
lua_shared_dict cats 1m;
--- config
location = /test {
content_by_lua '
local cats = ngx.shared.cats
ngx.say(cats:set(1234, "cat"))
ngx.say(cats:set("1234", "dog"))
ngx.say(cats:set(256, "bird"))
ngx.say(cats:get(1234))
ngx.say(cats:get("1234"))
ngx.say(cats:get("256"))
';
}
--- request
GET /test
--- response_body
false
false
false
dog
dog
bird
=== TEST 5: different-size values set to the same key
--- http_config
lua_shared_dict dogs 1m;
--- config
location = /test {
content_by_lua '
local dogs = ngx.shared.dogs
dogs:set("foo", "hello")
ngx.say(dogs:get("foo"))
dogs:set("foo", "hello, world")
ngx.say(dogs:get("foo"))
dogs:set("foo", "hello")
ngx.say(dogs:get("foo"))
';
}
--- request
GET /test
--- response_body
hello
hello, world
hello
=== TEST 6: expired entries
--- http_config
lua_shared_dict dogs 1m;
--- config
location = /test {
content_by_lua '
local dogs = ngx.shared.dogs
dogs:set("foo", 32, 0.01)
ngx.location.capture("/sleep/0.01")
ngx.say(dogs:get("foo"))
';
}
location ~ '^/sleep/(.+)' {
echo_sleep $1;
}
--- request
GET /test
--- response_body
nil
=== TEST 7: not yet expired entries
--- http_config
lua_shared_dict dogs 1m;
--- config
location = /test {
content_by_lua '
local dogs = ngx.shared.dogs
dogs:set("foo", 32, 0.5)
ngx.location.capture("/sleep/0.01")
ngx.say(dogs:get("foo"))
';
}
location ~ '^/sleep/(.+)' {
echo_sleep $1;
}
--- request
GET /test
--- response_body
32
=== TEST 8: forcibly override other valid entries
--- http_config
lua_shared_dict dogs 100k;
--- config
location = /test {
content_by_lua '
local dogs = ngx.shared.dogs
local i = 0
while i < 1000 do
i = i + 1
local val = string.rep(" hello " .. i, 10)
local override = dogs:set("key_" .. i, val, 0.02)
if override then
break
end
end
ngx.say("abort at ", i)
ngx.say("value: ", dogs:get("key_" .. i))
';
}
--- pipelined_requests eval
["GET /test", "GET /test"]
--- response_body eval
["abort at 353\nvalue: " . (" hello 353" x 10) . "\n",
"abort at 1\nvalue: " . (" hello 1" x 10) . "\n"
]
=== TEST 9: dogs and cats dicts
--- http_config
lua_shared_dict dogs 1m;
lua_shared_dict cats 1m;
--- config
location = /test {
content_by_lua '
local dogs = ngx.shared.dogs
local cats = ngx.shared.cats
dogs:set("foo", 32)
cats:set("foo", "hello, world")
ngx.say(dogs:get("foo"))
ngx.say(cats:get("foo"))
dogs:set("foo", 56)
ngx.say(dogs:get("foo"))
ngx.say(cats:get("foo"))
';
}
--- request
GET /test
--- response_body
32
hello, world
56
hello, world

6
util/build2.sh
Просмотреть файл

@ -15,7 +15,7 @@ force=$2
ngx-build $force $version \
--add-module=$root/../ndk-nginx-module \
--add-module=$root/../set-misc-nginx-module \
--with-cc-opt=$'-O3' \
--with-cc-opt=$'-O2' \
--with-ld-opt="-Wl,-rpath,/opt/drizzle/lib:/usr/local/lib:/home/lz/lib:/usr/local/openresty/luajit/lib" \
--without-mail_pop3_module \
--without-mail_imap_module \
@ -35,6 +35,6 @@ ngx-build $force $version \
--add-module=$root/../drizzle-nginx-module \
--add-module=$home/work/nginx/ngx_http_upstream_keepalive-2ce9d8a1ca93 \
--add-module=$root/../rds-json-nginx-module \
$opts \
--with-debug
$opts #\
#--with-debug