diff --git a/lib/ngx/balancer.lua b/lib/ngx/balancer.lua index 18bdc2cbf..9ca45d202 100644 --- a/lib/ngx/balancer.lua +++ b/lib/ngx/balancer.lua @@ -39,7 +39,8 @@ if subsystem == 'http' then char **err); int ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r, - unsigned long timeout, unsigned int max_requests, char **err); + unsigned long timeout, unsigned int max_requests, + unsigned int pool_size, char **err); int ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r, int count, char **err); @@ -201,7 +202,7 @@ else end if subsystem == "http" then - function _M.enable_keepalive(idle_timeout, max_requests) + function _M.enable_keepalive(idle_timeout, max_requests, pool_size) local r = get_request() if not r then error("no request found") @@ -232,8 +233,20 @@ if subsystem == "http" then error("bad argument #2 to 'enable_keepalive' (expected >= 0)", 2) end + if not pool_size then + pool_size = 0 + + elseif type(pool_size) ~= "number" then + error("bad argument #3 to 'enable_keepalive' " .. + "(number expected, got " .. type(pool_size) .. ")", 2) + + elseif pool_size < 0 then + error("bad argument #3 to 'enable_keepalive' (expected >= 0)", 2) + end + local rc = ngx_lua_ffi_balancer_enable_keepalive(r, idle_timeout, - max_requests, errmsg) + max_requests, + pool_size, errmsg) if rc == FFI_OK then return true end diff --git a/lib/ngx/balancer.md b/lib/ngx/balancer.md index 6c14212aa..5e85a143d 100644 --- a/lib/ngx/balancer.md +++ b/lib/ngx/balancer.md @@ -191,14 +191,20 @@ In case of an error, this function returns `nil` and a string describing the err enable_keepalive ---------------- -**syntax:** `ok, err = balancer.enable_keepalive(idle_timeout?, max_requests?)` +**syntax:** `ok, err = balancer.enable_keepalive(idle_timeout?, max_requests?, pool_size?)` **context:** *balancer_by_lua** Instructs the current upstream connection to be kept-alive once the current -request succeeds. The connection will be inserted in the pool specified by the -`pool` option of the [set_current_peer](#set_current_peer) function (if -unspecified, the default pool name will be `":"`). +request succeeds. The connection will be inserted in the pool of the current +peer, which is identified by the peer address, the `host` argument of +[set_current_peer](#set_current_peer) (for HTTPS upstreams, the name set by +`proxy_ssl_name` when `host` is `nil`) and the local address, if any (e.g. +bound by [bind_to_local_addr](#bind_to_local_addr) or `proxy_bind`). All the +pools of an upstream block share the capacity set by the +[balancer_keepalive](https://github.com/openresty/lua-nginx-module#balancer_keepalive) +directive (`32` by default): once it is reached, the least recently used idle +connection, whichever pool it belongs to, is closed to make room. The keepalive capabilities offered via this function are similar to that of the [keepalive](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) @@ -229,6 +235,18 @@ This argument is identical to the directive of the ngx_http_upstream_module, but can be set dynamically for each upstream connection. +The third optional argument `pool_size` may be a number used to specify the +maximum amount of idle connections cached for the pool of the current +connection. When the connection is about to be inserted into a pool which +already holds `pool_size` idle connections, the least recently used idle +connection of that same pool is closed first, so that one busy pool cannot +evict the idle connections of the other pools. If omitted or `0`, no per-pool +limit is applied. `pool_size` never raises the total capacity set by +`balancer_keepalive`, so it only takes effect when it is smaller than that +capacity. The limit is checked against the value passed by the request whose +connection is being inserted, so all the requests using the same pool should +pass the same `pool_size`. + This function returns `true` upon success, or `nil` and a string describing the error otherwise. diff --git a/t/balancer-keepalive-localaddr.t b/t/balancer-keepalive-localaddr.t index 74f8ee8e8..3da8ef158 100644 --- a/t/balancer-keepalive-localaddr.t +++ b/t/balancer-keepalive-localaddr.t @@ -453,3 +453,58 @@ lua balancer: keepalive saving connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_N lua balancer: keepalive reusing connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com lua balancer: keepalive saving connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com $/ + + + +=== TEST 8: pool_size applies per local address +--- http_upstream + upstream test_upstream { + server 0.0.0.1; + keepalive 0; + + balancer_by_lua_block { + local b = require "ngx.balancer" + + local ok, err = b.set_current_peer("127.0.0.2", + $TEST_NGINX_SERVER_SSL_PORT, + "test.com") + if not ok then + ngx.log(ngx.ERR, "failed to set current peer: ", err) + return + end + + ok, err = b.bind_to_local_addr(ngx.var.arg_src_ip) + if not ok then + ngx.log(ngx.ERR, "failed to set local addr: ", err) + return + end + + local ok, err = b.enable_keepalive(nil, nil, 1) + if not ok then + ngx.log(ngx.ERR, "failed to set keepalive: ", err) + return + end + } + } +--- config + location = /t { + echo_subrequest GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.11'; + echo_subrequest_async GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.10'; + echo_subrequest_async GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.10'; + echo_sleep 0.3; + echo_subrequest GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.11'; + } +--- response_body +127.0.0.11 +127.0.0.10 +127.0.0.10 +127.0.0.11 +--- grep_error_log eval +qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: [\w.]+/ +--- grep_error_log_out eval +qr/^lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com +lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com +lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com +lua balancer: keepalive reusing connection \S+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com +$/ diff --git a/t/balancer-keepalive.t b/t/balancer-keepalive.t index d45e6f2d6..90110d31f 100644 --- a/t/balancer-keepalive.t +++ b/t/balancer-keepalive.t @@ -1288,3 +1288,222 @@ qr/keepalive reusing connection[^\n]*/ qr/keepalive reusing connection [0-9A-F]+, host: 127.0.0.1:\d+, name: keepalive1.longname.test.host.com keepalive reusing connection [0-9A-F]+, host: 127.0.0.2:\d+, name: keepalive2.longname.test.host.com / + + + +=== TEST 29: enable_keepalive: bad argument #3 'pool_size' +--- config + location = /t { + content_by_lua_block { + local b = require "ngx.balancer" + + local values = { true, "1", -1 } + + for _, val in ipairs(values) do + local pok, perr = pcall(b.enable_keepalive, nil, nil, val) + if not pok then + ngx.say(perr) + end + end + } + } +--- response_body +bad argument #3 to 'enable_keepalive' (number expected, got boolean) +bad argument #3 to 'enable_keepalive' (number expected, got string) +bad argument #3 to 'enable_keepalive' (expected >= 0) +--- grep_error_log_out + + + +=== TEST 30: enable_keepalive: 'pool_size' caps idle connections of a pool +--- http_upstream + upstream test_upstream { + server 0.0.0.1; + keepalive 0; + + balancer_by_lua_block { + local b = require "ngx.balancer" + + local ok, err = b.set_current_peer("127.0.0.1", + $TEST_NGINX_SERVER_SSL_PORT, + "one") + if not ok then + ngx.log(ngx.ERR, "failed to set current peer: ", err) + return + end + + local ok, err = b.enable_keepalive(nil, nil, 1) + if not ok then + ngx.log(ngx.ERR, "failed to set keepalive: ", err) + return + end + } + } +--- config + location = /t { + echo_subrequest_async GET '/proxy/'; + echo_subrequest_async GET '/proxy/'; + echo_subrequest_async GET '/proxy/'; + echo_sleep 0.3; + echo_subrequest_async GET '/proxy/'; + echo_subrequest_async GET '/proxy/'; + } +--- response_body +--- grep_error_log eval +qr/lua balancer: keepalive (?:no free|reusing|saving|pool full)[^\n]*name: \w+/ +--- grep_error_log_out eval +qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +$/ + + + +=== TEST 31: enable_keepalive: a full pool does not evict idle connections of other pools +--- http_upstream + upstream test_upstream { + server 0.0.0.1; + keepalive 0; + balancer_keepalive 2; + + balancer_by_lua_block { + local b = require "ngx.balancer" + + local ok, err = b.set_current_peer("127.0.0.1", + $TEST_NGINX_SERVER_SSL_PORT, + ngx.var.arg_pool) + if not ok then + ngx.log(ngx.ERR, "failed to set current peer: ", err) + return + end + + local ok, err = b.enable_keepalive(nil, nil, 1) + if not ok then + ngx.log(ngx.ERR, "failed to set keepalive: ", err) + return + end + } + } +--- config + location = /t { + echo_subrequest GET '/proxy/' -q 'pool=b'; + echo_subrequest_async GET '/proxy/' -q 'pool=a'; + echo_subrequest_async GET '/proxy/' -q 'pool=a'; + echo_sleep 0.3; + echo_subrequest GET '/proxy/' -q 'pool=b'; + } +--- response_body +--- grep_error_log eval +qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/ +--- grep_error_log_out eval +qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a +lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b +$/ + + + +=== TEST 32: enable_keepalive: 'pool_size' 0 behaves like no pool_size +--- http_upstream + upstream test_upstream { + server 0.0.0.1; + keepalive 0; + balancer_keepalive 2; + + balancer_by_lua_block { + local b = require "ngx.balancer" + + local ok, err = b.set_current_peer("127.0.0.1", + $TEST_NGINX_SERVER_SSL_PORT, + ngx.var.arg_pool) + if not ok then + ngx.log(ngx.ERR, "failed to set current peer: ", err) + return + end + + local ok, err = b.enable_keepalive(nil, nil, 0) + if not ok then + ngx.log(ngx.ERR, "failed to set keepalive: ", err) + return + end + } + } +--- config + location = /t { + echo_subrequest GET '/proxy/' -q 'pool=b'; + echo_subrequest_async GET '/proxy/' -q 'pool=a'; + echo_subrequest_async GET '/proxy/' -q 'pool=a'; + echo_sleep 0.3; + echo_subrequest GET '/proxy/' -q 'pool=b'; + } +--- response_body +--- grep_error_log eval +qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/ +--- grep_error_log_out eval +qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b +$/ + + + +=== TEST 33: enable_keepalive: 'pool_size' applies to pools split by SNI +--- http_upstream + upstream test_upstream { + server 0.0.0.1; + keepalive 0; + balancer_keepalive 2; + + balancer_by_lua_block { + local b = require "ngx.balancer" + + -- no host: the pool is named after proxy_ssl_name + local ok, err = b.set_current_peer("127.0.0.1", + $TEST_NGINX_SERVER_SSL_PORT) + if not ok then + ngx.log(ngx.ERR, "failed to set current peer: ", err) + return + end + + local ok, err = b.enable_keepalive(nil, nil, 1) + if not ok then + ngx.log(ngx.ERR, "failed to set keepalive: ", err) + return + end + } + } +--- config + location = /t { + echo_subrequest GET '/proxy/echo_sni' -q 'sni=two'; + echo_subrequest_async GET '/proxy/echo_sni' -q 'sni=one'; + echo_subrequest_async GET '/proxy/echo_sni' -q 'sni=one'; + echo_sleep 0.3; + echo_subrequest GET '/proxy/echo_sni' -q 'sni=two'; + } +--- response_body +SNI=two +SNI=one +SNI=one +SNI=two +--- grep_error_log eval +qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/ +--- grep_error_log_out eval +qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: two +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one +lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: two +$/