feature: ngx.balancer: added the pool_size argument to enable_keepalive().

lib/ngx/balancer.lua

@@ -39,7 +39,8 @@ if subsystem == 'http' then
         char **err);
 
     int ngx_http_lua_ffi_balancer_enable_keepalive(ngx_http_request_t *r,
-        unsigned long timeout, unsigned int max_requests, char **err);
+        unsigned long timeout, unsigned int max_requests,
+        unsigned int pool_size, char **err);
 
     int ngx_http_lua_ffi_balancer_set_more_tries(ngx_http_request_t *r,
         int count, char **err);
@@ -201,7 +202,7 @@ else
 end
 
 if subsystem == "http" then
-    function _M.enable_keepalive(idle_timeout, max_requests)
+    function _M.enable_keepalive(idle_timeout, max_requests, pool_size)
         local r = get_request()
         if not r then
             error("no request found")
@@ -232,8 +233,20 @@ if subsystem == "http" then
             error("bad argument #2 to 'enable_keepalive' (expected >= 0)", 2)
         end
 
+        if not pool_size then
+            pool_size = 0
+
+        elseif type(pool_size) ~= "number" then
+            error("bad argument #3 to 'enable_keepalive' " ..
+                  "(number expected, got " .. type(pool_size) .. ")", 2)
+
+        elseif pool_size < 0 then
+            error("bad argument #3 to 'enable_keepalive' (expected >= 0)", 2)
+        end
+
         local rc = ngx_lua_ffi_balancer_enable_keepalive(r, idle_timeout,
-                                                         max_requests, errmsg)
+                                                         max_requests,
+                                                         pool_size, errmsg)
         if rc == FFI_OK then
             return true
         end

lib/ngx/balancer.md

@@ -191,14 +191,20 @@ In case of an error, this function returns `nil` and a string describing the err
 
 enable_keepalive
 ----------------
-**syntax:** `ok, err = balancer.enable_keepalive(idle_timeout?, max_requests?)`
+**syntax:** `ok, err = balancer.enable_keepalive(idle_timeout?, max_requests?, pool_size?)`
 
 **context:** *balancer_by_lua**
 
 Instructs the current upstream connection to be kept-alive once the current
-request succeeds. The connection will be inserted in the pool specified by the
-`pool` option of the [set_current_peer](#set_current_peer) function (if
-unspecified, the default pool name will be `"<host>:<port>"`).
+request succeeds. The connection will be inserted in the pool of the current
+peer, which is identified by the peer address, the `host` argument of
+[set_current_peer](#set_current_peer) (for HTTPS upstreams, the name set by
+`proxy_ssl_name` when `host` is `nil`) and the local address, if any (e.g.
+bound by [bind_to_local_addr](#bind_to_local_addr) or `proxy_bind`). All the
+pools of an upstream block share the capacity set by the
+[balancer_keepalive](https://github.com/openresty/lua-nginx-module#balancer_keepalive)
+directive (`32` by default): once it is reached, the least recently used idle
+connection, whichever pool it belongs to, is closed to make room.
 
 The keepalive capabilities offered via this function are similar to that of the
 [keepalive](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive)
@@ -229,6 +235,18 @@ This argument is identical to the
 directive of the ngx_http_upstream_module, but can be set dynamically for each
 upstream connection.
 
+The third optional argument `pool_size` may be a number used to specify the
+maximum amount of idle connections cached for the pool of the current
+connection. When the connection is about to be inserted into a pool which
+already holds `pool_size` idle connections, the least recently used idle
+connection of that same pool is closed first, so that one busy pool cannot
+evict the idle connections of the other pools. If omitted or `0`, no per-pool
+limit is applied. `pool_size` never raises the total capacity set by
+`balancer_keepalive`, so it only takes effect when it is smaller than that
+capacity. The limit is checked against the value passed by the request whose
+connection is being inserted, so all the requests using the same pool should
+pass the same `pool_size`.
+
 This function returns `true` upon success, or `nil` and a string describing the
 error otherwise.
 

t/balancer-keepalive-localaddr.t

@@ -453,3 +453,58 @@ lua balancer: keepalive saving connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_N
 lua balancer: keepalive reusing connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
 lua balancer: keepalive saving connection [0-9A-F]+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
 $/
+
+
+
+=== TEST 8: pool_size applies per local address
+--- http_upstream
+    upstream test_upstream {
+        server 0.0.0.1;
+        keepalive 0;
+
+        balancer_by_lua_block {
+            local b = require "ngx.balancer"
+
+            local ok, err = b.set_current_peer("127.0.0.2",
+                                               $TEST_NGINX_SERVER_SSL_PORT,
+                                               "test.com")
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set current peer: ", err)
+                return
+            end
+
+            ok, err = b.bind_to_local_addr(ngx.var.arg_src_ip)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set local addr: ", err)
+                return
+            end
+
+            local ok, err = b.enable_keepalive(nil, nil, 1)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set keepalive: ", err)
+                return
+            end
+        }
+    }
+--- config
+    location = /t {
+        echo_subrequest GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.11';
+        echo_subrequest_async GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.10';
+        echo_subrequest_async GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.10';
+        echo_sleep 0.3;
+        echo_subrequest GET '/proxy/echo_client_addr' -q 'src_ip=127.0.0.11';
+    }
+--- response_body
+127.0.0.11
+127.0.0.10
+127.0.0.10
+127.0.0.11
+--- grep_error_log eval
+qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: [\w.]+/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
+lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
+lua balancer: keepalive no free connection, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
+lua balancer: keepalive reusing connection \S+, host: 127.0.0.2:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: test.com
+$/

t/balancer-keepalive.t

@@ -1288,3 +1288,222 @@ qr/keepalive reusing connection[^\n]*/
 qr/keepalive reusing connection [0-9A-F]+, host: 127.0.0.1:\d+, name: keepalive1.longname.test.host.com
 keepalive reusing connection [0-9A-F]+, host: 127.0.0.2:\d+, name: keepalive2.longname.test.host.com
 /
+
+
+
+=== TEST 29: enable_keepalive: bad argument #3 'pool_size'
+--- config
+    location = /t {
+        content_by_lua_block {
+            local b = require "ngx.balancer"
+
+            local values = { true, "1", -1 }
+
+            for _, val in ipairs(values) do
+                local pok, perr = pcall(b.enable_keepalive, nil, nil, val)
+                if not pok then
+                    ngx.say(perr)
+                end
+            end
+        }
+    }
+--- response_body
+bad argument #3 to 'enable_keepalive' (number expected, got boolean)
+bad argument #3 to 'enable_keepalive' (number expected, got string)
+bad argument #3 to 'enable_keepalive' (expected >= 0)
+--- grep_error_log_out
+
+
+
+=== TEST 30: enable_keepalive: 'pool_size' caps idle connections of a pool
+--- http_upstream
+    upstream test_upstream {
+        server 0.0.0.1;
+        keepalive 0;
+
+        balancer_by_lua_block {
+            local b = require "ngx.balancer"
+
+            local ok, err = b.set_current_peer("127.0.0.1",
+                                               $TEST_NGINX_SERVER_SSL_PORT,
+                                               "one")
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set current peer: ", err)
+                return
+            end
+
+            local ok, err = b.enable_keepalive(nil, nil, 1)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set keepalive: ", err)
+                return
+            end
+        }
+    }
+--- config
+    location = /t {
+        echo_subrequest_async GET '/proxy/';
+        echo_subrequest_async GET '/proxy/';
+        echo_subrequest_async GET '/proxy/';
+        echo_sleep 0.3;
+        echo_subrequest_async GET '/proxy/';
+        echo_subrequest_async GET '/proxy/';
+    }
+--- response_body
+--- grep_error_log eval
+qr/lua balancer: keepalive (?:no free|reusing|saving|pool full)[^\n]*name: \w+/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive saving connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+$/
+
+
+
+=== TEST 31: enable_keepalive: a full pool does not evict idle connections of other pools
+--- http_upstream
+    upstream test_upstream {
+        server 0.0.0.1;
+        keepalive 0;
+        balancer_keepalive 2;
+
+        balancer_by_lua_block {
+            local b = require "ngx.balancer"
+
+            local ok, err = b.set_current_peer("127.0.0.1",
+                                               $TEST_NGINX_SERVER_SSL_PORT,
+                                               ngx.var.arg_pool)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set current peer: ", err)
+                return
+            end
+
+            local ok, err = b.enable_keepalive(nil, nil, 1)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set keepalive: ", err)
+                return
+            end
+        }
+    }
+--- config
+    location = /t {
+        echo_subrequest GET '/proxy/' -q 'pool=b';
+        echo_subrequest_async GET '/proxy/' -q 'pool=a';
+        echo_subrequest_async GET '/proxy/' -q 'pool=a';
+        echo_sleep 0.3;
+        echo_subrequest GET '/proxy/' -q 'pool=b';
+    }
+--- response_body
+--- grep_error_log eval
+qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a
+lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b
+$/
+
+
+
+=== TEST 32: enable_keepalive: 'pool_size' 0 behaves like no pool_size
+--- http_upstream
+    upstream test_upstream {
+        server 0.0.0.1;
+        keepalive 0;
+        balancer_keepalive 2;
+
+        balancer_by_lua_block {
+            local b = require "ngx.balancer"
+
+            local ok, err = b.set_current_peer("127.0.0.1",
+                                               $TEST_NGINX_SERVER_SSL_PORT,
+                                               ngx.var.arg_pool)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set current peer: ", err)
+                return
+            end
+
+            local ok, err = b.enable_keepalive(nil, nil, 0)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set keepalive: ", err)
+                return
+            end
+        }
+    }
+--- config
+    location = /t {
+        echo_subrequest GET '/proxy/' -q 'pool=b';
+        echo_subrequest_async GET '/proxy/' -q 'pool=a';
+        echo_subrequest_async GET '/proxy/' -q 'pool=a';
+        echo_sleep 0.3;
+        echo_subrequest GET '/proxy/' -q 'pool=b';
+    }
+--- response_body
+--- grep_error_log eval
+qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: a
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: b
+$/
+
+
+
+=== TEST 33: enable_keepalive: 'pool_size' applies to pools split by SNI
+--- http_upstream
+    upstream test_upstream {
+        server 0.0.0.1;
+        keepalive 0;
+        balancer_keepalive 2;
+
+        balancer_by_lua_block {
+            local b = require "ngx.balancer"
+
+            -- no host: the pool is named after proxy_ssl_name
+            local ok, err = b.set_current_peer("127.0.0.1",
+                                               $TEST_NGINX_SERVER_SSL_PORT)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set current peer: ", err)
+                return
+            end
+
+            local ok, err = b.enable_keepalive(nil, nil, 1)
+            if not ok then
+                ngx.log(ngx.ERR, "failed to set keepalive: ", err)
+                return
+            end
+        }
+    }
+--- config
+    location = /t {
+        echo_subrequest GET '/proxy/echo_sni' -q 'sni=two';
+        echo_subrequest_async GET '/proxy/echo_sni' -q 'sni=one';
+        echo_subrequest_async GET '/proxy/echo_sni' -q 'sni=one';
+        echo_sleep 0.3;
+        echo_subrequest GET '/proxy/echo_sni' -q 'sni=two';
+    }
+--- response_body
+SNI=two
+SNI=one
+SNI=one
+SNI=two
+--- grep_error_log eval
+qr/lua balancer: keepalive (?:no free|reusing|pool full)[^\n]*name: \w+/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: two
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive no free connection, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive pool full, evicting connection \S+, pool size: 1, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: one
+lua balancer: keepalive reusing connection \S+, host: 127.0.0.1:$ENV{TEST_NGINX_SERVER_SSL_PORT}, name: two
+$/