From 769f0455ab20553fb03f85198a74554c0973d11f Mon Sep 17 00:00:00 2001 From: spacewander Date: Fri, 5 Feb 2021 17:05:35 +0800 Subject: [PATCH] change: default to cache DNS record according to the TTL Since lua-resty-dns-client provides an internal cache, we don't need to cache it twice. Signed-off-by: spacewander --- .../apisix_cli_test/test_validate_config.sh | 35 ++++ apisix/cli/ngx_tpl.lua | 4 +- apisix/cli/ops.lua | 6 + apisix/core/utils.lua | 6 + apisix/init.lua | 26 +-- conf/config-default.yaml | 2 +- t/coredns/db.test.local | 3 + t/node/upstream-domain-with-special-dns.t | 162 ++++++++++++++++++ t/node/upstream-node-dns.t | 142 --------------- 9 files changed, 217 insertions(+), 169 deletions(-) create mode 100755 .travis/apisix_cli_test/test_validate_config.sh diff --git a/.travis/apisix_cli_test/test_validate_config.sh b/.travis/apisix_cli_test/test_validate_config.sh new file mode 100755 index 000000000000..7d597b1c68e9 --- /dev/null +++ b/.travis/apisix_cli_test/test_validate_config.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# validate the config.yaml + +. ./.travis/apisix_cli_test/common.sh + +echo ' +apisix: + dns_resolver_valid: "/apisix" +' > conf/config.yaml + +out=$(make init 2>&1 || true) +if ! echo "$out" | grep 'dns_resolver_valid should be a number'; then + echo "failed: dns_resolver_valid should be a number" + exit 1 +fi + +echo "passed: dns_resolver_valid should be a number" diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 38081ad6f40c..466ae3e62195 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -65,7 +65,7 @@ stream { lua_shared_dict lrucache-lock-stream 10m; - resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} valid={*dns_resolver_valid*}; + resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} {% if dns_resolver_valid then %}valid={*dns_resolver_valid*}{% end %}; resolver_timeout {*resolver_timeout*}; # stream configuration snippet starts @@ -187,7 +187,7 @@ http { lua_socket_log_errors off; - resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} valid={*dns_resolver_valid*}; + resolver {% for _, dns_addr in ipairs(dns_resolver or {}) do %} {*dns_addr*} {% end %} {% if dns_resolver_valid then %}valid={*dns_resolver_valid*}{% end %}; resolver_timeout {*resolver_timeout*}; lua_http10_buffering off; diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 1f2da5ec0f99..11ae1af46c82 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -279,6 +279,12 @@ Please modify "admin_key" in conf/config.yaml . end end + if yaml_conf.apisix.dns_resolver_valid then + if tonumber(yaml_conf.apisix.dns_resolver_valid) == nil then + util.die("apisix->dns_resolver_valid should be a number") + end + end + -- Using template.render local sys_conf = { use_or_1_15 = use_or_1_15, diff --git a/apisix/core/utils.lua b/apisix/core/utils.lua index 21a40b107e60..788f3e2a2eb9 100644 --- a/apisix/core/utils.lua +++ b/apisix/core/utils.lua @@ -14,6 +14,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- +local config_local = require("apisix.core.config_local") local core_str = require("apisix.core.string") local table = require("apisix.core.table") local json = require("apisix.core.json") @@ -85,11 +86,16 @@ end local function dns_parse(domain) if dns_resolvers ~= current_inited_resolvers then + local local_conf = config_local.local_conf() + local valid = table.try_read_attr(local_conf, "apisix", "dns_resolver_valid") + local opts = { ipv6 = true, nameservers = table.clone(dns_resolvers), retrans = 5, -- 5 retransmissions on receive timeout timeout = 2000, -- 2 sec + order = {"last", "A", "AAAA", "CNAME"}, -- avoid querying SRV (we don't support it yet) + validTtl = valid, } local ok, err = dns_client.init(opts) if not ok then diff --git a/apisix/init.lua b/apisix/init.lua index e7b0d1401bcc..28a363f85ac0 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -47,7 +47,6 @@ end local load_balancer local local_conf local dns_resolver -local lru_resolved_domain local ver_header = "APISIX/" .. core.version.VERSION @@ -121,12 +120,6 @@ function _M.http_init_worker() require("apisix.upstream").init_worker() local_conf = core.config.local_conf() - local dns_resolver_valid = local_conf and local_conf.apisix and - local_conf.apisix.dns_resolver_valid - - lru_resolved_domain = core.lrucache.new({ - ttl = dns_resolver_valid, count = 512, invalid_stale = true, - }) if local_conf.apisix and local_conf.apisix.enable_server_tokens == false then ver_header = "APISIX" @@ -420,16 +413,8 @@ function _M.http_access_phase() end if upstream.has_domain then - -- try to fetch the resolved domain, if we got `nil`, - -- it means we need to create the cache by handle. - -- the `api_ctx.conf_version` is different after we called - -- `parse_domain_in_up`, need to recreate the cache by new - -- `api_ctx.conf_version` local err - upstream, err = lru_resolved_domain(upstream, - upstream.modifiedIndex, - parse_domain_in_up, - upstream) + upstream, err = parse_domain_in_up(upstream) if err then core.log.error("failed to get resolved upstream: ", err) return core.response.exit(500) @@ -454,8 +439,7 @@ function _M.http_access_phase() else if route.has_domain then local err - route, err = lru_resolved_domain(route, api_ctx.conf_version, - parse_domain_in_route, route) + route, err = parse_domain_in_route(route) if err then core.log.error("failed to get resolved route: ", err) return core.response.exit(500) @@ -806,12 +790,6 @@ function _M.stream_init_worker() load_balancer = require("apisix.balancer").run local_conf = core.config.local_conf() - local dns_resolver_valid = local_conf and local_conf.apisix and - local_conf.apisix.dns_resolver_valid - - lru_resolved_domain = core.lrucache.new({ - ttl = dns_resolver_valid, count = 512, invalid_stale = true, - }) end diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 356d3fcaae8d..c625f342a765 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -106,7 +106,7 @@ apisix: # dns_resolver: # If not set, read from `/etc/resolv.conf` # - 1.1.1.1 # - 8.8.8.8 - dns_resolver_valid: 30 # valid time for dns result 30 seconds + # dns_resolver_valid: 30 # if given, override the TTL of the valid records. The unit is second. resolver_timeout: 5 # resolver timeout ssl: enable: true diff --git a/t/coredns/db.test.local b/t/coredns/db.test.local index 1c406bff72d5..c760b69ce620 100644 --- a/t/coredns/db.test.local +++ b/t/coredns/db.test.local @@ -11,3 +11,6 @@ $ORIGIN test.local. 3600 IN NS b.iana-servers.net. ipv6 IN AAAA ::1 + +ttl 300 IN A 127.0.0.1 +ttl.1s 1 IN A 127.0.0.1 diff --git a/t/node/upstream-domain-with-special-dns.t b/t/node/upstream-domain-with-special-dns.t index e968ec85ff5b..0481ab7e4a79 100644 --- a/t/node/upstream-domain-with-special-dns.t +++ b/t/node/upstream-domain-with-special-dns.t @@ -69,3 +69,165 @@ upstreams: GET /hello --- response_body hello world + + + +=== TEST 2: default ttl +--- log_level: debug +--- apisix_yaml +upstreams: + - + id: 1 + nodes: + ttl.test.local:1980: 1 + type: roundrobin +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + for i = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + if not res or res.body ~= "hello world\n" then + ngx.say(err) + return + end + end + } + } +--- request +GET /t +--- error_log +"ttl":300 +--- grep_error_log eval +qr/connect to 127.0.0.1:1053/ +--- grep_error_log_out +connect to 127.0.0.1:1053 + + + +=== TEST 3: override ttl +--- log_level: debug +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + dns_resolver_valid: 900 +--- apisix_yaml +upstreams: + - + id: 1 + nodes: + ttl.test.local:1980: 1 + type: roundrobin +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + for i = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + if not res or res.body ~= "hello world\n" then + ngx.say(err) + return + end + end + } + } +--- request +GET /t +--- grep_error_log eval +qr/connect to 127.0.0.1:1053/ +--- grep_error_log_out +connect to 127.0.0.1:1053 +--- error_log +"ttl":900 + + + +=== TEST 4: cache expire +--- log_level: debug +--- apisix_yaml +upstreams: + - + id: 1 + nodes: + ttl.1s.test.local:1980: 1 + type: roundrobin +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + for i = 1, 2 do + for j = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + if not res or res.body ~= "hello world\n" then + ngx.say(err) + return + end + end + + if i < 2 then + ngx.sleep(1.1) + end + end + } + } +--- request +GET /t +--- grep_error_log eval +qr/connect to 127.0.0.1:1053/ +--- grep_error_log_out +connect to 127.0.0.1:1053 +connect to 127.0.0.1:1053 + + + +=== TEST 5: cache expire (override ttl) +--- log_level: debug +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false + dns_resolver_valid: 1 +--- apisix_yaml +upstreams: + - + id: 1 + nodes: + ttl.test.local:1980: 1 + type: roundrobin +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + for i = 1, 2 do + for j = 1, 3 do + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + if not res or res.body ~= "hello world\n" then + ngx.say(err) + return + end + end + + if i < 2 then + ngx.sleep(1.1) + end + end + } + } +--- request +GET /t +--- grep_error_log eval +qr/connect to 127.0.0.1:1053/ +--- grep_error_log_out +connect to 127.0.0.1:1053 +connect to 127.0.0.1:1053 diff --git a/t/node/upstream-node-dns.t b/t/node/upstream-node-dns.t index c70e60f55385..c26ccf723432 100644 --- a/t/node/upstream-node-dns.t +++ b/t/node/upstream-node-dns.t @@ -21,19 +21,6 @@ log_level('info'); no_root_location(); no_shuffle(); -our $yaml_config = <<_EOC_; -apisix: - node_listen: 1984 - admin_key: ~ - dns_resolver_valid: 1 -_EOC_ - -add_block_preprocessor(sub { - my ($block) = @_; - - $block->set_value("yaml_config", $yaml_config); -}); - run_tests(); __DATA__ @@ -120,15 +107,6 @@ location /t { local t = require("lib.test_admin").test core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) } } @@ -140,13 +118,6 @@ qr/dns resolver domain: test.com to 127.0.0.\d|call \/hello|proxy request to 127 call /hello dns resolver domain: test.com to 127.0.0.1 proxy request to 127.0.0.1:1980 -call /hello -dns resolver domain: test.com to 127.0.0.2 -proxy request to 127.0.0.2:1980 -proxy request to 127.0.0.2:1980 -call /hello -dns resolver domain: test.com to 127.0.0.3 -proxy request to 127.0.0.3:1980 @@ -234,18 +205,6 @@ location /t { core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) core.log.warn("code: ", code) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - core.log.warn("code: ", code) - local code, body = t('/hello', ngx.HTTP_GET) - core.log.warn("code: ", code) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - core.log.warn("code: ", code) } } @@ -260,19 +219,6 @@ dns resolver domain: test2.com to 127.0.0.2| dns resolver domain: test2.com to 127.0.0.1 dns resolver domain: test.com to 127.0.0.2) proxy request to 127.0.0.[12]:1980 -call \/hello( -dns resolver domain: test.com to 127.0.0.3 -dns resolver domain: test2.com to 127.0.0.4| -dns resolver domain: test2.com to 127.0.0.3 -dns resolver domain: test.com to 127.0.0.4) -proxy request to 127.0.0.[34]:1980 -proxy request to 127.0.0.[34]:1980 -call \/hello( -dns resolver domain: test.com to 127.0.0.5 -dns resolver domain: test2.com to 127.0.0.6| -dns resolver domain: test2.com to 127.0.0.5 -dns resolver domain: test.com to 127.0.0.6) -proxy request to 127.0.0.[56]:1980 / @@ -361,15 +307,6 @@ location /t { local t = require("lib.test_admin").test core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) } } @@ -381,13 +318,6 @@ qr/dns resolver domain: test.com to 127.0.0.\d|call \/hello|proxy request to 127 call /hello dns resolver domain: test.com to 127.0.0.1 proxy request to 127.0.0.1:1980 -call /hello -dns resolver domain: test.com to 127.0.0.2 -proxy request to 127.0.0.2:1980 -proxy request to 127.0.0.2:1980 -call /hello -dns resolver domain: test.com to 127.0.0.3 -proxy request to 127.0.0.3:1980 @@ -448,17 +378,6 @@ location /t { local t = require("lib.test_admin").test core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) } } @@ -473,21 +392,6 @@ dns resolver domain: test2.com to 127.0.0.2| dns resolver domain: test2.com to 127.0.0.1 dns resolver domain: test.com to 127.0.0.2) proxy request to 127.0.0.[12]:1980 -call \/hello( -dns resolver domain: test.com to 127.0.0.3 -dns resolver domain: test2.com to 127.0.0.4| -dns resolver domain: test2.com to 127.0.0.3 -dns resolver domain: test.com to 127.0.0.4) -proxy request to 127.0.0.[34]:1980 -proxy request to 127.0.0.[34]:1980 -proxy request to 127.0.0.[34]:1980 -proxy request to 127.0.0.[34]:1980 -call \/hello( -dns resolver domain: test2.com to 127.0.0.5 -dns resolver domain: test.com to 127.0.0.6| -dns resolver domain: test.com to 127.0.0.5 -dns resolver domain: test2.com to 127.0.0.6) -proxy request to 127.0.0.[56]:1980 / @@ -515,17 +419,6 @@ location /t { local t = require("lib.test_admin").test core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) } } @@ -540,21 +433,6 @@ dns resolver domain: test2.com to 127.0.0.1| dns resolver domain: test2.com to 127.0.0.1 dns resolver domain: test.com to 127.0.0.1) proxy request to 127.0.0.1:1980 -call \/hello( -dns resolver domain: test.com to 127.0.0.1 -dns resolver domain: test2.com to 127.0.0.1| -dns resolver domain: test2.com to 127.0.0.1 -dns resolver domain: test.com to 127.0.0.1) -proxy request to 127.0.0.1:1980 -proxy request to 127.0.0.1:1980 -proxy request to 127.0.0.1:1980 -proxy request to 127.0.0.1:1980 -call \/hello( -dns resolver domain: test.com to 127.0.0.1 -dns resolver domain: test2.com to 127.0.0.1| -dns resolver domain: test2.com to 127.0.0.1 -dns resolver domain: test.com to 127.0.0.1) -proxy request to 127.0.0.1:1980 / @@ -615,17 +493,6 @@ location /t { local t = require("lib.test_admin").test core.log.info("call /hello") local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - local code, body = t('/hello', ngx.HTTP_GET) - - ngx.sleep(1.1) -- cache expired - core.log.info("call /hello") - local code, body = t('/hello', ngx.HTTP_GET) } } @@ -637,13 +504,4 @@ qr/dns resolver domain: \w+.com to 127.0.0.\d|call \/hello|proxy request to 127. qr/call \/hello dns resolver domain: test.com to 127.0.0.1 proxy request to 127.0.0.(1:1980|5:1981) -call \/hello -dns resolver domain: test.com to 127.0.0.2 -proxy request to 127.0.0.(2:1980|5:1981) -proxy request to 127.0.0.(2:1980|5:1981) -proxy request to 127.0.0.(2:1980|5:1981) -proxy request to 127.0.0.(2:1980|5:1981) -call \/hello -dns resolver domain: test.com to 127.0.0.3 -proxy request to 127.0.0.(3:1980|5:1981) /