diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 9a332acddb8..b9d51af7194 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -293,6 +293,8 @@ os_process_limit = 100 ; Timeout for how long a response from a busy view group server can take. ; "infinity" is also a valid configuration value. ;group_info_timeout = 5000 +;query_limit = 268435456 +;partition_query_limit = 268435456 [mango] ; Set to true to disable the "index all fields" text index, which can lead diff --git a/src/couch_mrview/include/couch_mrview.hrl b/src/couch_mrview/include/couch_mrview.hrl index e17aaba9337..29fe52bdbdb 100644 --- a/src/couch_mrview/include/couch_mrview.hrl +++ b/src/couch_mrview/include/couch_mrview.hrl @@ -60,6 +60,7 @@ view_states=nil }). +-define(MAX_VIEW_LIMIT, 16#10000000). -record(mrargs, { view_type, @@ -74,7 +75,7 @@ keys, direction = fwd, - limit = 16#10000000, + limit = ?MAX_VIEW_LIMIT, skip = 0, group_level = 0, group = undefined, diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index b879d1242c8..eb68124a03e 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -496,9 +496,10 @@ fold_reduce({NthRed, Lang, View}, Fun, Acc, Options) -> couch_btree:fold_reduce(Bt, WrapperFun, Acc, Options). -validate_args(Db, DDoc, Args) -> +validate_args(Db, DDoc, Args0) -> {ok, State} = couch_mrview_index:init(Db, DDoc), - validate_args(State, Args). + Args1 = apply_limit(State#mrst.partitioned, Args0), + validate_args(State, Args1). validate_args(#mrst{} = State, Args0) -> @@ -523,6 +524,28 @@ validate_args(#mrst{} = State, Args0) -> end. +apply_limit(ViewPartitioned, Args) -> + LimitType = case ViewPartitioned of + true -> "partition_query_limit"; + false -> "query_limit" + end, + + MaxLimit = config:get_integer("query_server_config", + LimitType, ?MAX_VIEW_LIMIT), + + % Set the highest limit possible if a user has not + % specified a limit + Args1 = case Args#mrargs.limit == ?MAX_VIEW_LIMIT of + true -> Args#mrargs{limit = MaxLimit}; + false -> Args + end, + + if Args1#mrargs.limit =< MaxLimit -> Args1; true -> + Fmt = "Limit is too large, must not exceed ~p", + mrverror(io_lib:format(Fmt, [MaxLimit])) + end. + + validate_all_docs_args(Db, Args0) -> Args = validate_args(Args0), @@ -533,7 +556,8 @@ validate_all_docs_args(Db, Args0) -> {false, <<_/binary>>} -> mrverror(<<"`partition` parameter is not supported on this db">>); {_, <<_/binary>>} -> - apply_all_docs_partition(Args, Partition); + Args1 = apply_limit(true, Args), + apply_all_docs_partition(Args1, Partition); _ -> Args end. diff --git a/test/elixir/test/partition_all_docs_test.exs b/test/elixir/test/partition_all_docs_test.exs index 308c1b4393e..87bab341c5b 100644 --- a/test/elixir/test/partition_all_docs_test.exs +++ b/test/elixir/test/partition_all_docs_test.exs @@ -116,36 +116,82 @@ defmodule PartitionAllDocsTest do assert ids == ["foo:22"] end + test "partition all docs can set query limits", context do + set_config({"query_server_config", "partition_query_limit", "2000"}) + + db_name = context[:db_name] + create_partition_docs(db_name) + create_partition_ddoc(db_name) + + url = "/#{db_name}/_partition/foo/_all_docs" + + resp = + Couch.get(url, + query: %{ + limit: 20 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 20 + + resp = Couch.get(url) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 50 + + resp = + Couch.get(url, + query: %{ + limit: 2000 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 50 + + resp = + Couch.get(url, + query: %{ + limit: 2001 + } + ) + + assert resp.status_code == 400 + %{:body => %{"reason" => reason}} = resp + assert Regex.match?(~r/Limit is too large/, reason) + + resp = + Couch.get(url, + query: %{ + limit: 2000, + skip: 25 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 25 + end + # This test is timing based so it could be a little flaky. # If that turns out to be the case we should probably just skip it test "partition _all_docs with timeout", context do - on_exit(fn -> - resp = Couch.get("/_membership") - %{:body => body} = resp - - Enum.each(body["all_nodes"], fn node -> - resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"3600000\"") - assert resp.status_code == 200 - end) - end) - - resp = Couch.get("/_membership") - %{:body => body} = resp - - Enum.each(body["all_nodes"], fn node -> - resp = Couch.put("/_node/#{node}/_config/fabric/partition_view_timeout", body: "\"1\"") - assert resp.status_code == 200 - end) + set_config({"fabric", "partition_view_timeout", "1"}) db_name = context[:db_name] create_partition_docs(db_name) retry_until(fn -> url = "/#{db_name}/_partition/foo/_all_docs" + case Couch.get(url) do %{:body => %{"reason" => reason}} -> Regex.match?(~r/not be processed in a reasonable amount of time./, reason) - _ -> + + _ -> false end end) diff --git a/test/elixir/test/partition_view_test.exs b/test/elixir/test/partition_view_test.exs index a255391725b..890cb8858be 100644 --- a/test/elixir/test/partition_view_test.exs +++ b/test/elixir/test/partition_view_test.exs @@ -286,6 +286,66 @@ defmodule ViewPartitionTest do ] end + test "partition query can set query limits", context do + set_config({"query_server_config", "partition_query_limit", "2000"}) + + db_name = context[:db_name] + create_partition_docs(db_name) + create_partition_ddoc(db_name) + + url = "/#{db_name}/_partition/foo/_design/mrtest/_view/some" + + resp = + Couch.get(url, + query: %{ + limit: 20 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 20 + + resp = Couch.get(url) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 50 + + resp = + Couch.get(url, + query: %{ + limit: 2000 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 50 + + resp = + Couch.get(url, + query: %{ + limit: 2001 + } + ) + + assert resp.status_code == 400 + %{:body => %{"reason" => reason}} = resp + assert Regex.match?(~r/Limit is too large/, reason) + + resp = + Couch.get(url, + query: %{ + limit: 2000, + skip: 25 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 25 + end + test "include_design works correctly", context do db_name = context[:db_name]