From be1fe529794b1b8e4b6080d7ac5b71ae610d77da Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 28 Nov 2018 10:58:42 -0600 Subject: [PATCH] Add Elixir tests for database partitions Co-authored-by: Garren Smith Co-authored-by: Robert Newson --- test/elixir/lib/couch/db_test.ex | 12 +- test/elixir/test/partition_crud_test.exs | 320 ++++++++++ .../test/partition_design_docs_test.exs | 16 + test/elixir/test/partition_helpers.exs | 76 +++ test/elixir/test/partition_mango_test.exs | 591 ++++++++++++++++++ test/elixir/test/partition_size_test.exs | 135 ++++ test/elixir/test/partition_view_test.exs | 299 +++++++++ .../test/partition_view_update_test.exs | 73 +++ test/elixir/test/test_helper.exs | 1 + 9 files changed, 1520 insertions(+), 3 deletions(-) create mode 100644 test/elixir/test/partition_crud_test.exs create mode 100644 test/elixir/test/partition_design_docs_test.exs create mode 100644 test/elixir/test/partition_helpers.exs create mode 100644 test/elixir/test/partition_mango_test.exs create mode 100644 test/elixir/test/partition_size_test.exs create mode 100644 test/elixir/test/partition_view_test.exs create mode 100644 test/elixir/test/partition_view_update_test.exs diff --git a/test/elixir/lib/couch/db_test.ex b/test/elixir/lib/couch/db_test.ex index 8992376350b..ba65a6d4eab 100644 --- a/test/elixir/lib/couch/db_test.ex +++ b/test/elixir/lib/couch/db_test.ex @@ -18,6 +18,12 @@ defmodule Couch.DBTest do |> Map.put(:db_name, random_db_name(db_name)) |> Map.put(:with_db, true) + %{:with_partitioned_db => true} -> + context + |> Map.put(:db_name, random_db_name()) + |> Map.put(:query, %{partitioned: true}) + |> Map.put(:with_db, true) + %{:with_db => true} -> Map.put(context, :db_name, random_db_name()) @@ -29,7 +35,7 @@ defmodule Couch.DBTest do end if Map.has_key?(context, :with_db) do - {:ok, _} = create_db(context[:db_name]) + {:ok, _} = create_db(context[:db_name], query: context[:query]) on_exit(fn -> delete_db(context[:db_name]) end) end @@ -154,8 +160,8 @@ defmodule Couch.DBTest do Map.put(user_doc, "_rev", resp.body["rev"]) end - def create_db(db_name) do - resp = Couch.put("/#{db_name}") + def create_db(db_name, opts \\ []) do + resp = Couch.put("/#{db_name}", opts) assert resp.status_code in [201, 202] assert resp.body == %{"ok" => true} {:ok, resp} diff --git a/test/elixir/test/partition_crud_test.exs b/test/elixir/test/partition_crud_test.exs new file mode 100644 index 00000000000..fc3af4a153e --- /dev/null +++ b/test/elixir/test/partition_crud_test.exs @@ -0,0 +1,320 @@ +defmodule PartitionCrudTest do + use CouchTestCase + + @tag :with_partitioned_db + test "Sets partition in db info", context do + db_name = context[:db_name] + resp = Couch.get("/#{db_name}") + %{body: body} = resp + assert body["props"] == %{"partitioned" => true} + end + + @tag :with_partitioned_db + test "PUT and GET document", context do + db_name = context[:db_name] + id = "my-partition:doc" + url = "/#{db_name}/#{id}" + + resp = Couch.put(url, body: %{partitioned_doc: true}) + %{body: doc} = resp + assert resp.status_code == 201 + assert doc["id"] == id + + resp = Couch.get(url) + assert resp.status_code == 200 + + %{body: doc} = resp + assert doc["_id"] == id + end + + @tag :with_partitioned_db + test "PUT fails if a partition key is not supplied", context do + db_name = context[:db_name] + id = "not-partitioned" + url = "/#{db_name}/#{id}" + + resp = Couch.put(url, body: %{partitioned_doc: false}) + assert resp.status_code == 400 + + error = %{ + "error" => "illegal_docid", + "reason" => "Doc id must be of form partition:id" + } + + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "PUT fails for partitions with _", context do + db_name = context[:db_name] + id = "_bad:partitioned" + url = "/#{db_name}/#{id}" + + resp = Couch.put(url, body: %{partitioned_doc: false}) + + error = %{ + "error" => "illegal_docid", + "reason" => "Only reserved document ids may start with underscore." + } + + assert resp.status_code == 400 + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "PUT fails for bad partitions", context do + db_name = context[:db_name] + id = "bad:" + url = "/#{db_name}/#{id}" + + resp = Couch.put(url, body: %{partitioned_doc: false}) + + error = %{ + "error" => "illegal_docid", + "reason" => "Document id must not be empty" + } + + assert resp.status_code == 400 + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "POST and GET document", context do + db_name = context[:db_name] + id = "my-partition-post:doc" + url = "/#{db_name}" + + resp = Couch.post(url, body: %{_id: id, partitioned_doc: true}) + assert resp.status_code == 201 + + resp = Couch.get("#{url}/#{id}") + assert resp.status_code == 200 + + %{body: doc} = resp + assert doc["_id"] == id + end + + @tag :with_partitioned_db + test "POST and _bulk_get document", context do + db_name = context[:db_name] + id = "my-partition-post:doc" + url = "/#{db_name}" + + resp = Couch.post(url, body: %{_id: id, partitioned_doc: true}) + assert resp.status_code == 201 + + resp = Couch.post("#{url}/_bulk_get", body: %{docs: [%{id: id}]}) + assert resp.status_code == 200 + + %{body: body} = resp + + assert %{ + "results" => [ + %{ + "docs" => [ + %{ + "ok" => %{ + "_id" => "my-partition-post:doc", + "_rev" => "1-43d86359741cb629c0953a2beb6e9d7a", + "partitioned_doc" => true + } + } + ], + "id" => "my-partition-post:doc" + } + ] + } == body + end + + @tag :with_partitioned_db + test "_bulk_get bad partitioned document", context do + db_name = context[:db_name] + id = "my-partition-post" + url = "/#{db_name}" + + resp = Couch.post("#{url}/_bulk_get", body: %{docs: [%{id: id}]}) + assert resp.status_code == 200 + %{:body => body} = resp + + assert %{ + "results" => [ + %{ + "docs" => [ + %{ + "error" => %{ + "error" => "illegal_docid", + "id" => "my-partition-post", + "reason" => "Doc id must be of form partition:id", + "rev" => :null + } + } + ], + "id" => "my-partition-post" + } + ] + } == body + end + + @tag :with_partitioned_db + test "POST fails if a partition key is not supplied", context do + db_name = context[:db_name] + id = "not-partitioned-post" + url = "/#{db_name}" + + resp = Couch.post(url, body: %{_id: id, partitited_doc: false}) + assert resp.status_code == 400 + end + + @tag :with_partitioned_db + test "_bulk_docs saves docs with partition key", context do + db_name = context[:db_name] + + docs = [ + %{_id: "foo:1"}, + %{_id: "bar:1"} + ] + + url = "/#{db_name}" + resp = Couch.post("#{url}/_bulk_docs", body: %{:docs => docs}) + assert resp.status_code == 201 + + resp = Couch.get("#{url}/foo:1") + assert resp.status_code == 200 + + resp = Couch.get("#{url}/bar:1") + assert resp.status_code == 200 + end + + @tag :with_partitioned_db + test "_bulk_docs errors with missing partition key", context do + db_name = context[:db_name] + + docs = [ + %{_id: "foo1"} + ] + + error = %{ + "error" => "illegal_docid", + "reason" => "Doc id must be of form partition:id" + } + + url = "/#{db_name}" + resp = Couch.post("#{url}/_bulk_docs", body: %{:docs => docs}) + assert resp.status_code == 400 + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "_bulk_docs errors with bad partition key", context do + db_name = context[:db_name] + + docs = [ + %{_id: "_foo:1"} + ] + + error = %{ + "error" => "illegal_docid", + "reason" => "Only reserved document ids may start with underscore." + } + + url = "/#{db_name}" + resp = Couch.post("#{url}/_bulk_docs", body: %{:docs => docs}) + assert resp.status_code == 400 + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "_bulk_docs errors with bad doc key", context do + db_name = context[:db_name] + + docs = [ + %{_id: "foo:"} + ] + + error = %{ + "error" => "illegal_docid", + "reason" => "Document id must not be empty" + } + + url = "/#{db_name}" + resp = Couch.post("#{url}/_bulk_docs", body: %{:docs => docs}) + assert resp.status_code == 400 + assert Map.get(resp, :body) == error + end + + @tag :with_partitioned_db + test "saves attachment with partitioned doc", context do + db_name = context[:db_name] + id = "foo:doc-with-attachment" + + doc = %{ + _id: id, + _attachments: %{ + "foo.txt": %{ + content_type: "text/plain", + data: Base.encode64("This is a text document to save") + } + } + } + + resp = Couch.put("/#{db_name}/#{id}", body: doc) + + assert resp.status_code == 201 + + resp = Couch.get("/#{db_name}/#{id}") + assert resp.status_code == 200 + body = Map.get(resp, :body) + rev = Map.get(body, "_rev") + + assert body["_attachments"] == %{ + "foo.txt" => %{ + "content_type" => "text/plain", + "digest" => "md5-OW2BoZAtMqs1E+fAnLpNBw==", + "length" => 31, + "revpos" => 1, + "stub" => true + } + } + + resp = Couch.get("/#{db_name}/#{id}/foo.txt") + assert Map.get(resp, :body) == "This is a text document to save" + + resp = + Couch.put("/#{db_name}/#{id}/bar.txt?rev=#{rev}", + headers: ["Content-Type": "text/plain"], + body: "This is another document" + ) + + assert resp.status_code == 201 + %{:body => body} = resp + assert body["ok"] == true + assert body["id"] == id + end + + test "create database with bad `partitioned` value", _context do + resp = Couch.put("/bad-db?partitioned=tru") + assert resp.status_code == 400 + + assert Map.get(resp, :body) == %{ + "error" => "bad_request", + "reason" => "Invalid `partitioned` parameter" + } + end + + test "can create unpartitioned system db", _context do + Couch.delete("/_replicator") + resp = Couch.put("/_replicator") + assert resp.status_code == 201 + assert resp.body == %{"ok" => true} + end + + test "cannot create partitioned system db", _context do + Couch.delete("/_replicator") + + resp = Couch.put("/_replicator?partitioned=true") + assert resp.status_code == 400 + + %{:body => %{"reason" => reason}} = resp + assert Regex.match?(~r/Cannot partition a system database/, reason) + end +end diff --git a/test/elixir/test/partition_design_docs_test.exs b/test/elixir/test/partition_design_docs_test.exs new file mode 100644 index 00000000000..42a2ced7726 --- /dev/null +++ b/test/elixir/test/partition_design_docs_test.exs @@ -0,0 +1,16 @@ +defmodule PartitionDesignDocsTest do + use CouchTestCase + + @moduledoc """ + Test Partition functionality for partition design docs + """ + + @tag :with_partitioned_db + test "/_partition/:pk/_design/doc 404", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/fake-key/_design/mrtest/" + resp = Couch.get(url) + assert resp.status_code == 404 + end +end diff --git a/test/elixir/test/partition_helpers.exs b/test/elixir/test/partition_helpers.exs new file mode 100644 index 00000000000..6eac2b1a49e --- /dev/null +++ b/test/elixir/test/partition_helpers.exs @@ -0,0 +1,76 @@ +defmodule PartitionHelpers do + use ExUnit.Case + + def create_partition_docs(db_name, pk1 \\ "foo", pk2 \\ "bar") do + docs = + for i <- 1..100 do + id = + if rem(i, 2) == 0 do + "#{pk1}:#{i}" + else + "#{pk2}:#{i}" + end + + group = + if rem(i, 3) == 0 do + "one" + else + "two" + end + + %{ + :_id => id, + :value => i, + :some => "field", + :group => group + } + end + + resp = Couch.post("/#{db_name}/_bulk_docs", body: %{:w => 3, :docs => docs}) + assert resp.status_code == 201 + end + + def create_partition_ddoc(db_name, opts \\ %{}) do + map_fn = """ + function(doc) { + if (doc.some) { + emit(doc.value, doc.some); + } + } + """ + + default_ddoc = %{ + views: %{ + some: %{ + map: map_fn + } + } + } + + ddoc = Enum.into(opts, default_ddoc) + + resp = Couch.put("/#{db_name}/_design/mrtest", body: ddoc) + assert resp.status_code == 201 + assert Map.has_key?(resp.body, "ok") == true + end + + def get_ids(resp) do + %{:body => %{"rows" => rows}} = resp + Enum.map(rows, fn row -> row["id"] end) + end + + def get_partitions(resp) do + %{:body => %{"rows" => rows}} = resp + + Enum.map(rows, fn row -> + [partition, _] = String.split(row["id"], ":") + partition + end) + end + + def assert_correct_partition(partitions, correct_partition) do + assert Enum.all?(partitions, fn partition -> + partition == correct_partition + end) + end +end diff --git a/test/elixir/test/partition_mango_test.exs b/test/elixir/test/partition_mango_test.exs new file mode 100644 index 00000000000..1471ddb0ace --- /dev/null +++ b/test/elixir/test/partition_mango_test.exs @@ -0,0 +1,591 @@ +defmodule PartitionMangoTest do + use CouchTestCase + import PartitionHelpers, except: [get_partitions: 1] + + @moduledoc """ + Test Partition functionality for mango + """ + def create_index(db_name, fields \\ ["some"], opts \\ %{}) do + default_index = %{ + index: %{ + fields: fields + } + } + + index = Enum.into(opts, default_index) + resp = Couch.post("/#{db_name}/_index", body: index) + + assert resp.status_code == 200 + assert resp.body["result"] == "created" + end + + def get_partitions(resp) do + %{:body => %{"docs" => docs}} = resp + + Enum.map(docs, fn doc -> + [partition, _] = String.split(doc["_id"], ":") + partition + end) + end + + @tag :with_partitioned_db + test "query using _id and partition works", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + _id: %{ + "$gt": "foo:" + } + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + _id: %{ + "$lt": "foo:" + } + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "bar") + end + + @tag :with_partitioned_db + test "query using _id works for global and local query", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + _id: %{ + "$gt": 0 + } + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + _id: %{ + "$gt": 0 + } + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "bar") + end + + @tag :with_partitioned_db + test "query with partitioned:true using index and $eq", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_partition/bar/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "bar") + end + + @tag :with_partitioned_db + test "partitioned query using _all_docs with $eq", context do + db_name = context[:db_name] + create_partition_docs(db_name) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_partition/bar/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + limit: 20 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 20 + assert_correct_partition(partitions, "bar") + end + + @tag :with_db + test "non-partitioned query using _all_docs and $eq", context do + db_name = context[:db_name] + create_partition_docs(db_name) + + url = "/#{db_name}/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + skip: 40, + limit: 5 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert partitions == ["bar", "bar", "bar", "bar", "bar"] + + url = "/#{db_name}/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + }, + skip: 50, + limit: 5 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert partitions == ["foo", "foo", "foo", "foo", "foo"] + end + + @tag :with_partitioned_db + test "partitioned query using index and range scan", context do + db_name = context[:db_name] + create_partition_docs(db_name, "foo", "bar42") + create_index(db_name, ["value"]) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_partition/bar42/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "bar42") + end + + @tag :with_partitioned_db + test "partitioned query using _all_docs and range scan", context do + db_name = context[:db_name] + create_partition_docs(db_name) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_partition/bar/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "bar") + end + + @tag :with_partitioned_db + test "partitioned query using _all_docs", context do + db_name = context[:db_name] + create_partition_docs(db_name, "foo", "bar42") + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "foo") + + url = "/#{db_name}/_partition/bar42/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert_correct_partition(partitions, "bar42") + end + + @tag :with_partitioned_db + test "explain works with partitions", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["some"]) + + url = "/#{db_name}/_partition/foo/_explain" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + %{:body => body} = resp + + assert body["index"]["name"] == "_all_docs" + assert body["mrargs"]["partition"] == "foo" + + url = "/#{db_name}/_partition/bar/_explain" + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + } + } + ) + + %{:body => body} = resp + + assert body["index"]["def"] == %{"fields" => [%{"some" => "asc"}]} + assert body["mrargs"]["partition"] == "bar" + end + + @tag :with_db + test "explain works with non partitioned db", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["some"]) + + url = "/#{db_name}/_explain" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + } + } + ) + + %{:body => body} = resp + + assert body["index"]["name"] == "_all_docs" + assert body["mrargs"]["partition"] == :null + + resp = + Couch.post(url, + body: %{ + selector: %{ + some: "field" + } + } + ) + + %{:body => body} = resp + + assert body["index"]["def"] == %{"fields" => [%{"some" => "asc"}]} + assert body["mrargs"]["partition"] == :null + end + + @tag :with_partitioned_db + test "partitioned query using bookmarks", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["value"]) + + url = "/#{db_name}/_partition/foo/_find" + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + }, + limit: 3 + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 3 + assert_correct_partition(partitions, "foo") + + %{:body => %{"bookmark" => bookmark}} = resp + + resp = + Couch.post(url, + body: %{ + selector: %{ + value: %{ + "$gte": 6, + "$lt": 16 + } + }, + limit: 3, + bookmark: bookmark + } + ) + + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 2 + assert_correct_partition(partitions, "foo") + end + + @tag :with_partitioned_db + test "global query uses global index", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["some"], %{partitioned: false}) + + url = "/#{db_name}/_explain" + + selector = %{ + selector: %{ + some: "field" + }, + limit: 100 + } + + resp = Couch.post(url, body: selector) + assert resp.status_code == 200 + %{:body => body} = resp + assert body["index"]["def"] == %{"fields" => [%{"some" => "asc"}]} + + url = "/#{db_name}/_find" + resp = Couch.post(url, body: selector) + assert resp.status_code == 200 + + partitions = get_partitions(resp) + assert length(partitions) == 100 + end + + @tag :with_partitioned_db + test "global query does not use partition index", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["some"]) + + url = "/#{db_name}/_explain" + + selector = %{ + selector: %{ + some: "field" + }, + limit: 100 + } + + resp = Couch.post(url, body: selector) + %{:body => body} = resp + assert body["index"]["name"] == "_all_docs" + + url = "/#{db_name}/_find" + resp = Couch.post(url, body: selector) + + assert resp.status_code == 200 + + partitions = get_partitions(resp) + assert length(partitions) == 100 + end + + @tag :with_partitioned_db + test "partitioned query does not use global index", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_index(db_name, ["some"], %{partitioned: false}) + + url = "/#{db_name}/_partition/foo/_explain" + + selector = %{ + selector: %{ + some: "field" + }, + limit: 50 + } + + resp = Couch.post(url, body: selector) + assert resp.status_code == 200 + %{:body => body} = resp + assert body["index"]["name"] == "_all_docs" + + url = "/#{db_name}/_partition/foo/_find" + resp = Couch.post(url, body: selector) + assert resp.status_code == 200 + + partitions = get_partitions(resp) + assert length(partitions) == 50 + assert_correct_partition(partitions, "foo") + end +end diff --git a/test/elixir/test/partition_size_test.exs b/test/elixir/test/partition_size_test.exs new file mode 100644 index 00000000000..91e79f7b310 --- /dev/null +++ b/test/elixir/test/partition_size_test.exs @@ -0,0 +1,135 @@ +defmodule PartitionSizeTest do + use CouchTestCase + import PartitionHelpers + + @moduledoc """ + Test Partition size functionality + """ + @tag :with_partitioned_db + test "get partition size", context do + db_name = context[:db_name] + create_partition_docs(db_name) + + info_resp = Couch.get("/#{db_name}") + %{:body => info} = info_resp + external_size = info["sizes"]["external"] + + url = "/#{db_name}/_partition/foo" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 50 + assert body["partition"] == "foo" + assert body["sizes"]["external"] == external_size / 2 + + url = "/#{db_name}/_partition/bar" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 50 + assert body["partition"] == "bar" + assert body["sizes"]["external"] == external_size / 2 + end + + @tag :with_partitioned_db + test "get partition size for two partitions on same shard", context do + db_name = context[:db_name] + create_partition_docs(db_name, "foo", "bar42") + + info_resp = Couch.get("/#{db_name}") + %{:body => info} = info_resp + external_size = info["sizes"]["external"] + + url = "/#{db_name}/_partition/foo" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 50 + assert body["partition"] == "foo" + assert body["sizes"]["external"] == external_size / 2 + + url = "/#{db_name}/_partition/bar42" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 50 + assert body["partition"] == "bar42" + assert body["sizes"]["external"] == external_size / 2 + end + + @tag :with_partitioned_db + test "get partition size with attachment", context do + db_name = context[:db_name] + # create_partition_docs(db_name, "foo", "bar42") + + id = "foo:doc-with-attachment" + + doc = %{ + _id: id, + _attachments: %{ + "foo.txt": %{ + content_type: "text/plain", + data: Base.encode64("This is a text document to save") + } + } + } + + Couch.put("/#{db_name}/#{id}", body: doc) + + info_resp = Couch.get("/#{db_name}") + %{:body => info} = info_resp + external_size = info["sizes"]["external"] + + url = "/#{db_name}/_partition/foo" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 1 + assert body["partition"] == "foo" + assert body["sizes"]["external"] == external_size + end + + @tag :with_partitioned_db + test "get multiple partition sizes with attachment", context do + db_name = context[:db_name] + create_partition_docs(db_name, "foo", "bar42") + + id = "foo:doc-with-attachment" + + doc = %{ + _id: id, + _attachments: %{ + "foo.txt": %{ + content_type: "text/plain", + data: Base.encode64("This is a text document to save") + } + } + } + + Couch.put("/#{db_name}/#{id}", body: doc) + + url = "/#{db_name}/_partition/foo" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 51 + assert body["partition"] == "foo" + # hard coding in values here. Not sure if this is a good idea long term + assert body["sizes"]["external"] == 4503 + + url = "/#{db_name}/_partition/bar42" + resp = Couch.get(url) + + assert resp.status_code == 200 + %{:body => body} = resp + assert body["doc_count"] == 50 + assert body["partition"] == "bar42" + assert body["sizes"]["external"] == 4450 + end +end diff --git a/test/elixir/test/partition_view_test.exs b/test/elixir/test/partition_view_test.exs new file mode 100644 index 00000000000..a255391725b --- /dev/null +++ b/test/elixir/test/partition_view_test.exs @@ -0,0 +1,299 @@ +defmodule ViewPartitionTest do + use CouchTestCase + import PartitionHelpers + + @moduledoc """ + Test Partition functionality for views + """ + + setup_all do + db_name = random_db_name() + {:ok, _} = create_db(db_name, query: %{partitioned: true, q: 1}) + on_exit(fn -> delete_db(db_name) end) + + create_partition_docs(db_name) + + map_fun1 = """ + function(doc) { + if (doc.some) { + emit(doc.value, doc.some); + } + } + """ + + map_fun2 = """ + function(doc) { + if (doc.group) { + emit([doc.some, doc.group], 1); + } + } + """ + + query = %{:w => 3} + + body = %{ + :docs => [ + %{ + _id: "_design/map", + views: %{some: %{map: map_fun1}} + }, + %{ + _id: "_design/map_some", + views: %{some: %{map: map_fun2}} + }, + %{ + _id: "_design/partitioned_true", + views: %{some: %{map: map_fun1}}, + options: %{partitioned: true} + }, + %{ + _id: "_design/partitioned_false", + views: %{some: %{map: map_fun1}}, + options: %{partitioned: false} + }, + %{ + _id: "_design/reduce", + views: %{some: %{map: map_fun2, reduce: "_count"}} + }, + %{ + _id: "_design/include_ddocs", + views: %{some: %{map: map_fun1}}, + options: %{include_design: true} + } + ] + } + + resp = Couch.post("/#{db_name}/_bulk_docs", query: query, body: body) + Enum.each(resp.body, &assert(&1["ok"])) + + {:ok, [db_name: db_name]} + end + + def get_reduce_result(resp) do + %{:body => %{"rows" => rows}} = resp + rows + end + + test "query with partitioned:true returns partitioned fields", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/partitioned_true/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert Enum.dedup(partitions) == ["foo"] + + url = "/#{db_name}/_partition/bar/_design/partitioned_true/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert Enum.dedup(partitions) == ["bar"] + end + + test "default view query returns partitioned fields", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert Enum.dedup(partitions) == ["foo"] + + url = "/#{db_name}/_partition/bar/_design/map/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert Enum.dedup(partitions) == ["bar"] + end + + test "query will return zero results for wrong inputs", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{start_key: "\"foo:12\""}) + assert resp.status_code == 200 + assert Map.get(resp, :body)["rows"] == [] + end + + test "partitioned ddoc cannot be used in global query", context do + db_name = context[:db_name] + + url = "/#{db_name}/_design/map/_view/some" + resp = Couch.get(url) + %{:body => %{"reason" => reason}} = resp + assert resp.status_code == 400 + assert Regex.match?(~r/mandatory for queries to this view./, reason) + end + + test "partitioned query cannot be used with global ddoc", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/partitioned_false/_view/some" + resp = Couch.get(url) + %{:body => %{"reason" => reason}} = resp + assert resp.status_code == 400 + assert Regex.match?(~r/is not supported in this design doc/, reason) + end + + test "view query returns all docs for global query", context do + db_name = context[:db_name] + + url = "/#{db_name}/_design/partitioned_false/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 100 + end + + test "partition query errors with incorrect partition supplied", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/_bar/_design/map/_view/some" + resp = Couch.get(url) + assert resp.status_code == 400 + + url = "/#{db_name}/_partition//_design/map/_view/some" + resp = Couch.get(url) + assert resp.status_code == 400 + end + + test "partitioned query works with startkey, endkey range", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{start_key: 12, end_key: 20}) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert Enum.dedup(partitions) == ["foo"] + end + + test "partitioned query works with keys", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.post(url, body: %{keys: [2, 4, 6]}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 3 + assert ids == ["foo:2", "foo:4", "foo:6"] + end + + test "global query works with keys", context do + db_name = context[:db_name] + + url = "/#{db_name}/_design/partitioned_false/_view/some" + resp = Couch.post(url, body: %{keys: [2, 4, 6]}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 3 + assert ids == ["foo:2", "foo:4", "foo:6"] + end + + test "partition query works with limit", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{limit: 5}) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 5 + assert Enum.dedup(partitions) == ["foo"] + end + + test "partition query with descending", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{descending: true, limit: 5}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 5 + assert ids == ["foo:100", "foo:98", "foo:96", "foo:94", "foo:92"] + + resp = Couch.get(url, query: %{descending: false, limit: 5}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 5 + assert ids == ["foo:2", "foo:4", "foo:6", "foo:8", "foo:10"] + end + + test "partition query with skip", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{skip: 5, limit: 5}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 5 + assert ids == ["foo:12", "foo:14", "foo:16", "foo:18", "foo:20"] + end + + test "partition query with key", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map/_view/some" + resp = Couch.get(url, query: %{key: 22}) + assert resp.status_code == 200 + ids = get_ids(resp) + assert length(ids) == 1 + assert ids == ["foo:22"] + end + + test "partition query with startkey_docid and endkey_docid", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/map_some/_view/some" + + resp = + Couch.get(url, + query: %{ + startkey: "[\"field\",\"one\"]", + endkey: "[\"field\",\"one\"]", + startkey_docid: "foo:12", + endkey_docid: "foo:30" + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert ids == ["foo:12", "foo:18", "foo:24", "foo:30"] + end + + test "query with reduce works", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/reduce/_view/some" + resp = Couch.get(url, query: %{reduce: true, group_level: 1}) + assert resp.status_code == 200 + results = get_reduce_result(resp) + assert results == [%{"key" => ["field"], "value" => 50}] + + resp = Couch.get(url, query: %{reduce: true, group_level: 2}) + results = get_reduce_result(resp) + + assert results == [ + %{"key" => ["field", "one"], "value" => 16}, + %{"key" => ["field", "two"], "value" => 34} + ] + + resp = Couch.get(url, query: %{reduce: true, group: true}) + results = get_reduce_result(resp) + + assert results == [ + %{"key" => ["field", "one"], "value" => 16}, + %{"key" => ["field", "two"], "value" => 34} + ] + end + + test "include_design works correctly", context do + db_name = context[:db_name] + + url = "/#{db_name}/_partition/foo/_design/include_ddocs/_view/some" + resp = Couch.get(url) + assert resp.status_code == 200 + partitions = get_partitions(resp) + assert length(partitions) == 50 + assert Enum.dedup(partitions) == ["foo"] + end +end diff --git a/test/elixir/test/partition_view_update_test.exs b/test/elixir/test/partition_view_update_test.exs new file mode 100644 index 00000000000..1fc98392657 --- /dev/null +++ b/test/elixir/test/partition_view_update_test.exs @@ -0,0 +1,73 @@ +defmodule PartitionViewUpdateTest do + use CouchTestCase + import PartitionHelpers + + @moduledoc """ + Test Partition view update functionality + """ + @tag :with_partitioned_db + test "view updates properly remove old keys", context do + db_name = context[:db_name] + create_partition_docs(db_name, "foo", "bar") + create_partition_ddoc(db_name) + + check_key = fn key, num_rows -> + url = "/#{db_name}/_partition/foo/_design/mrtest/_view/some" + resp = Couch.get(url, query: [key: key]) + assert resp.status_code == 200 + assert length(resp.body["rows"]) == num_rows + end + + check_key.(2, 1) + + resp = Couch.get("/#{db_name}/foo:2") + doc = Map.put(resp.body, "value", 4) + resp = Couch.put("/#{db_name}/foo:2", query: [w: 3], body: doc) + assert resp.status_code >= 201 and resp.status_code <= 202 + + check_key.(4, 2) + check_key.(2, 0) + end + + @tag :with_partitioned_db + test "query with update=false works", context do + db_name = context[:db_name] + create_partition_docs(db_name) + create_partition_ddoc(db_name) + + url = "/#{db_name}/_partition/foo/_design/mrtest/_view/some" + + resp = + Couch.get(url, + query: %{ + update: "true", + limit: 3 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert ids == ["foo:2", "foo:4", "foo:6"] + + # Avoid race conditions by attempting to get a full response + # from every shard before we do our update:false test + for _ <- 1..12 do + resp = Couch.get(url) + assert resp.status_code == 200 + end + + Couch.put("/#{db_name}/foo:1", body: %{some: "field"}) + + resp = + Couch.get(url, + query: %{ + update: "false", + limit: 3 + } + ) + + assert resp.status_code == 200 + ids = get_ids(resp) + assert ids == ["foo:2", "foo:4", "foo:6"] + end +end diff --git a/test/elixir/test/test_helper.exs b/test/elixir/test/test_helper.exs index 33041fd02bb..d6843eb2181 100644 --- a/test/elixir/test/test_helper.exs +++ b/test/elixir/test/test_helper.exs @@ -1,2 +1,3 @@ ExUnit.configure(exclude: [pending: true]) ExUnit.start() +Code.require_file("partition_helpers.exs", __DIR__)