From d046adcfa9419371381d8bbe36ef2277a2b7c172 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Wed, 17 Apr 2019 09:54:51 +0200 Subject: [PATCH 01/15] switches to using pooled connections via poolboy --- lib/bigtable.ex | 10 +++++++- lib/connection/connection.ex | 23 +++++++++++------ lib/connection/worker.ex | 39 +++++++++++++++++++++++++++++ lib/utils.ex | 27 ++++++++++++-------- mix.exs | 5 ++-- mix.lock | 1 + test/connection/connection_test.exs | 2 +- 7 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 lib/connection/worker.ex diff --git a/lib/bigtable.ex b/lib/bigtable.ex index 0a81211..c4441c4 100644 --- a/lib/bigtable.ex +++ b/lib/bigtable.ex @@ -6,8 +6,16 @@ defmodule Bigtable do @doc false def start(_type, _args) do + poolboy_config = [ + {:name, {:local, :connection_pool}}, + {:worker_module, Bigtable.Connection.Worker}, + {:size, Application.get_env(:bigtable, :pool_size, 128)}, + {:max_overflow, Application.get_env(:bigtable, :pool_overflow, 0)} + ] + children = [ - Bigtable.Supervisor + Bigtable.Supervisor, + :poolboy.child_spec(:connection_pool, poolboy_config, []) ] opts = [strategy: :one_for_one, name: Bigtable] diff --git a/lib/connection/connection.ex b/lib/connection/connection.ex index 83052cf..ebed6b2 100644 --- a/lib/connection/connection.ex +++ b/lib/connection/connection.ex @@ -19,9 +19,13 @@ defmodule Bigtable.Connection do @doc """ Returns the configured `GRPC.Channel` """ - @spec get_connection() :: GRPC.Channel.t() - def get_connection do - GenServer.call(__MODULE__, :get_connection) + @spec connect() :: GRPC.Channel.t() + def connect do + GenServer.call(__MODULE__, :connect) + end + + def disconnect(channel) do + GenServer.cast(__MODULE__, {:disconnect, channel}) end @spec get_metadata() :: Keyword.t() @@ -33,7 +37,7 @@ defmodule Bigtable.Connection do # Server Callbacks @doc false - @spec init(:ok) :: {:ok, GRPC.Channel.t()} + @spec init(:ok) :: {:ok, map()} def init(:ok) do # Fetches the url to use for Bigtable gRPC connection endpoint = @@ -46,17 +50,22 @@ defmodule Bigtable.Connection do # Connects the stub to the Bigtable gRPC server + {:ok, %{endpoint: endpoint, opts: opts}} + end + + def handle_call(:connect, _from, %{endpoint: endpoint, opts: opts} = state) do {:ok, channel} = GRPC.Stub.connect( endpoint, opts ) - {:ok, channel} + {:reply, channel, state} end - def handle_call(:get_connection, _from, state) do - {:reply, state, state} + def handle_cast({:disconnect, channel}, state) do + GRPC.Stub.disconnect(channel) + {:noreply, state} end def handle_info(_msg, state) do diff --git a/lib/connection/worker.ex b/lib/connection/worker.ex new file mode 100644 index 0000000..120161c --- /dev/null +++ b/lib/connection/worker.ex @@ -0,0 +1,39 @@ +defmodule Bigtable.Connection.Worker do + alias Bigtable.Connection + use GenServer + + def start_link(_) do + GenServer.start_link(__MODULE__, nil, []) + end + + def get_connection(pid) do + GenServer.call(pid, :get_connection) + end + + def init(_) do + Process.flag(:trap_exit, true) + {:ok, Connection.connect()} + end + + def handle_call(:get_connection, _from, state) do + {:reply, state, state} + end + + def handle_info({:EXIT, _from, reason}, state) do + disconnect(state) + {:stop, reason, state} + end + + def handle_info(_msg, state) do + {:noreply, state} + end + + def terminate(_reason, state) do + disconnect(state) + state + end + + defp disconnect(%{connection: connection}) do + Connection.disconnect(connection) + end +end diff --git a/lib/utils.ex b/lib/utils.ex index c7e5cfb..83b715e 100644 --- a/lib/utils.ex +++ b/lib/utils.ex @@ -1,15 +1,22 @@ defmodule Bigtable.Utils do @moduledoc false + alias Bigtable.Connection + alias Connection.Worker def process_request(request, request_fn, opts \\ []) do - metadata = Connection.get_metadata() - - connection = Connection.get_connection() - result = - connection - |> request_fn.(request, metadata) + :poolboy.transaction( + :connection_pool, + fn pid -> + connection = Worker.get_connection(pid) + metadata = Connection.get_metadata() + + connection + |> request_fn.(request, metadata) + end, + 10_000 + ) case result do {:ok, response, _} -> @@ -23,11 +30,11 @@ defmodule Bigtable.Utils do {:ok, response} end - {:error, error} when is_map(error) -> - {:error, Map.get(error, :message, "unknown error")} + {:error, error} -> + {:error, inspect(error)} - _ -> - {:error, "unknown error"} + error -> + {:error, inspect(error)} end end diff --git a/mix.exs b/mix.exs index 8a86563..78eb7a6 100644 --- a/mix.exs +++ b/mix.exs @@ -39,7 +39,7 @@ defmodule Bigtable.MixProject do def application do [ mod: {Bigtable, []}, - extra_applications: [:logger, :grpc] + extra_applications: [:logger, :grpc, :poolboy] ] end @@ -106,7 +106,8 @@ defmodule Bigtable.MixProject do {:mix_test_watch, "~> 0.8", only: :dev, runtime: false}, {:protobuf, "~> 0.5.3"}, {:google_protos, "~> 0.1"}, - {:grpc, "~> 0.3.1"} + {:grpc, "~> 0.3.1"}, + {:poolboy, "~> 1.5"} ] end end diff --git a/mix.lock b/mix.lock index a6d2428..9d0be5b 100644 --- a/mix.lock +++ b/mix.lock @@ -33,6 +33,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "pre_commit": {:hex, :pre_commit, "0.3.4", "e2850f80be8090d50ad8019ef2426039307ff5dfbe70c736ad0d4d401facf304", [:mix], [], "hexpm"}, "protobuf": {:hex, :protobuf, "0.5.4", "2e1b8eec211aff034ad8a14e3674220b0158bfb9a3c7128ac9d2a1ed1b3724d3", [:mix], [], "hexpm"}, "ranch": {:hex, :ranch, "1.6.2", "6db93c78f411ee033dbb18ba8234c5574883acb9a75af0fb90a9b82ea46afa00", [:rebar3], [], "hexpm"}, diff --git a/test/connection/connection_test.exs b/test/connection/connection_test.exs index 4f74782..88c030e 100644 --- a/test/connection/connection_test.exs +++ b/test/connection/connection_test.exs @@ -19,7 +19,7 @@ defmodule ConnectionTest do scheme: "http" } - connection = Connection.get_connection() + connection = Connection.connect() result = %{ connection From ee7bfc5df7a088d05f95787f9ba3e4f0ec111b0a Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Thu, 18 Apr 2019 09:28:45 +0200 Subject: [PATCH 02/15] fixes disconnect clause --- lib/connection/worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/connection/worker.ex b/lib/connection/worker.ex index 120161c..2e7b2d3 100644 --- a/lib/connection/worker.ex +++ b/lib/connection/worker.ex @@ -33,7 +33,7 @@ defmodule Bigtable.Connection.Worker do state end - defp disconnect(%{connection: connection}) do + defp disconnect(connection) do Connection.disconnect(connection) end end From 16daab161d26044f795ae7fcee70480fb8588168 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 11:00:37 +0200 Subject: [PATCH 03/15] moves column range into row filter module --- lib/row_filter/column_range.ex | 43 --------------------------------- lib/row_filter/row_filter.ex | 44 +++++++++++++++++++++++++++++++--- mix.exs | 11 +-------- 3 files changed, 42 insertions(+), 56 deletions(-) delete mode 100644 lib/row_filter/column_range.ex diff --git a/lib/row_filter/column_range.ex b/lib/row_filter/column_range.ex deleted file mode 100644 index 9a0d08e..0000000 --- a/lib/row_filter/column_range.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule Bigtable.RowFilter.ColumnRange do - @moduledoc false - alias Google.Bigtable.V2.ColumnRange - - @type column_range :: {binary(), binary(), boolean()} | {binary(), binary()} - - @spec create_range(binary(), column_range) :: ColumnRange.t() - def create_range(family_name, {start_qualifier, end_qualifier, inclusive}) do - range = translate_range(start_qualifier, end_qualifier, inclusive) - - range - |> Keyword.put(:family_name, family_name) - |> ColumnRange.new() - end - - def create_range(family_name, {start_qualifier, end_qualifier}) do - create_range(family_name, {start_qualifier, end_qualifier, true}) - end - - @spec translate_range(binary(), binary(), boolean()) :: Keyword.t() - defp translate_range(start_qualifier, end_qualifier, inclusive) do - case inclusive do - true -> inclusive_range(start_qualifier, end_qualifier) - false -> exclusive_range(start_qualifier, end_qualifier) - end - end - - @spec exclusive_range(binary(), binary()) :: Keyword.t() - defp exclusive_range(start_qualifier, end_qualifier) do - [ - start_qualifier: {:start_qualifier_open, start_qualifier}, - end_qualifier: {:end_qualifier_open, end_qualifier} - ] - end - - @spec inclusive_range(binary(), binary()) :: Keyword.t() - defp inclusive_range(start_qualifier, end_qualifier) do - [ - start_qualifier: {:start_qualifier_closed, start_qualifier}, - end_qualifier: {:end_qualifier_closed, end_qualifier} - ] - end -end diff --git a/lib/row_filter/row_filter.ex b/lib/row_filter/row_filter.ex index 6036ca3..9f8b79f 100644 --- a/lib/row_filter/row_filter.ex +++ b/lib/row_filter/row_filter.ex @@ -1,6 +1,7 @@ defmodule Bigtable.RowFilter do - alias Bigtable.RowFilter.ColumnRange - alias Google.Bigtable.V2.{ReadRowsRequest, RowFilter, TimestampRange} + alias Google.Bigtable.V2.{ColumnRange, ReadRowsRequest, RowFilter, TimestampRange} + + @type column_range :: {binary(), binary(), boolean()} | {binary(), binary()} @moduledoc """ Provides functions for creating `Google.Bigtable.V2.RowFilter` and applying them to a `Google.Bigtable.V2.ReadRowsRequest` or `Google.Bigtable.V2.RowFilter.Chain`. @@ -330,7 +331,7 @@ defmodule Bigtable.RowFilter do @spec column_range(binary(), {binary(), binary(), boolean()} | {binary(), binary()}) :: RowFilter.t() def column_range(family_name, range) do - range = ColumnRange.create_range(family_name, range) + range = create_range(family_name, range) {:column_range_filter, range} |> build_filter() @@ -546,4 +547,41 @@ defmodule Bigtable.RowFilter do defp apply_filter(%RowFilter{} = filter, %ReadRowsRequest{} = request) do %{request | filter: filter} end + + @spec create_range(binary(), column_range) :: ColumnRange.t() + def create_range(family_name, {start_qualifier, end_qualifier, inclusive}) do + range = translate_range(start_qualifier, end_qualifier, inclusive) + + range + |> Keyword.put(:family_name, family_name) + |> ColumnRange.new() + end + + def create_range(family_name, {start_qualifier, end_qualifier}) do + create_range(family_name, {start_qualifier, end_qualifier, true}) + end + + @spec translate_range(binary(), binary(), boolean()) :: Keyword.t() + defp translate_range(start_qualifier, end_qualifier, inclusive) do + case inclusive do + true -> inclusive_range(start_qualifier, end_qualifier) + false -> exclusive_range(start_qualifier, end_qualifier) + end + end + + @spec exclusive_range(binary(), binary()) :: Keyword.t() + defp exclusive_range(start_qualifier, end_qualifier) do + [ + start_qualifier: {:start_qualifier_open, start_qualifier}, + end_qualifier: {:end_qualifier_open, end_qualifier} + ] + end + + @spec inclusive_range(binary(), binary()) :: Keyword.t() + defp inclusive_range(start_qualifier, end_qualifier) do + [ + start_qualifier: {:start_qualifier_closed, start_qualifier}, + end_qualifier: {:end_qualifier_closed, end_qualifier} + ] + end end diff --git a/mix.exs b/mix.exs index 78eb7a6..0fc8eac 100644 --- a/mix.exs +++ b/mix.exs @@ -74,16 +74,7 @@ defmodule Bigtable.MixProject do end defp groups_for_modules do - [ - "Typed Bigtable": [ - Bigtable.Schema - ], - Operations: [ - Bigtable.ReadRows, - Bigtable.MutateRow, - Bigtable.MutateRows - ] - ] + [] end defp aliases do From 29c8514b4848edca761ef4fa29574e09c1c2d05b Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 11:48:37 +0200 Subject: [PATCH 04/15] adds data module prefix to data functionality --- lib/connection/worker.ex | 1 + lib/data/check_and_mutate_row.ex | 6 +- lib/data/chunk_reader.ex | 2 +- lib/data/mutate_row.ex | 2 +- lib/data/mutate_rows.ex | 2 +- lib/data/mutations.ex | 4 +- lib/data/read_modify_write_row.ex | 2 +- lib/data/read_rows.ex | 9 +-- lib/{row_filter => data}/row_filter.ex | 58 +++++++++---------- lib/data/row_set.ex | 20 +++---- lib/data/sample_row_keys.ex | 6 +- lib/supervisor.ex | 2 +- mix.exs | 17 +++++- test/data/check_and_mutate_row_test.exs | 2 +- test/data/mutate_row_test.exs | 2 +- test/data/mutate_rows_test.exs | 3 +- test/data/mutations_test.exs | 2 +- test/data/read_modify_write_row_test.exs | 11 ++-- test/data/read_rows_test.exs | 5 +- test/data/row_filter_integration_test.exs | 4 +- test/data/row_filter_test.exs | 2 +- test/data/row_set_test.exs | 2 +- test/data/sample_row_keys_test.exs | 2 +- .../read_rows_acceptance_test.exs | 6 +- 24 files changed, 92 insertions(+), 80 deletions(-) rename lib/{row_filter => data}/row_filter.ex (87%) diff --git a/lib/connection/worker.ex b/lib/connection/worker.ex index 2e7b2d3..4eca6de 100644 --- a/lib/connection/worker.ex +++ b/lib/connection/worker.ex @@ -1,4 +1,5 @@ defmodule Bigtable.Connection.Worker do + @moduledoc false alias Bigtable.Connection use GenServer diff --git a/lib/data/check_and_mutate_row.ex b/lib/data/check_and_mutate_row.ex index 9b641ae..9eaab76 100644 --- a/lib/data/check_and_mutate_row.ex +++ b/lib/data/check_and_mutate_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.CheckAndMutateRow do +defmodule Bigtable.Data.CheckAndMutateRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. """ @@ -14,7 +14,7 @@ defmodule Bigtable.CheckAndMutateRow do ## Examples ### Default Table - iex> Bigtable.CheckAndMutateRow.build("Test#123") + iex> Bigtable.Data.CheckAndMutateRow.build("Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], @@ -26,7 +26,7 @@ defmodule Bigtable.CheckAndMutateRow do ### Custom Table iex> table_name = "projects/[project_id]/instances/[instnace_id]/tables/[table_name]" - iex> Bigtable.CheckAndMutateRow.build(table_name, "Test#123") + iex> Bigtable.Data.CheckAndMutateRow.build(table_name, "Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], diff --git a/lib/data/chunk_reader.ex b/lib/data/chunk_reader.ex index 9b7841e..45afbbf 100644 --- a/lib/data/chunk_reader.ex +++ b/lib/data/chunk_reader.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.ChunkReader do +defmodule Bigtable.Data.ChunkReader do @moduledoc """ Reads chunks from `Google.Bigtable.V2.ReadRowsResponse` and parses them into complete cells grouped by rowkey. """ diff --git a/lib/data/mutate_row.ex b/lib/data/mutate_row.ex index 5e1440d..bbc972d 100644 --- a/lib/data/mutate_row.ex +++ b/lib/data/mutate_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.MutateRow do +defmodule Bigtable.Data.MutateRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowRequest` and submit them to Bigtable. """ diff --git a/lib/data/mutate_rows.ex b/lib/data/mutate_rows.ex index 3a96818..165c0bd 100644 --- a/lib/data/mutate_rows.ex +++ b/lib/data/mutate_rows.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.MutateRows do +defmodule Bigtable.Data.MutateRows do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowsRequest` and submit them to Bigtable. """ diff --git a/lib/data/mutations.ex b/lib/data/mutations.ex index 885570f..196807e 100644 --- a/lib/data/mutations.ex +++ b/lib/data/mutations.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Mutations do +defmodule Bigtable.Data.Mutations do @moduledoc """ Provides functions to build Bigtable mutations that are used when forming row mutation requests. @@ -12,7 +12,7 @@ defmodule Bigtable.Mutations do ## Examples - iex> Bigtable.Mutations.build("Row#123") + iex> Bigtable.Data.Mutations.build("Row#123") %Google.Bigtable.V2.MutateRowsRequest.Entry{mutations: [], row_key: "Row#123"} """ @spec build(binary()) :: Entry.t() diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index 76e16f4..0173ba3 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.ReadModifyWriteRow do +defmodule Bigtable.Data.ReadModifyWriteRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadModifyWriteRowRequest` and submit them to Bigtable. """ diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index 9ca3291..751b525 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -1,8 +1,9 @@ -defmodule Bigtable.ReadRows do +defmodule Bigtable.Data.ReadRows do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. """ - alias Bigtable.{ChunkReader, Utils} + alias Bigtable.Data.ChunkReader + alias Bigtable.Utils alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -11,7 +12,7 @@ defmodule Bigtable.ReadRows do ## Examples iex> table_name = "projects/[project_id]/instances/[instnace_id]/tables/[table_name]" - iex> Bigtable.ReadRows.build(table_name) + iex> Bigtable.Data.ReadRows.build(table_name) %Google.Bigtable.V2.ReadRowsRequest{ app_profile_id: "", filter: nil, @@ -29,7 +30,7 @@ defmodule Bigtable.ReadRows do Builds a `Google.Bigtable.V2.ReadRowsRequest` with the configured table name. ## Examples - iex> Bigtable.ReadRows.build() + iex> Bigtable.Data.ReadRows.build() %Google.Bigtable.V2.ReadRowsRequest{ app_profile_id: "", filter: nil, diff --git a/lib/row_filter/row_filter.ex b/lib/data/row_filter.ex similarity index 87% rename from lib/row_filter/row_filter.ex rename to lib/data/row_filter.ex index 9f8b79f..14a0fed 100644 --- a/lib/row_filter/row_filter.ex +++ b/lib/data/row_filter.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.RowFilter do +defmodule Bigtable.Data.RowFilter do alias Google.Bigtable.V2.{ColumnRange, ReadRowsRequest, RowFilter, TimestampRange} @type column_range :: {binary(), binary(), boolean()} | {binary(), binary()} @@ -12,8 +12,8 @@ defmodule Bigtable.RowFilter do ## Examples - iex> filters = [Bigtable.RowFilter.cells_per_column(2), Bigtable.RowFilter.row_key_regex("^Test#\w+")] - iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowFilter.chain(filters) + iex> filters = [Bigtable.Data.RowFilter.cells_per_column(2), Bigtable.Data.RowFilter.row_key_regex("^Test#\w+")] + iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowFilter.chain(filters) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:chain, @@ -40,7 +40,7 @@ defmodule Bigtable.RowFilter do Adds a cells per column `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_column(2) + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_column(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_column_limit_filter, 2} @@ -58,7 +58,7 @@ defmodule Bigtable.RowFilter do Creates a cells per column `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.cells_per_column(2) + iex> Bigtable.Data.RowFilter.cells_per_column(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_column_limit_filter, 2} } @@ -73,7 +73,7 @@ defmodule Bigtable.RowFilter do Adds a cells per row `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_row(2) + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_row(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_limit_filter, 2} @@ -91,7 +91,7 @@ defmodule Bigtable.RowFilter do Creates a cells per row `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.cells_per_row(2) + iex> Bigtable.Data.RowFilter.cells_per_row(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_limit_filter, 2} } @@ -106,7 +106,7 @@ defmodule Bigtable.RowFilter do Adds a cells per row offset `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_row_offset(2) + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_row_offset(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_offset_filter, 2} @@ -124,7 +124,7 @@ defmodule Bigtable.RowFilter do Creates a cells per row offset `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.cells_per_row_offset(2) + iex> Bigtable.Data.RowFilter.cells_per_row_offset(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_offset_filter, 2} } @@ -139,7 +139,7 @@ defmodule Bigtable.RowFilter do Adds a row key regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.row_key_regex("^Test#\\w+") + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.row_key_regex("^Test#\\w+") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:row_key_regex_filter, "^Test#\\w+"} @@ -157,7 +157,7 @@ defmodule Bigtable.RowFilter do Creates a row key regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.row_key_regex("^Test#\\w+") + iex> Bigtable.Data.RowFilter.row_key_regex("^Test#\\w+") %Google.Bigtable.V2.RowFilter{ filter: {:row_key_regex_filter, "^Test#\\w+"} } @@ -172,7 +172,7 @@ defmodule Bigtable.RowFilter do Adds a value regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.value_regex("^test$") + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.value_regex("^test$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:value_regex_filter, "^test$"} @@ -190,7 +190,7 @@ defmodule Bigtable.RowFilter do Creates a value regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.value_regex("^test$") + iex> Bigtable.Data.RowFilter.value_regex("^test$") %Google.Bigtable.V2.RowFilter{ filter: {:value_regex_filter, "^test$"} } @@ -205,7 +205,7 @@ defmodule Bigtable.RowFilter do Adds a family name regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.family_name_regex("^testFamily$") + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.family_name_regex("^testFamily$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:family_name_regex_filter, "^testFamily$"} @@ -223,7 +223,7 @@ defmodule Bigtable.RowFilter do Creates a family name regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.family_name_regex("^testFamily$") + iex> Bigtable.Data.RowFilter.family_name_regex("^testFamily$") %Google.Bigtable.V2.RowFilter{ filter: {:family_name_regex_filter, "^testFamily$"} } @@ -238,7 +238,7 @@ defmodule Bigtable.RowFilter do Adds a column qualifier regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.column_qualifier_regex("^testColumn$") + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.column_qualifier_regex("^testColumn$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:column_qualifier_regex_filter, "^testColumn$"} @@ -256,7 +256,7 @@ defmodule Bigtable.RowFilter do Creates a family name regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.column_qualifier_regex("^testColumn$") + iex> Bigtable.Data.RowFilter.column_qualifier_regex("^testColumn$") %Google.Bigtable.V2.RowFilter{ filter: {:column_qualifier_regex_filter, "^testColumn$"} } @@ -276,7 +276,7 @@ defmodule Bigtable.RowFilter do ## Examples iex> range = {"column2", "column4"} - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.column_range("family", range) + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.column_range("family", range) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: { @@ -315,7 +315,7 @@ defmodule Bigtable.RowFilter do ## Examples iex> range = {"column2", "column4"} - iex> Bigtable.RowFilter.column_range("family", range) + iex> Bigtable.Data.RowFilter.column_range("family", range) %Google.Bigtable.V2.RowFilter{ filter: { :column_range_filter, @@ -345,7 +345,7 @@ defmodule Bigtable.RowFilter do ## Examples iex> range = [start_timestamp: 1000, end_timestamp: 2000] - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.timestamp_range(range) + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.timestamp_range(range) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: { @@ -373,7 +373,7 @@ defmodule Bigtable.RowFilter do ## Examples iex> range = [start_timestamp: 1000, end_timestamp: 2000] - iex> Bigtable.RowFilter.timestamp_range(range) + iex> Bigtable.Data.RowFilter.timestamp_range(range) %Google.Bigtable.V2.RowFilter{ filter: { :timestamp_range_filter, @@ -401,7 +401,7 @@ defmodule Bigtable.RowFilter do ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.pass_all() + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.pass_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:pass_all_filter, true} @@ -421,7 +421,7 @@ defmodule Bigtable.RowFilter do Matches all cells, regardless of input. Functionally equivalent to leaving filter unset, but included for completeness. ## Examples - iex> Bigtable.RowFilter.pass_all() + iex> Bigtable.Data.RowFilter.pass_all() %Google.Bigtable.V2.RowFilter{ filter: {:pass_all_filter, true} } @@ -437,7 +437,7 @@ defmodule Bigtable.RowFilter do ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.block_all() + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.block_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:block_all_filter, true} @@ -457,7 +457,7 @@ defmodule Bigtable.RowFilter do Does not match any cells, regardless of input. Useful for temporarily disabling just part of a filter. ## Examples - iex> Bigtable.RowFilter.block_all() + iex> Bigtable.Data.RowFilter.block_all() %Google.Bigtable.V2.RowFilter{ filter: {:block_all_filter, true} } @@ -473,7 +473,7 @@ defmodule Bigtable.RowFilter do ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.strip_value_transformer() + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.strip_value_transformer() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:strip_value_transformer, true} @@ -492,7 +492,7 @@ defmodule Bigtable.RowFilter do ## Examples - iex> Bigtable.RowFilter.strip_value_transformer() + iex> Bigtable.Data.RowFilter.strip_value_transformer() %Google.Bigtable.V2.RowFilter{ filter: {:strip_value_transformer, true} } @@ -508,7 +508,7 @@ defmodule Bigtable.RowFilter do ## Examples - iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.apply_label_transformer("label") + iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.apply_label_transformer("label") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:apply_label_transformer, "label"} @@ -525,7 +525,7 @@ defmodule Bigtable.RowFilter do @doc """ Creates an apply label transformer `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.RowFilter.apply_label_transformer("label") + iex> Bigtable.Data.RowFilter.apply_label_transformer("label") %Google.Bigtable.V2.RowFilter{ filter: {:apply_label_transformer, "label"} } diff --git a/lib/data/row_set.ex b/lib/data/row_set.ex index 997c198..35b12fb 100644 --- a/lib/data/row_set.ex +++ b/lib/data/row_set.ex @@ -1,8 +1,8 @@ -defmodule Bigtable.RowSet do +defmodule Bigtable.Data.RowSet do @moduledoc """ Provides functions to build a `Google.Bigtable.V2.RowSet` and apply it to a `Google.Bigtable.V2.ReadRowsRequest` """ - alias Bigtable.ReadRows + alias Bigtable.Data.ReadRows alias Google.Bigtable.V2 @doc """ @@ -13,12 +13,12 @@ defmodule Bigtable.RowSet do ## Examples #### Single Key - iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_keys("Row#123") + iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_keys("Row#123") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123"], row_ranges: []} #### Multiple Keys - iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_keys(["Row#123", "Row#124"]) + iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_keys(["Row#123", "Row#124"]) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123", "Row#124"], row_ranges: []} """ @@ -42,12 +42,12 @@ defmodule Bigtable.RowSet do ## Examples #### Single Key - iex> request = Bigtable.RowSet.row_keys("Row#123") + iex> request = Bigtable.Data.RowSet.row_keys("Row#123") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123"], row_ranges: []} #### Multiple Keys - iex> request = Bigtable.RowSet.row_keys(["Row#123", "Row#124"]) + iex> request = Bigtable.Data.RowSet.row_keys(["Row#123", "Row#124"]) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123", "Row#124"], row_ranges: []} """ @@ -71,7 +71,7 @@ defmodule Bigtable.RowSet do ## Examples #### Single Range - iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_ranges({"start", "end"}) + iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_ranges({"start", "end"}) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -86,7 +86,7 @@ defmodule Bigtable.RowSet do #### Multiple Ranges iex> ranges = [{"start1", "end1"}, {"start2", "end2", false}] - iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_ranges(ranges) + iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_ranges(ranges) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -128,7 +128,7 @@ defmodule Bigtable.RowSet do ## Examples #### Single Range - iex> request = Bigtable.RowSet.row_ranges({"start", "end"}) + iex> request = Bigtable.Data.RowSet.row_ranges({"start", "end"}) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -143,7 +143,7 @@ defmodule Bigtable.RowSet do #### Multiple Ranges iex> ranges = [{"start1", "end1"}, {"start2", "end2", false}] - iex> request = Bigtable.RowSet.row_ranges(ranges) + iex> request = Bigtable.Data.RowSet.row_ranges(ranges) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index 6509c1e..1af6222 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.SampleRowKeys do +defmodule Bigtable.Data.SampleRowKeys do @moduledoc """ Provides functions to build `Google.Bigtable.V2.SampleRowKeysRequest` and submit them to Bigtable. """ @@ -14,7 +14,7 @@ defmodule Bigtable.SampleRowKeys do ## Examples ### Default Table - iex> Bigtable.SampleRowKeys.build() + iex> Bigtable.Data.SampleRowKeys.build() %Google.Bigtable.V2.SampleRowKeysRequest{ app_profile_id: "", table_name: "projects/dev/instances/dev/tables/test", @@ -22,7 +22,7 @@ defmodule Bigtable.SampleRowKeys do ### Custom Table iex> table_name = "projects/[project_id]/instances/[instance_id]/tables/[table_name]" - iex> Bigtable.SampleRowKeys.build(table_name) + iex> Bigtable.Data.SampleRowKeys.build(table_name) %Google.Bigtable.V2.SampleRowKeysRequest{ app_profile_id: "", table_name: "projects/[project_id]/instances/[instance_id]/tables/[table_name]", diff --git a/lib/supervisor.ex b/lib/supervisor.ex index f4d6b5d..e53b1f3 100644 --- a/lib/supervisor.ex +++ b/lib/supervisor.ex @@ -9,7 +9,7 @@ defmodule Bigtable.Supervisor do def init(:ok) do children = [ Bigtable.Connection, - {DynamicSupervisor, name: Bigtable.ChunkReader.Supervisor, strategy: :one_for_one} + {DynamicSupervisor, name: Bigtable.Data.ChunkReader.Supervisor, strategy: :one_for_one} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/mix.exs b/mix.exs index 0fc8eac..e2c8cf8 100644 --- a/mix.exs +++ b/mix.exs @@ -1,6 +1,8 @@ defmodule Bigtable.MixProject do use Mix.Project + alias Bigtable.{Admin, Connection} + @version "0.6.1" def project do @@ -54,7 +56,8 @@ defmodule Bigtable.MixProject do formatters: ["html", "epub"], groups_for_modules: groups_for_modules(), extras: extras(), - groups_for_extras: groups_for_extras() + groups_for_extras: groups_for_extras(), + nest_modules_by_prefix: [Bigtable.Admin, Bigtable.Data] ] end @@ -74,7 +77,17 @@ defmodule Bigtable.MixProject do end defp groups_for_modules do - [] + [ + Admin: [ + Admin.GcRule, + Admin.Modification, + Admin.Table, + Admin.TableAdmin + ], + Connection: [ + Connection + ] + ] end defp aliases do diff --git a/test/data/check_and_mutate_row_test.exs b/test/data/check_and_mutate_row_test.exs index f64431f..267d686 100644 --- a/test/data/check_and_mutate_row_test.exs +++ b/test/data/check_and_mutate_row_test.exs @@ -1,6 +1,6 @@ defmodule CheckAndMutateRowTest do @moduledoc false - alias Bigtable.{CheckAndMutateRow, ChunkReader, MutateRow, Mutations, ReadRows, RowFilter} + alias Bigtable.Data.{CheckAndMutateRow, ChunkReader, MutateRow, Mutations, ReadRows, RowFilter} alias ChunkReader.ReadCell use ExUnit.Case diff --git a/test/data/mutate_row_test.exs b/test/data/mutate_row_test.exs index 95b6313..24ff71b 100644 --- a/test/data/mutate_row_test.exs +++ b/test/data/mutate_row_test.exs @@ -1,7 +1,7 @@ defmodule MutateRowTest do @moduledoc false # TODO: Integration tests including errors - alias Bigtable.{MutateRow, Mutations} + alias Bigtable.Data.{MutateRow, Mutations} use ExUnit.Case diff --git a/test/data/mutate_rows_test.exs b/test/data/mutate_rows_test.exs index 085ed3a..7498597 100644 --- a/test/data/mutate_rows_test.exs +++ b/test/data/mutate_rows_test.exs @@ -2,8 +2,7 @@ defmodule MutateRowsTest do @moduledoc false # TODO: Integration tests including errors - alias Bigtable.{MutateRows, Mutations} - + alias Bigtable.Data.{MutateRows, Mutations} use ExUnit.Case setup do diff --git a/test/data/mutations_test.exs b/test/data/mutations_test.exs index 041944e..cb0f6d7 100644 --- a/test/data/mutations_test.exs +++ b/test/data/mutations_test.exs @@ -1,5 +1,5 @@ defmodule MutationsTest do - alias Bigtable.Mutations + alias Bigtable.Data.Mutations alias Google.Bigtable.V2.MutateRowsRequest.Entry use ExUnit.Case diff --git a/test/data/read_modify_write_row_test.exs b/test/data/read_modify_write_row_test.exs index 8090704..b995c46 100644 --- a/test/data/read_modify_write_row_test.exs +++ b/test/data/read_modify_write_row_test.exs @@ -1,7 +1,6 @@ defmodule ReadModifyWriteRowTest do @moduledoc false - alias Bigtable.{ReadModifyWriteRow, MutateRow, Mutations, ReadRows} - + alias Bigtable.Data.{ReadModifyWriteRow, MutateRow, Mutations, Row, ReadRows, RowFilter} use ExUnit.Case doctest ReadModifyWriteRow @@ -42,7 +41,7 @@ defmodule ReadModifyWriteRowTest do {:ok, result} = ReadRows.build() - |> Bigtable.RowFilter.cells_per_column(1) + |> RowFilter.cells_per_column(1) |> ReadRows.read() new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) @@ -62,7 +61,7 @@ defmodule ReadModifyWriteRowTest do {:ok, result} = ReadRows.build() - |> Bigtable.RowFilter.cells_per_column(1) + |> RowFilter.cells_per_column(1) |> ReadRows.read() new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) @@ -88,7 +87,7 @@ defmodule ReadModifyWriteRowTest do {:ok, result} = ReadRows.build() - |> Bigtable.RowFilter.cells_per_column(1) + |> RowFilter.cells_per_column(1) |> ReadRows.read() new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) @@ -108,7 +107,7 @@ defmodule ReadModifyWriteRowTest do {:ok, result} = ReadRows.build() - |> Bigtable.RowFilter.cells_per_column(1) + |> RowFilter.cells_per_column(1) |> ReadRows.read() new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) diff --git a/test/data/read_rows_test.exs b/test/data/read_rows_test.exs index 23ca75d..690e6d6 100644 --- a/test/data/read_rows_test.exs +++ b/test/data/read_rows_test.exs @@ -1,7 +1,6 @@ defmodule ReadRowsTest do # TODO: Integration tests including errors - alias Bigtable.{MutateRow, MutateRows, Mutations, ReadRows} - + alias Bigtable.Data.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows} use ExUnit.Case doctest ReadRows @@ -86,7 +85,7 @@ defmodule ReadRowsTest do for row_key <- row_keys, into: %{} do {row_key, [ - %Bigtable.ChunkReader.ReadCell{ + %ChunkReader.ReadCell{ family_name: %Google.Protobuf.StringValue{value: context.column_family}, label: "", qualifier: %Google.Protobuf.BytesValue{value: context.column_qualifier}, diff --git a/test/data/row_filter_integration_test.exs b/test/data/row_filter_integration_test.exs index 293f531..2d6e830 100644 --- a/test/data/row_filter_integration_test.exs +++ b/test/data/row_filter_integration_test.exs @@ -1,6 +1,6 @@ defmodule RowFilterIntegration do @moduledoc false - alias Bigtable.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows, RowFilter} + alias Bigtable.Data.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows, RowFilter} alias ChunkReader.ReadCell alias Google.Protobuf.{BytesValue, StringValue} @@ -1192,7 +1192,7 @@ defmodule RowFilterIntegration do expected = %{ "Test#1" => [ - %Bigtable.ChunkReader.ReadCell{ + %ReadCell{ family_name: %Google.Protobuf.StringValue{value: "cf1"}, label: "", qualifier: %Google.Protobuf.BytesValue{value: "column"}, diff --git a/test/data/row_filter_test.exs b/test/data/row_filter_test.exs index 4cc7103..56f3262 100644 --- a/test/data/row_filter_test.exs +++ b/test/data/row_filter_test.exs @@ -1,6 +1,6 @@ defmodule RowFilterTest do @moduledoc false - alias Bigtable.RowFilter + alias Bigtable.Data.RowFilter use ExUnit.Case diff --git a/test/data/row_set_test.exs b/test/data/row_set_test.exs index 6fa5ba3..1f5533b 100644 --- a/test/data/row_set_test.exs +++ b/test/data/row_set_test.exs @@ -1,5 +1,5 @@ defmodule RowSetTest do - alias Bigtable.RowSet + alias Bigtable.Data.RowSet use ExUnit.Case diff --git a/test/data/sample_row_keys_test.exs b/test/data/sample_row_keys_test.exs index f733685..1a7cf7d 100644 --- a/test/data/sample_row_keys_test.exs +++ b/test/data/sample_row_keys_test.exs @@ -1,6 +1,6 @@ defmodule SampleRowKeysTest do @moduledoc false - alias Bigtable.SampleRowKeys + alias Bigtable.Data.SampleRowKeys use ExUnit.Case diff --git a/test/google_acceptance/read_rows_acceptance_test.exs b/test/google_acceptance/read_rows_acceptance_test.exs index 9cdcd4c..66e4c14 100644 --- a/test/google_acceptance/read_rows_acceptance_test.exs +++ b/test/google_acceptance/read_rows_acceptance_test.exs @@ -1,5 +1,5 @@ defmodule TestResult do - alias Bigtable.ChunkReader.ReadCell + alias Bigtable.Data.ChunkReader.ReadCell alias Google.Bigtable.V2.ReadRowsResponse.CellChunk def from_chunk(row_key, %ReadCell{} = ri) do @@ -16,7 +16,7 @@ defmodule TestResult do end defmodule GoogleAcceptanceTest do - alias Bigtable.ChunkReader + alias Bigtable.Data.ChunkReader defmacro __using__(json: json) do json @@ -57,7 +57,7 @@ defmodule GoogleAcceptanceTest do end defmodule ReadRowsAcceptanceTest do - alias Bigtable.ChunkReader + alias Bigtable.Data.ChunkReader alias Google.Bigtable.V2.ReadRowsResponse.CellChunk use ExUnit.Case From 0a486380652f2ae175a52ba8d459deeef9f61d6c Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 12:18:10 +0200 Subject: [PATCH 05/15] restructures docs to group by grpc type --- lib/admin/table_admin/modification.ex | 2 +- lib/data/read_modify_write_row.ex | 2 +- lib/data/row_filter.ex | 4 +-- mix.exs | 35 ++++++++++++++++++------ mix.lock | 2 ++ test/data/read_modify_write_row_test.exs | 2 +- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/lib/admin/table_admin/modification.ex b/lib/admin/table_admin/modification.ex index 55229c2..dd216e0 100644 --- a/lib/admin/table_admin/modification.ex +++ b/lib/admin/table_admin/modification.ex @@ -1,4 +1,4 @@ defmodule Bigtable.Admin.Modification do - def create(id) do + def create(_id) do end end diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index 0173ba3..f0f4674 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -13,7 +13,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do } @doc """ - Builds a `Google.Bigtable.V2.ReadModifyWriteRowRequest` with a provided table name and row key`. + Builds a `Google.Bigtable.V2.ReadModifyWriteRowRequest` with a provided table name and row key """ @spec build(binary(), binary()) :: ReadModifyWriteRowRequest.t() def build(table_name \\ Bigtable.Utils.configured_table_name(), row_key) diff --git a/lib/data/row_filter.ex b/lib/data/row_filter.ex index 14a0fed..4f8cbf0 100644 --- a/lib/data/row_filter.ex +++ b/lib/data/row_filter.ex @@ -469,7 +469,7 @@ defmodule Bigtable.Data.RowFilter do end @doc """ - Adds a strip value transformer Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. + Adds a strip value transformer `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples @@ -504,7 +504,7 @@ defmodule Bigtable.Data.RowFilter do end @doc """ - Adds an apply label transformer Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. + Adds an apply label transformer `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples diff --git a/mix.exs b/mix.exs index e2c8cf8..e4c45e3 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Bigtable.MixProject do use Mix.Project - alias Bigtable.{Admin, Connection} + alias Bigtable.{Admin, Connection, Data} @version "0.6.1" @@ -57,7 +57,9 @@ defmodule Bigtable.MixProject do groups_for_modules: groups_for_modules(), extras: extras(), groups_for_extras: groups_for_extras(), - nest_modules_by_prefix: [Bigtable.Admin, Bigtable.Data] + nest_modules_by_prefix: [ + Bigtable.Data.ChunkReader + ] ] end @@ -86,6 +88,19 @@ defmodule Bigtable.MixProject do ], Connection: [ Connection + ], + Data: [ + Data.CheckAndMutateRow, + Data.ChunkReader, + Data.ChunkReader.ReadCell, + Data.MutateRow, + Data.MutateRows, + Data.Mutations, + Data.ReadModifyWriteRow, + Data.ReadRows, + Data.RowFilter, + Data.RowSet, + Data.SampleRowKeys ] ] end @@ -101,17 +116,19 @@ defmodule Bigtable.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:poison, "~> 3.1"}, - {:lens, "~> 0.8.0"}, + {:google_protos, "~> 0.1"}, {:goth, "~> 0.11.0"}, + {:grpc, "~> 0.3.1"}, + {:lens, "~> 0.8.0"}, + {:poison, "~> 3.1"}, + {:poolboy, "~> 1.5"}, + {:protobuf, "~> 0.5.3"}, + # Dev Deps {:credo, "~> 1.0.0", only: [:dev, :test, :ci], runtime: false}, + {:dialyxir, "~> 1.0.0-rc.6", only: [:dev], runtime: false}, {:excoveralls, "~> 0.10", only: [:dev, :test, :ci]}, {:ex_doc, "~> 0.19", only: :dev, runtime: false}, - {:mix_test_watch, "~> 0.8", only: :dev, runtime: false}, - {:protobuf, "~> 0.5.3"}, - {:google_protos, "~> 0.1"}, - {:grpc, "~> 0.3.1"}, - {:poolboy, "~> 1.5"} + {:mix_test_watch, "~> 0.8", only: :dev, runtime: false} ] end end diff --git a/mix.lock b/mix.lock index 9d0be5b..cdf6674 100644 --- a/mix.lock +++ b/mix.lock @@ -5,9 +5,11 @@ "cowboy": {:hex, :cowboy, "2.5.0", "4ef3ae066ee10fe01ea3272edc8f024347a0d3eb95f6fbb9aed556dacbfc1337", [:rebar3], [{:cowlib, "~> 2.6.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.6.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, "cowlib": {:hex, :cowlib, "2.6.0", "8aa629f81a0fc189f261dc98a42243fa842625feea3c7ec56c48f4ccdb55490f", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "1.0.0", "aaa40fdd0543a0cf8080e8c5949d8c25f0a24e4fc8c1d83d06c388f5e5e0ea42", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, "diver": {:hex, :diver, "0.2.0", "8a2a85e97c7b1a989db501fe2ce2a5a0e82109b0d8b02e04c0c1a8a10b53795e", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.1", "73812f447f7a42358d3ba79283cfa3075a7580a3a2ed457616d6517ac3738cb9", [:mix], [], "hexpm"}, "elixir_make": {:hex, :elixir_make, "0.4.2", "332c649d08c18bc1ecc73b1befc68c647136de4f340b548844efc796405743bf", [:mix], [], "hexpm"}, + "erlex": {:hex, :erlex, "0.2.1", "cee02918660807cbba9a7229cae9b42d1c6143b768c781fa6cee1eaf03ad860b", [:mix], [], "hexpm"}, "erlport": {:hex, :erlport, "0.10.0", "2436ec2f4ed62538c6e9c52f523f9315b6002ee7e298d9bd10b35abc3f6b32e7", [:rebar3], [], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.19.2", "6f4081ccd9ed081b6dc0bd5af97a41e87f5554de469e7d76025fba535180565f", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.10.4", "b86230f0978bbc630c139af5066af7cd74fd16536f71bc047d1037091f9f63a9", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/data/read_modify_write_row_test.exs b/test/data/read_modify_write_row_test.exs index b995c46..0d0b0d5 100644 --- a/test/data/read_modify_write_row_test.exs +++ b/test/data/read_modify_write_row_test.exs @@ -1,6 +1,6 @@ defmodule ReadModifyWriteRowTest do @moduledoc false - alias Bigtable.Data.{ReadModifyWriteRow, MutateRow, Mutations, Row, ReadRows, RowFilter} + alias Bigtable.Data.{ReadModifyWriteRow, MutateRow, Mutations, ReadRows, RowFilter} use ExUnit.Case doctest ReadModifyWriteRow From a130807091caaaf9f1347e98e429ffe14d101b0b Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 12:48:29 +0200 Subject: [PATCH 06/15] fixes credo issues --- lib/admin/table_admin/gc_rule.ex | 3 ++ lib/admin/table_admin/modification.ex | 1 + lib/admin/table_admin/table.ex | 4 ++ lib/admin/table_admin/table_admin.ex | 3 +- lib/data/chunk_reader.ex | 8 ++-- lib/data/read_rows.ex | 2 +- lib/data/row_filter.ex | 11 ++--- lib/data/sample_row_keys.ex | 2 +- lib/utils.ex | 4 +- test/admin/gc_rule_test.exs | 28 ++++++++---- test/admin/table.exs | 9 ++-- test/data/check_and_mutate_row_test.exs | 53 +++++++++++++---------- test/data/mutate_row_test.exs | 1 - test/data/mutate_rows_test.exs | 1 - test/data/read_modify_write_row_test.exs | 30 ++++++++----- test/data/read_rows_test.exs | 1 - test/data/row_filter_integration_test.exs | 27 ++++++++---- test/data/row_filter_test.exs | 2 +- 18 files changed, 116 insertions(+), 74 deletions(-) diff --git a/lib/admin/table_admin/gc_rule.ex b/lib/admin/table_admin/gc_rule.ex index ee4f528..2488554 100644 --- a/lib/admin/table_admin/gc_rule.ex +++ b/lib/admin/table_admin/gc_rule.ex @@ -1,4 +1,7 @@ defmodule Bigtable.Admin.GcRule do + @moduledoc """ + Provides functions for creating garbage collection rules + """ alias Google.Bigtable.Admin.V2 alias Google.Protobuf.Duration alias V2.GcRule.{Intersection, Union} diff --git a/lib/admin/table_admin/modification.ex b/lib/admin/table_admin/modification.ex index dd216e0..bcc8cb4 100644 --- a/lib/admin/table_admin/modification.ex +++ b/lib/admin/table_admin/modification.ex @@ -1,4 +1,5 @@ defmodule Bigtable.Admin.Modification do + @moduledoc false def create(_id) do end end diff --git a/lib/admin/table_admin/table.ex b/lib/admin/table_admin/table.ex index a39b158..137867a 100644 --- a/lib/admin/table_admin/table.ex +++ b/lib/admin/table_admin/table.ex @@ -1,4 +1,8 @@ defmodule Bigtable.Admin.Table do + @moduledoc """ + Provides functionality for building `Google.Bigtable.Admin.V2.Table`. + """ + alias Google.Bigtable.Admin.V2 def build(column_families) when is_map(column_families) do diff --git a/lib/admin/table_admin/table_admin.ex b/lib/admin/table_admin/table_admin.ex index 42a21d5..57cc12d 100644 --- a/lib/admin/table_admin/table_admin.ex +++ b/lib/admin/table_admin/table_admin.ex @@ -7,7 +7,8 @@ defmodule Bigtable.Admin.TableAdmin do alias V2.BigtableTableAdmin.Stub def list_tables(opts \\ []) do - Keyword.put_new(opts, :parent, Utils.configured_instance_name()) + opts + |> Keyword.put_new(:parent, Utils.configured_instance_name()) |> V2.ListTablesRequest.new() |> Utils.process_request(&Stub.list_tables/3) end diff --git a/lib/data/chunk_reader.ex b/lib/data/chunk_reader.ex index 45afbbf..7921f3d 100644 --- a/lib/data/chunk_reader.ex +++ b/lib/data/chunk_reader.ex @@ -56,7 +56,7 @@ defmodule Bigtable.Data.ChunkReader do Opens a `Bigtable.ChunkReader`. """ @spec open() :: :ignore | {:error, any()} | {:ok, pid()} | {:ok, pid(), any()} - def open() do + def open do DynamicSupervisor.start_child(__MODULE__.Supervisor, __MODULE__) end @@ -243,7 +243,8 @@ defmodule Bigtable.Data.ChunkReader do Map.get(cc, :labels, "") end - Map.put(cr, :cur_val, next_value) + cr + |> Map.put(:cur_val, next_value) |> Map.put(:cur_label, next_label) |> Map.put(:state, :cell_in_progress) end @@ -263,7 +264,8 @@ defmodule Bigtable.Data.ChunkReader do Map.get(cc, :labels, "") end - Map.put(cr, :cur_val, next_value) + cr + |> Map.put(:cur_val, next_value) |> Map.put(:cur_label, next_label) |> finish_cell(cc) end diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index 751b525..850909a 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -86,7 +86,7 @@ defmodule Bigtable.Data.ReadRows do Returns a list of `{:ok, %Google.Bigtable.V2.ReadRowsResponse{}}`. """ @spec read() :: - {:error, GRPC.RPCError.t()} + {:error, any()} | [ok: V2.ReadRowsResponse.t()] def read do request = build() diff --git a/lib/data/row_filter.ex b/lib/data/row_filter.ex index 4f8cbf0..b1ced4f 100644 --- a/lib/data/row_filter.ex +++ b/lib/data/row_filter.ex @@ -399,7 +399,6 @@ defmodule Bigtable.Data.RowFilter do @doc """ Adds a pass all `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. - ## Examples iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.pass_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter @@ -427,7 +426,7 @@ defmodule Bigtable.Data.RowFilter do } """ @spec pass_all() :: RowFilter.t() - def pass_all() do + def pass_all do {:pass_all_filter, true} |> build_filter() end @@ -435,7 +434,6 @@ defmodule Bigtable.Data.RowFilter do @doc """ Adds a block all `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. - ## Examples iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.block_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter @@ -463,7 +461,7 @@ defmodule Bigtable.Data.RowFilter do } """ @spec block_all() :: RowFilter.t() - def block_all() do + def block_all do {:block_all_filter, true} |> build_filter() end @@ -471,7 +469,6 @@ defmodule Bigtable.Data.RowFilter do @doc """ Adds a strip value transformer `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. - ## Examples iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.strip_value_transformer() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter @@ -490,7 +487,6 @@ defmodule Bigtable.Data.RowFilter do @doc """ Creates a strip value transformer `Google.Bigtable.V2.RowFilter`. - ## Examples iex> Bigtable.Data.RowFilter.strip_value_transformer() %Google.Bigtable.V2.RowFilter{ @@ -498,7 +494,7 @@ defmodule Bigtable.Data.RowFilter do } """ @spec strip_value_transformer() :: RowFilter.t() - def strip_value_transformer() do + def strip_value_transformer do {:strip_value_transformer, true} |> build_filter() end @@ -506,7 +502,6 @@ defmodule Bigtable.Data.RowFilter do @doc """ Adds an apply label transformer `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. - ## Examples iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.apply_label_transformer("label") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index 1af6222..6963efc 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -44,7 +44,7 @@ defmodule Bigtable.Data.SampleRowKeys do end @spec read() :: {:ok, V2.SampleRowKeysResponse} | {:error, any()} - def read() do + def read do build() |> read() end diff --git a/lib/utils.ex b/lib/utils.ex index 83b715e..fab4030 100644 --- a/lib/utils.ex +++ b/lib/utils.ex @@ -52,11 +52,11 @@ defmodule Bigtable.Utils do "projects/#{project}/instances/#{instance}" end - defp get_project() do + defp get_project do Application.get_env(:bigtable, :project) end - defp get_instance() do + defp get_instance do Application.get_env(:bigtable, :instance) end diff --git a/test/admin/gc_rule_test.exs b/test/admin/gc_rule_test.exs index de00c97..ece2f37 100644 --- a/test/admin/gc_rule_test.exs +++ b/test/admin/gc_rule_test.exs @@ -16,9 +16,12 @@ defmodule GcRuleTest do describe("Bigtagble.Admin.GcRule.max_age/1") do test "should create a table with a max age gc rule", context do - Table.build(%{ + cf = %{ "cf1" => GcRule.max_age(2_592_000_500) - }) + } + + cf + |> Table.build() |> TableAdmin.create_table("gc_rule") expected = %{ @@ -37,9 +40,12 @@ defmodule GcRuleTest do describe("Bigtable.Admin.GcRule.max_num_versions/1") do test "should create a table with a max version gc rule", context do - Table.build(%{ + cf = %{ "cf1" => GcRule.max_num_versions(1) - }) + } + + cf + |> Table.build() |> TableAdmin.create_table("gc_rule") expected = %{ @@ -62,9 +68,12 @@ defmodule GcRuleTest do GcRule.max_age(3000) ] - Table.build(%{ + cf = %{ "cf1" => GcRule.union(rules) - }) + } + + cf + |> Table.build() |> TableAdmin.create_table("gc_rule") expected = %{ @@ -91,9 +100,12 @@ defmodule GcRuleTest do GcRule.max_age(3000) ] - Table.build(%{ + cf = %{ "cf1" => GcRule.intersection(rules) - }) + } + + cf + |> Table.build() |> TableAdmin.create_table("gc_rule") expected = %{ diff --git a/test/admin/table.exs b/test/admin/table.exs index 88c81ad..6efdfea 100644 --- a/test/admin/table.exs +++ b/test/admin/table.exs @@ -50,9 +50,12 @@ defmodule TableAdminTest do context.table_name ) - Table.build(%{ - "cf1" => GcRule.max_age(30000) - }) + cf = %{ + "cf1" => GcRule.max_age(30_000) + } + + cf + |> Table.build() |> TableAdmin.create_table("created") {:ok, after_insert} = TableAdmin.list_tables() diff --git a/test/data/check_and_mutate_row_test.exs b/test/data/check_and_mutate_row_test.exs index 267d686..828d2d6 100644 --- a/test/data/check_and_mutate_row_test.exs +++ b/test/data/check_and_mutate_row_test.exs @@ -14,13 +14,14 @@ defmodule CheckAndMutateRowTest do qualifier = "column" {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", qualifier, "value", 0) |> MutateRow.build() |> MutateRow.mutate() on_exit(fn -> - mutation = Mutations.build(row_key) |> Mutations.delete_from_row() + mutation = row_key |> Mutations.build() |> Mutations.delete_from_row() mutation |> MutateRow.mutate() end) @@ -34,10 +35,11 @@ defmodule CheckAndMutateRowTest do describe "CheckAndMutateRow.mutate/2" do test "should apply a single true mutation when no predicate set and row exists", context do mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.if_true(mutation) |> CheckAndMutateRow.mutate() @@ -69,13 +71,14 @@ defmodule CheckAndMutateRowTest do test "should apply a multiple true mutation when no predicate set and row exists", context do mutation1 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) mutation2 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "alsoTruthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "alsoTruthy", "true", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.if_true([mutation1, mutation2]) |> CheckAndMutateRow.mutate() @@ -116,7 +119,7 @@ defmodule CheckAndMutateRowTest do test "should not apply a true mutation when no predicate set and row does not exist", context do mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) {:ok, _result} = CheckAndMutateRow.build("Doesnt#Exist") @@ -145,10 +148,11 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex(context.qualifier) mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_true(mutation) |> CheckAndMutateRow.mutate() @@ -183,13 +187,14 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex(context.qualifier) mutation1 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) mutation2 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "alsoTruthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "alsoTruthy", "true", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_true([mutation1, mutation2]) |> CheckAndMutateRow.mutate() @@ -232,10 +237,11 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex("doesntexist") mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "truthy", "true", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "truthy", "true", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_true(mutation) |> CheckAndMutateRow.mutate() @@ -262,10 +268,11 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex("doesntexist") mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "false", "false", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "false", "false", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_false(mutation) |> CheckAndMutateRow.mutate() @@ -300,13 +307,14 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex("doesntexist") mutation1 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "false", "false", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "false", "false", 0) mutation2 = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "false2", "false2", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "false2", "false2", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_false([mutation1, mutation2]) |> CheckAndMutateRow.mutate() @@ -349,10 +357,11 @@ defmodule CheckAndMutateRowTest do filter = RowFilter.column_qualifier_regex(context.qualifier) mutation = - Mutations.build(context.row_key) |> Mutations.set_cell("cf1", "false", "false", 0) + context.row_key |> Mutations.build() |> Mutations.set_cell("cf1", "false", "false", 0) {:ok, _result} = - CheckAndMutateRow.build(context.row_key) + context.row_key + |> CheckAndMutateRow.build() |> CheckAndMutateRow.predicate(filter) |> CheckAndMutateRow.if_false(mutation) |> CheckAndMutateRow.mutate() diff --git a/test/data/mutate_row_test.exs b/test/data/mutate_row_test.exs index 24ff71b..38d4e8d 100644 --- a/test/data/mutate_row_test.exs +++ b/test/data/mutate_row_test.exs @@ -1,6 +1,5 @@ defmodule MutateRowTest do @moduledoc false - # TODO: Integration tests including errors alias Bigtable.Data.{MutateRow, Mutations} use ExUnit.Case diff --git a/test/data/mutate_rows_test.exs b/test/data/mutate_rows_test.exs index 7498597..13022a8 100644 --- a/test/data/mutate_rows_test.exs +++ b/test/data/mutate_rows_test.exs @@ -1,6 +1,5 @@ defmodule MutateRowsTest do @moduledoc false - # TODO: Integration tests including errors alias Bigtable.Data.{MutateRows, Mutations} use ExUnit.Case diff --git a/test/data/read_modify_write_row_test.exs b/test/data/read_modify_write_row_test.exs index 0d0b0d5..d80dd3f 100644 --- a/test/data/read_modify_write_row_test.exs +++ b/test/data/read_modify_write_row_test.exs @@ -1,6 +1,6 @@ defmodule ReadModifyWriteRowTest do @moduledoc false - alias Bigtable.Data.{ReadModifyWriteRow, MutateRow, Mutations, ReadRows, RowFilter} + alias Bigtable.Data.{MutateRow, Mutations, ReadModifyWriteRow, ReadRows, RowFilter} use ExUnit.Case doctest ReadModifyWriteRow @@ -11,7 +11,7 @@ defmodule ReadModifyWriteRowTest do row_key = "Test#123" on_exit(fn -> - mutation = Mutations.build(row_key) |> Mutations.delete_from_row() + mutation = row_key |> Mutations.build() |> Mutations.delete_from_row() mutation |> MutateRow.mutate() end) @@ -28,12 +28,14 @@ defmodule ReadModifyWriteRowTest do val = <<1::integer-signed-64>> {:ok, _result} = - Mutations.build(context.row_key) + context.row_key + |> Mutations.build() |> Mutations.set_cell(context.family, qual, val, 0) |> MutateRow.mutate() {:ok, _result} = - ReadModifyWriteRow.build(context.row_key) + context.row_key + |> ReadModifyWriteRow.build() |> ReadModifyWriteRow.increment_amount(context.family, qual, 1) |> ReadModifyWriteRow.mutate() @@ -44,7 +46,7 @@ defmodule ReadModifyWriteRowTest do |> RowFilter.cells_per_column(1) |> ReadRows.read() - new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) + new_value = result |> Map.values() |> List.flatten() |> List.first() |> Map.get(:value) assert new_value == expected end @@ -53,7 +55,8 @@ defmodule ReadModifyWriteRowTest do qual = "num" {:ok, _result} = - ReadModifyWriteRow.build(context.row_key) + context.row_key + |> ReadModifyWriteRow.build() |> ReadModifyWriteRow.increment_amount(context.family, qual, 3) |> ReadModifyWriteRow.mutate() @@ -64,7 +67,7 @@ defmodule ReadModifyWriteRowTest do |> RowFilter.cells_per_column(1) |> ReadRows.read() - new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) + new_value = result |> Map.values() |> List.flatten() |> List.first() |> Map.get(:value) assert new_value == expected end @@ -74,12 +77,14 @@ defmodule ReadModifyWriteRowTest do val = "hello" {:ok, _result} = - Mutations.build(context.row_key) + context.row_key + |> Mutations.build() |> Mutations.set_cell(context.family, qual, val, 0) |> MutateRow.mutate() {:ok, _result} = - ReadModifyWriteRow.build(context.row_key) + context.row_key + |> ReadModifyWriteRow.build() |> ReadModifyWriteRow.append_value(context.family, qual, "world") |> ReadModifyWriteRow.mutate() @@ -90,7 +95,7 @@ defmodule ReadModifyWriteRowTest do |> RowFilter.cells_per_column(1) |> ReadRows.read() - new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) + new_value = result |> Map.values() |> List.flatten() |> List.first() |> Map.get(:value) assert new_value == expected end @@ -99,7 +104,8 @@ defmodule ReadModifyWriteRowTest do qual = "string" {:ok, _result} = - ReadModifyWriteRow.build(context.row_key) + context.row_key + |> ReadModifyWriteRow.build() |> ReadModifyWriteRow.append_value(context.family, qual, "world") |> ReadModifyWriteRow.mutate() @@ -110,7 +116,7 @@ defmodule ReadModifyWriteRowTest do |> RowFilter.cells_per_column(1) |> ReadRows.read() - new_value = Map.values(result) |> List.flatten() |> List.first() |> Map.get(:value) + new_value = result |> Map.values() |> List.flatten() |> List.first() |> Map.get(:value) assert new_value == expected end diff --git a/test/data/read_rows_test.exs b/test/data/read_rows_test.exs index 690e6d6..4b21dc0 100644 --- a/test/data/read_rows_test.exs +++ b/test/data/read_rows_test.exs @@ -1,5 +1,4 @@ defmodule ReadRowsTest do - # TODO: Integration tests including errors alias Bigtable.Data.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows} use ExUnit.Case diff --git a/test/data/row_filter_integration_test.exs b/test/data/row_filter_integration_test.exs index 2d6e830..5f9f945 100644 --- a/test/data/row_filter_integration_test.exs +++ b/test/data/row_filter_integration_test.exs @@ -157,14 +157,16 @@ defmodule RowFilterIntegration do test "should properly filter multiple rows based on value" do first_mutation = - Mutations.build("Test#1") + "Test#1" + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "foo", 0) |> Mutations.set_cell("cf1", "column2", "bar", 0) |> Mutations.set_cell("cf2", "column1", "bar", 0) |> Mutations.set_cell("cf2", "column2", "foo", 0) second_mutation = - Mutations.build("Test#2") + "Test#2" + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "foo", 0) |> Mutations.set_cell("cf1", "column2", "bar", 0) |> Mutations.set_cell("cf2", "column1", "bar", 0) @@ -1029,7 +1031,8 @@ defmodule RowFilterIntegration do [row_key | _rest] = context.row_keys {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column", "value", 0) |> MutateRow.mutate() @@ -1060,7 +1063,8 @@ defmodule RowFilterIntegration do [row_key | _rest] = context.row_keys {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "value", 0) |> Mutations.set_cell("cf1", "column2", "value", 0) |> Mutations.set_cell("cf1", "column3", "value", 0) @@ -1109,7 +1113,8 @@ defmodule RowFilterIntegration do [row_key | _rest] = context.row_keys {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "value", 4000) |> Mutations.set_cell("cf2", "column2", "value", 1000) |> Mutations.set_cell("cf1", "column2", "value", 1000) @@ -1136,7 +1141,8 @@ defmodule RowFilterIntegration do [row_key | _rest] = context.row_keys {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "value", 0) |> Mutations.set_cell("cf1", "column2", "value", 0) |> Mutations.set_cell("cf1", "column3", "value", 0) @@ -1226,7 +1232,8 @@ defmodule RowFilterIntegration do defp seed_timestamp_range(row_key) do {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "value1", 1000) |> Mutations.set_cell("cf1", "column1", "value2", 2000) |> Mutations.set_cell("cf1", "column1", "value3", 3000) @@ -1243,7 +1250,8 @@ defmodule RowFilterIntegration do defp seed_range(row_key) do {:ok, _} = - Mutations.build(row_key) + row_key + |> Mutations.build() |> Mutations.set_cell("cf1", "column1", "value1", 0) |> Mutations.set_cell("cf1", "column2", "value2", 0) |> Mutations.set_cell("cf1", "column3", "value3", 0) @@ -1258,7 +1266,8 @@ defmodule RowFilterIntegration do defp seed_values(context) do Enum.each(context.row_keys, fn key -> {:ok, _} = - Mutations.build(key) + key + |> Mutations.build() |> Mutations.set_cell("cf1", "column", "value", 0) |> MutateRow.build() |> MutateRow.mutate() diff --git a/test/data/row_filter_test.exs b/test/data/row_filter_test.exs index 56f3262..ce5f244 100644 --- a/test/data/row_filter_test.exs +++ b/test/data/row_filter_test.exs @@ -36,7 +36,7 @@ defmodule RowFilterTest do } ] - expected = expected_chain(filters) |> expected_request() + expected = filters |> expected_chain() |> expected_request() assert RowFilter.chain(context.request, filters) == expected end From b8aa8eff86f2056a6de6527e89f24bebc576594b Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 12:59:46 +0200 Subject: [PATCH 07/15] fixes more credo issues --- lib/data/chunk_reader.ex | 2 +- .../read_rows_acceptance_test.exs | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/data/chunk_reader.ex b/lib/data/chunk_reader.ex index 7921f3d..5e25074 100644 --- a/lib/data/chunk_reader.ex +++ b/lib/data/chunk_reader.ex @@ -46,7 +46,7 @@ defmodule Bigtable.Data.ChunkReader do @typedoc """ A map containging lists of `Bigtable.ChunkReader.ReadCell` keyed by row key. """ - @type chunk_reader_result :: %{optional(binary()) => [Bigtable.ChunkReader.ReadCell.t()]} + @type chunk_reader_result :: %{optional(binary()) => [ReadCell.t()]} def start_link(_) do GenServer.start_link(__MODULE__, %ReaderState{}, []) diff --git a/test/google_acceptance/read_rows_acceptance_test.exs b/test/google_acceptance/read_rows_acceptance_test.exs index 66e4c14..507d4b5 100644 --- a/test/google_acceptance/read_rows_acceptance_test.exs +++ b/test/google_acceptance/read_rows_acceptance_test.exs @@ -42,11 +42,7 @@ defmodule GoogleAcceptanceTest do true -> converted = processed_result - |> Enum.flat_map(fn {row_key, read_items} -> - read_items - |> Enum.map(&TestResult.from_chunk(row_key, &1)) - |> Enum.reverse() - end) + |> convert_result() assert converted == expected end @@ -105,4 +101,13 @@ defmodule ReadRowsAcceptanceTest do end defp results_error?(results), do: Enum.any?(results, &Map.get(&1, :error, false)) + + defp convert_result(result) do + result + |> Enum.flat_map(fn {row_key, read_items} -> + read_items + |> Enum.map(&TestResult.from_chunk(row_key, &1)) + |> Enum.reverse() + end) + end end From df3cea26e2787d8330f5572eb9560423626d2ce8 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 13:18:26 +0200 Subject: [PATCH 08/15] fixes last credo issue --- lib/data/read_rows.ex | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index 850909a..2b6d27d 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -101,15 +101,23 @@ defmodule Bigtable.Data.ReadRows do response |> Enum.filter(&contains_chunks?/1) |> Enum.flat_map(fn {:ok, resp} -> resp.chunks end) - |> Enum.reduce({:ok, %{}}, fn chunk, accum -> - if match?({:error, _}, accum) do - accum - else - ChunkReader.process(cr, chunk) - end - end) - - ChunkReader.close(cr) + |> process_response(nil, cr) + end + + defp process_response([], _result, chunk_reader) do + ChunkReader.close(chunk_reader) + end + + defp process_response(_chunks, {:error, _}, chunk_reader) do + ChunkReader.close(chunk_reader) + end + + defp process_response([h | t], _result, chunk_reader) do + result = + chunk_reader + |> ChunkReader.process(h) + + process_response(t, result, chunk_reader) end defp contains_chunks?({:ok, response}), do: !Enum.empty?(response.chunks) From 3fba3d42d21075b22841c6c0a17679067be6f4c6 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 13:45:01 +0200 Subject: [PATCH 09/15] moves request out into its own module --- lib/admin/table_admin/table_admin.ex | 10 +++--- lib/data/check_and_mutate_row.ex | 4 +-- lib/data/mutate_row.ex | 4 +-- lib/data/mutate_rows.ex | 4 +-- lib/data/read_modify_write_row.ex | 4 +-- lib/data/read_rows.ex | 4 +-- lib/data/sample_row_keys.ex | 4 +-- lib/request.ex | 52 ++++++++++++++++++++++++++++ lib/utils.ex | 51 ++------------------------- 9 files changed, 71 insertions(+), 66 deletions(-) create mode 100644 lib/request.ex diff --git a/lib/admin/table_admin/table_admin.ex b/lib/admin/table_admin/table_admin.ex index 57cc12d..e4135e2 100644 --- a/lib/admin/table_admin/table_admin.ex +++ b/lib/admin/table_admin/table_admin.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Admin.TableAdmin do @moduledoc """ Provides functions to build `Google.Bigtable.Admin.V2.ListTablesRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.Admin.V2 alias V2.BigtableTableAdmin.Stub @@ -10,7 +10,7 @@ defmodule Bigtable.Admin.TableAdmin do opts |> Keyword.put_new(:parent, Utils.configured_instance_name()) |> V2.ListTablesRequest.new() - |> Utils.process_request(&Stub.list_tables/3) + |> Request.process_request(&Stub.list_tables/3) end def create_table(table, table_id, opts \\ []) do @@ -20,16 +20,16 @@ defmodule Bigtable.Admin.TableAdmin do table: table, initial_splits: Keyword.get(opts, :initial_splits, []) ) - |> Utils.process_request(&Stub.create_table/3) + |> Request.process_request(&Stub.create_table/3) end def delete_table(name) do V2.DeleteTableRequest.new(name: name) - |> Utils.process_request(&Stub.delete_table/3) + |> Request.process_request(&Stub.delete_table/3) end def get_table(name) do V2.GetTableRequest.new(name: name) - |> Utils.process_request(&Stub.get_table/3) + |> Request.process_request(&Stub.get_table/3) end end diff --git a/lib/data/check_and_mutate_row.ex b/lib/data/check_and_mutate_row.ex index 9eaab76..eca257a 100644 --- a/lib/data/check_and_mutate_row.ex +++ b/lib/data/check_and_mutate_row.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.CheckAndMutateRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -79,7 +79,7 @@ defmodule Bigtable.Data.CheckAndMutateRow do {:ok, [V2.CheckAndMutateRowResponse]} | {:error, binary()} def mutate(%V2.CheckAndMutateRowRequest{} = request) do request - |> Utils.process_request(&Stub.check_and_mutate_row/3, single: true) + |> Request.process_request(&Stub.check_and_mutate_row/3, single: true) end defp extract_mutations(mutations) do diff --git a/lib/data/mutate_row.ex b/lib/data/mutate_row.ex index bbc972d..8dccce5 100644 --- a/lib/data/mutate_row.ex +++ b/lib/data/mutate_row.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.MutateRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub alias V2.MutateRowsRequest.Entry @@ -35,7 +35,7 @@ defmodule Bigtable.Data.MutateRow do @spec mutate(V2.MutateRowRequest.t()) :: {:ok, V2.MutateRowResponse.t()} | {:error, binary()} def mutate(%V2.MutateRowRequest{} = request) do request - |> Utils.process_request(&Stub.mutate_row/3, single: true) + |> Request.process_request(&Stub.mutate_row/3, single: true) end @spec mutate(V2.MutateRowsRequest.Entry.t()) :: diff --git a/lib/data/mutate_rows.ex b/lib/data/mutate_rows.ex index 165c0bd..e1cc480 100644 --- a/lib/data/mutate_rows.ex +++ b/lib/data/mutate_rows.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.MutateRows do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowsRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -37,7 +37,7 @@ defmodule Bigtable.Data.MutateRows do @spec mutate(V2.MutateRowsRequest.t()) :: {:ok, V2.MutateRowsResponse.t()} | {:error, binary()} def mutate(%V2.MutateRowsRequest{} = request) do request - |> Utils.process_request(&Stub.mutate_rows/3, stream: true) + |> Request.process_request(&Stub.mutate_rows/3, stream: true) end @spec mutate([V2.MutateRowsRequest.Entry.t()]) :: diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index f0f4674..7f12f63 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadModifyWriteRowRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -62,7 +62,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do {:ok, ReadModifyWriteRowResponse.t()} | {:error, binary()} def mutate(%ReadModifyWriteRowRequest{} = request) do request - |> Utils.process_request(&Stub.read_modify_write_row/3, single: true) + |> Request.process_request(&Stub.read_modify_write_row/3, single: true) end @spec add_rule(ReadModifyWriteRule.t(), ReadModifyWriteRowRequest.t()) :: diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index 2b6d27d..e2ef9e4 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -3,7 +3,7 @@ defmodule Bigtable.Data.ReadRows do Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. """ alias Bigtable.Data.ChunkReader - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -57,7 +57,7 @@ defmodule Bigtable.Data.ReadRows do def read(%V2.ReadRowsRequest{} = request) do result = request - |> Utils.process_request(&Stub.read_rows/3, stream: true) + |> Request.process_request(&Stub.read_rows/3, stream: true) case result do {:error, _} -> diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index 6963efc..c59e181 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.SampleRowKeys do @moduledoc """ Provides functions to build `Google.Bigtable.V2.SampleRowKeysRequest` and submit them to Bigtable. """ - alias Bigtable.Utils + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -40,7 +40,7 @@ defmodule Bigtable.Data.SampleRowKeys do @spec read(V2.SampleRowKeysRequest.t()) :: {:ok, V2.SampleRowKeysResponse} | {:error, any()} def read(%V2.SampleRowKeysRequest{} = request) do request - |> Utils.process_request(&Stub.sample_row_keys/3, stream: true) + |> Request.process_request(&Stub.sample_row_keys/3, stream: true) end @spec read() :: {:ok, V2.SampleRowKeysResponse} | {:error, any()} diff --git a/lib/request.ex b/lib/request.ex new file mode 100644 index 0000000..2360bca --- /dev/null +++ b/lib/request.ex @@ -0,0 +1,52 @@ +defmodule Bigtable.Request do + alias Bigtable.Connection + alias Connection.Worker + + def process_request(request, request_fn, opts \\ []) do + response = + :poolboy.transaction( + :connection_pool, + fn pid -> + connection = Worker.get_connection(pid) + metadata = Connection.get_metadata() + + connection + |> request_fn.(request, metadata) + end, + 10_000 + ) + + handle_response(response, opts) + end + + defp handle_response({:ok, response, _headers}, opts) do + if Keyword.get(opts, :stream, false) do + processed = + response + |> process_stream() + + {:ok, processed} + else + {:ok, response} + end + end + + defp handle_response(error, _opts) do + case error do + {:error, _msg} -> + error + + msg -> + {:error, msg} + end + end + + @spec process_stream(Enumerable.t()) :: [{atom(), any}] + defp process_stream(stream) do + stream + |> Stream.take_while(&remaining_resp?/1) + |> Enum.to_list() + end + + defp remaining_resp?({status, _}), do: status != :trailers +end diff --git a/lib/utils.ex b/lib/utils.ex index fab4030..edc9617 100644 --- a/lib/utils.ex +++ b/lib/utils.ex @@ -1,49 +1,11 @@ defmodule Bigtable.Utils do @moduledoc false - alias Bigtable.Connection - alias Connection.Worker - - def process_request(request, request_fn, opts \\ []) do - result = - :poolboy.transaction( - :connection_pool, - fn pid -> - connection = Worker.get_connection(pid) - metadata = Connection.get_metadata() - - connection - |> request_fn.(request, metadata) - end, - 10_000 - ) - - case result do - {:ok, response, _} -> - if Keyword.get(opts, :stream, false) do - processed = - response - |> process_stream() - - {:ok, processed} - else - {:ok, response} - end - - {:error, error} -> - {:error, inspect(error)} - - error -> - {:error, inspect(error)} - end - end - def configured_table_name do - project = get_project() - instance = get_instance() + instance = configured_instance_name() table = Application.get_env(:bigtable, :table) - "projects/#{project}/instances/#{instance}/tables/#{table}" + "#{instance}/tables/#{table}" end def configured_instance_name do @@ -59,13 +21,4 @@ defmodule Bigtable.Utils do defp get_instance do Application.get_env(:bigtable, :instance) end - - @spec process_stream(Enumerable.t()) :: [{atom(), any}] - defp process_stream(stream) do - stream - |> Stream.take_while(&remaining_resp?/1) - |> Enum.to_list() - end - - defp remaining_resp?({status, _}), do: status != :trailers end From c64c37cbdb267f0fe69aaec1848427a39f44e429 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 14:55:10 +0200 Subject: [PATCH 10/15] refactors operation modules --- lib/data/check_and_mutate_row.ex | 36 +++++-------- lib/data/mutate_row.ex | 16 +++--- lib/data/mutate_rows.ex | 14 ++--- lib/data/read_modify_write_row.ex | 7 +-- lib/data/read_rows.ex | 86 +++++++++---------------------- lib/data/sample_row_keys.ex | 10 +--- lib/request.ex | 10 ++-- 7 files changed, 65 insertions(+), 114 deletions(-) diff --git a/lib/data/check_and_mutate_row.ex b/lib/data/check_and_mutate_row.ex index eca257a..8c943fa 100644 --- a/lib/data/check_and_mutate_row.ex +++ b/lib/data/check_and_mutate_row.ex @@ -6,6 +6,8 @@ defmodule Bigtable.Data.CheckAndMutateRow do alias Google.Bigtable.V2 alias V2.Bigtable.Stub + @type entries() :: V2.MutateRowsRequest.Entry | [V2.MutateRowsRequest.Entry] + @doc """ Builds a `Google.Bigtable.V2.CheckAndMutateRowRequest` given a row_key and optional custom table name. @@ -25,19 +27,19 @@ defmodule Bigtable.Data.CheckAndMutateRow do } ### Custom Table - iex> table_name = "projects/[project_id]/instances/[instnace_id]/tables/[table_name]" + iex> table_name = "projects/[project_id]/instances/[instance_id]/tables/[table_name]" iex> Bigtable.Data.CheckAndMutateRow.build(table_name, "Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], predicate_filter: nil, row_key: "Test#123", - table_name: "projects/[project_id]/instances/[instnace_id]/tables/[table_name]", + table_name: "projects/[project_id]/instances/[instance_id]/tables/[table_name]", true_mutations: [] } """ @spec build(binary(), binary()) :: V2.CheckAndMutateRowRequest.t() - def build(table_name \\ Bigtable.Utils.configured_table_name(), row_key) + def build(table_name \\ Utils.configured_table_name(), row_key) when is_binary(table_name) and is_binary(row_key) do V2.CheckAndMutateRowRequest.new(table_name: table_name, app_profile_id: "", row_key: row_key) end @@ -48,30 +50,16 @@ defmodule Bigtable.Data.CheckAndMutateRow do %{request | predicate_filter: filter} end - @spec if_true(V2.CheckAndMutateRowRequest.t(), [V2.Mutation.t()]) :: - V2.CheckAndMutateRowRequest.t() - def if_true(%V2.CheckAndMutateRowRequest{} = request, mutations) when is_list(mutations) do + @spec if_true(V2.CheckAndMutateRowRequest.t(), entries) :: V2.CheckAndMutateRowRequest.t() + def if_true(%V2.CheckAndMutateRowRequest{} = request, mutations) do %{request | true_mutations: extract_mutations(mutations)} end - @spec if_true(V2.CheckAndMutateRowRequest.t(), V2.Mutation.t()) :: - V2.CheckAndMutateRowRequest.t() - def if_true(%V2.CheckAndMutateRowRequest{} = request, mutation) do - if_true(request, [mutation]) - end - - @spec if_false(V2.CheckAndMutateRowRequest.t(), [V2.Mutation.t()]) :: - V2.CheckAndMutateRowRequest.t() - def if_false(%V2.CheckAndMutateRowRequest{} = request, mutations) when is_list(mutations) do + @spec if_false(V2.CheckAndMutateRowRequest.t(), entries()) :: V2.CheckAndMutateRowRequest.t() + def if_false(%V2.CheckAndMutateRowRequest{} = request, mutations) do %{request | false_mutations: extract_mutations(mutations)} end - @spec if_false(V2.CheckAndMutateRowRequest.t(), V2.Mutation.t()) :: - V2.CheckAndMutateRowRequest.t() - def if_false(%V2.CheckAndMutateRowRequest{} = request, mutation) do - if_false(request, [mutation]) - end - @doc """ Submits a `Google.Bigtable.V2.CheckAndMutateRowRequest` to Bigtable. """ @@ -82,8 +70,10 @@ defmodule Bigtable.Data.CheckAndMutateRow do |> Request.process_request(&Stub.check_and_mutate_row/3, single: true) end - defp extract_mutations(mutations) do - mutations + @spec extract_mutations(entries()) :: [V2.Mutation.t()] + defp extract_mutations(entries) do + entries + |> List.wrap() |> Enum.flat_map(&Map.get(&1, :mutations)) end end diff --git a/lib/data/mutate_row.ex b/lib/data/mutate_row.ex index 8dccce5..1a1775c 100644 --- a/lib/data/mutate_row.ex +++ b/lib/data/mutate_row.ex @@ -2,11 +2,13 @@ defmodule Bigtable.Data.MutateRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowRequest` and submit them to Bigtable. """ - alias Bigtable.{Request, Utils} + alias Bigtable.Request alias Google.Bigtable.V2 alias V2.Bigtable.Stub alias V2.MutateRowsRequest.Entry + @type response :: {:ok, V2.MutateRowResponse.t()} | {:error, any()} + @doc """ Builds a `Google.Bigtable.V2.MutateRowRequest` with a provided table name and `Google.Bigtable.V2.MutateRowsRequest.Entry`. """ @@ -32,18 +34,16 @@ defmodule Bigtable.Data.MutateRow do Returns a `Google.Bigtable.V2.MutateRowResponse` """ - @spec mutate(V2.MutateRowRequest.t()) :: {:ok, V2.MutateRowResponse.t()} | {:error, binary()} + @spec mutate(V2.MutateRowRequest.t()) :: response() def mutate(%V2.MutateRowRequest{} = request) do request |> Request.process_request(&Stub.mutate_row/3, single: true) end - @spec mutate(V2.MutateRowsRequest.Entry.t()) :: - {:ok, V2.MutateRowResponse.t()} | {:error, binary()} - def mutate(%Entry{} = row_mutations) do - request = build(row_mutations) - - request + @spec mutate(V2.MutateRowsRequest.Entry.t()) :: response() + def mutate(%Entry{} = entry) do + entry + |> build() |> mutate() end end diff --git a/lib/data/mutate_rows.ex b/lib/data/mutate_rows.ex index e1cc480..dec8c86 100644 --- a/lib/data/mutate_rows.ex +++ b/lib/data/mutate_rows.ex @@ -2,10 +2,12 @@ defmodule Bigtable.Data.MutateRows do @moduledoc """ Provides functions to build `Google.Bigtable.V2.MutateRowsRequest` and submit them to Bigtable. """ - alias Bigtable.{Request, Utils} + alias Bigtable.Request alias Google.Bigtable.V2 alias V2.Bigtable.Stub + @type response :: {:ok, V2.MutateRowsResponse.t()} | {:error, any()} + @doc """ Builds a `Google.Bigtable.V2.MutateRowsRequest` with a provided table name and a list of `Google.Bigtable.V2.MutateRowsRequest.Entry`. """ @@ -34,18 +36,16 @@ defmodule Bigtable.Data.MutateRows do Returns a `Google.Bigtable.V2.MutateRowsResponse` """ - @spec mutate(V2.MutateRowsRequest.t()) :: {:ok, V2.MutateRowsResponse.t()} | {:error, binary()} + @spec mutate(V2.MutateRowsRequest.t()) :: response() def mutate(%V2.MutateRowsRequest{} = request) do request |> Request.process_request(&Stub.mutate_rows/3, stream: true) end - @spec mutate([V2.MutateRowsRequest.Entry.t()]) :: - {:ok, V2.MutateRowsResponse.t()} | {:error, binary()} + @spec mutate([V2.MutateRowsRequest.Entry.t()]) :: response() def mutate(entries) when is_list(entries) do - request = build(entries) - - request + entries + |> build() |> mutate end end diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index 7f12f63..b69f697 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do @moduledoc """ Provides functions to build `Google.Bigtable.V2.ReadModifyWriteRowRequest` and submit them to Bigtable. """ - alias Bigtable.{Request, Utils} + alias Bigtable.Request alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -12,6 +12,8 @@ defmodule Bigtable.Data.ReadModifyWriteRow do ReadModifyWriteRule } + @type response :: {:ok, ReadModifyWriteRowResponse.t()} | {:error, binary()} + @doc """ Builds a `Google.Bigtable.V2.ReadModifyWriteRowRequest` with a provided table name and row key """ @@ -58,8 +60,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do |> add_rule(request) end - @spec mutate(ReadModifyWriteRowRequest.t()) :: - {:ok, ReadModifyWriteRowResponse.t()} | {:error, binary()} + @spec mutate(ReadModifyWriteRowRequest.t()) :: response() def mutate(%ReadModifyWriteRowRequest{} = request) do request |> Request.process_request(&Stub.read_modify_write_row/3, single: true) diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index e2ef9e4..a4e14e3 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -7,6 +7,8 @@ defmodule Bigtable.Data.ReadRows do alias Google.Bigtable.V2 alias V2.Bigtable.Stub + @type response :: {:ok, ChunkReader.chunk_reader_result()} | {:error, any()} + @doc """ Builds a `Google.Bigtable.V2.ReadRowsRequest` with a provided table name. @@ -22,28 +24,10 @@ defmodule Bigtable.Data.ReadRows do } """ @spec build(binary()) :: V2.ReadRowsRequest.t() - def build(table_name) when is_binary(table_name) do + def build(table_name \\ Utils.configured_table_name()) when is_binary(table_name) do V2.ReadRowsRequest.new(table_name: table_name, app_profile_id: "") end - @doc """ - Builds a `Google.Bigtable.V2.ReadRowsRequest` with the configured table name. - - ## Examples - iex> Bigtable.Data.ReadRows.build() - %Google.Bigtable.V2.ReadRowsRequest{ - app_profile_id: "", - filter: nil, - rows: nil, - rows_limit: 0, - table_name: "projects/dev/instances/dev/tables/test" - } - """ - @spec build() :: V2.ReadRowsRequest.t() - def build do - build(Bigtable.Utils.configured_table_name()) - end - @doc """ Submits a `Google.Bigtable.V2.ReadRowsRequest` to Bigtable. @@ -51,73 +35,51 @@ defmodule Bigtable.Data.ReadRows do Returns a list of `{:ok, %Google.Bigtable.V2.ReadRowsResponse{}}`. """ - @spec read(V2.ReadRowsRequest.t()) :: - {:error, any()} - | {:ok, ChunkReader.chunk_reader_result()} - def read(%V2.ReadRowsRequest{} = request) do - result = - request - |> Request.process_request(&Stub.read_rows/3, stream: true) + @spec read(V2.ReadRowsRequest.t() | binary()) :: response() + def read(table_name \\ Utils.configured_table_name()) - case result do - {:error, _} -> - result - - {:ok, response} -> - process_response(response) - end + def read(%V2.ReadRowsRequest{} = request) do + request + |> Request.process_request(&Stub.read_rows/3, stream: true) + |> handle_response() end - @spec read(binary()) :: - {:error, any()} - | {:ok, ChunkReader.chunk_reader_result()} def read(table_name) when is_binary(table_name) do - request = build(table_name) - - request + table_name + |> build() |> read() end - @doc """ - Submits a `Google.Bigtable.V2.ReadRowsRequest` to Bigtable. - - Without arguments, `Bigtable.ReadRows.read` will read all rows from the configured table. - - Returns a list of `{:ok, %Google.Bigtable.V2.ReadRowsResponse{}}`. - """ - @spec read() :: - {:error, any()} - | [ok: V2.ReadRowsResponse.t()] - def read do - request = build() + defp handle_response({:error, _} = response), do: response - request - |> read + defp handle_response({:ok, response}) do + response + |> Enum.filter(&contains_chunks?/1) + |> Enum.flat_map(fn {:ok, resp} -> resp.chunks end) + |> process_chunks() end - defp process_response(response) do + defp process_chunks(chunks) do {:ok, cr} = ChunkReader.open() - response - |> Enum.filter(&contains_chunks?/1) - |> Enum.flat_map(fn {:ok, resp} -> resp.chunks end) - |> process_response(nil, cr) + chunks + |> process_chunks(nil, cr) end - defp process_response([], _result, chunk_reader) do + defp process_chunks([], _result, chunk_reader) do ChunkReader.close(chunk_reader) end - defp process_response(_chunks, {:error, _}, chunk_reader) do + defp process_chunks(_chunks, {:error, _}, chunk_reader) do ChunkReader.close(chunk_reader) end - defp process_response([h | t], _result, chunk_reader) do + defp process_chunks([h | t], _result, chunk_reader) do result = chunk_reader |> ChunkReader.process(h) - process_response(t, result, chunk_reader) + process_chunks(t, result, chunk_reader) end defp contains_chunks?({:ok, response}), do: !Enum.empty?(response.chunks) diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index c59e181..f78bd88 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -2,7 +2,7 @@ defmodule Bigtable.Data.SampleRowKeys do @moduledoc """ Provides functions to build `Google.Bigtable.V2.SampleRowKeysRequest` and submit them to Bigtable. """ - alias Bigtable.{Request, Utils} + alias Bigtable.Request alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -38,14 +38,8 @@ defmodule Bigtable.Data.SampleRowKeys do Submits a `Google.Bigtable.V2.SampleRowKeysRequest` to Bigtable. """ @spec read(V2.SampleRowKeysRequest.t()) :: {:ok, V2.SampleRowKeysResponse} | {:error, any()} - def read(%V2.SampleRowKeysRequest{} = request) do + def read(%V2.SampleRowKeysRequest{} = request \\ build()) do request |> Request.process_request(&Stub.sample_row_keys/3, stream: true) end - - @spec read() :: {:ok, V2.SampleRowKeysResponse} | {:error, any()} - def read do - build() - |> read() - end end diff --git a/lib/request.ex b/lib/request.ex index 2360bca..736be3d 100644 --- a/lib/request.ex +++ b/lib/request.ex @@ -1,16 +1,18 @@ defmodule Bigtable.Request do + @moduledoc false alias Bigtable.Connection alias Connection.Worker + @spec process_request(any(), function(), list()) :: {:ok, any()} | {:error, any()} def process_request(request, request_fn, opts \\ []) do response = :poolboy.transaction( :connection_pool, fn pid -> - connection = Worker.get_connection(pid) metadata = Connection.get_metadata() - connection + pid + |> Worker.get_connection() |> request_fn.(request, metadata) end, 10_000 @@ -19,6 +21,7 @@ defmodule Bigtable.Request do handle_response(response, opts) end + @spec handle_response(any(), list()) :: {:ok, any()} | {:error, any()} defp handle_response({:ok, response, _headers}, opts) do if Keyword.get(opts, :stream, false) do processed = @@ -41,12 +44,13 @@ defmodule Bigtable.Request do end end - @spec process_stream(Enumerable.t()) :: [{atom(), any}] + @spec process_stream(Enumerable.t()) :: [{:ok | :error, any}] defp process_stream(stream) do stream |> Stream.take_while(&remaining_resp?/1) |> Enum.to_list() end + @spec remaining_resp?({:ok | :error | :trailers, any()}) :: boolean() defp remaining_resp?({status, _}), do: status != :trailers end From ed39ad52eb93a3b03bdf8e6306a46b23e384cfe2 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Tue, 23 Apr 2019 15:15:49 +0200 Subject: [PATCH 11/15] makes operations documentation more consistent --- lib/data/check_and_mutate_row.ex | 10 +++++----- lib/data/mutate_row.ex | 19 ++++++------------- lib/data/mutate_rows.ex | 19 +++++-------------- lib/data/read_modify_write_row.ex | 4 ++-- lib/data/read_rows.ex | 14 +++++++------- lib/data/sample_row_keys.ex | 8 ++++---- 6 files changed, 29 insertions(+), 45 deletions(-) diff --git a/lib/data/check_and_mutate_row.ex b/lib/data/check_and_mutate_row.ex index 8c943fa..195066a 100644 --- a/lib/data/check_and_mutate_row.ex +++ b/lib/data/check_and_mutate_row.ex @@ -1,6 +1,6 @@ defmodule Bigtable.Data.CheckAndMutateRow do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. + Provides functionality for building and submitting a `Google.Bigtable.V2.CheckAndMutateRowRequest`. """ alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 @@ -9,9 +9,9 @@ defmodule Bigtable.Data.CheckAndMutateRow do @type entries() :: V2.MutateRowsRequest.Entry | [V2.MutateRowsRequest.Entry] @doc """ - Builds a `Google.Bigtable.V2.CheckAndMutateRowRequest` given a row_key and optional custom table name. + Builds a `Google.Bigtable.V2.CheckAndMutateRowRequest` given a row key and optional custom table name. - Defaults to configured table name. + Defaults to the configured table name if none is provided. ## Examples @@ -27,14 +27,14 @@ defmodule Bigtable.Data.CheckAndMutateRow do } ### Custom Table - iex> table_name = "projects/[project_id]/instances/[instance_id]/tables/[table_name]" + iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" iex> Bigtable.Data.CheckAndMutateRow.build(table_name, "Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], predicate_filter: nil, row_key: "Test#123", - table_name: "projects/[project_id]/instances/[instance_id]/tables/[table_name]", + table_name: "projects/project-id/instances/instance-id/tables/table-name", true_mutations: [] } """ diff --git a/lib/data/mutate_row.ex b/lib/data/mutate_row.ex index 1a1775c..93136a4 100644 --- a/lib/data/mutate_row.ex +++ b/lib/data/mutate_row.ex @@ -1,8 +1,8 @@ defmodule Bigtable.Data.MutateRow do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.MutateRowRequest` and submit them to Bigtable. + Provides functionality for building and submitting a `Google.Bigtable.V2.MutateRowRequest`. """ - alias Bigtable.Request + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub alias V2.MutateRowsRequest.Entry @@ -10,10 +10,11 @@ defmodule Bigtable.Data.MutateRow do @type response :: {:ok, V2.MutateRowResponse.t()} | {:error, any()} @doc """ - Builds a `Google.Bigtable.V2.MutateRowRequest` with a provided table name and `Google.Bigtable.V2.MutateRowsRequest.Entry`. + Builds a `Google.Bigtable.V2.MutateRowRequest` given a `Google.Bigtable.V2.MutateRowsRequest.Entry` and optional table name. """ @spec build(V2.MutateRowsRequest.Entry.t(), binary()) :: V2.MutateRowRequest.t() - def build(%Entry{} = row_mutations, table_name) when is_binary(table_name) do + def build(%Entry{} = row_mutations, table_name \\ Utils.configured_table_name()) + when is_binary(table_name) do V2.MutateRowRequest.new( table_name: table_name, row_key: row_mutations.row_key, @@ -21,18 +22,10 @@ defmodule Bigtable.Data.MutateRow do ) end - @doc """ - Builds a `Google.Bigtable.V2.MutateRowRequest` with default table name and provided `Google.Bigtable.V2.MutateRowsRequest.Entry`. - """ - @spec build(V2.MutateRowsRequest.Entry.t()) :: V2.MutateRowRequest.t() - def build(%Entry{} = row_mutations) do - build(row_mutations, Bigtable.Utils.configured_table_name()) - end - @doc """ Submits a `Google.Bigtable.V2.MutateRowRequest` given either a `Google.Bigtable.V2.MutateRowsRequest.Entry` or a `Google.Bigtable.V2.MutateRowRequest`. - Returns a `Google.Bigtable.V2.MutateRowResponse` + Returns a `Google.Bigtable.V2.MutateRowResponse`. """ @spec mutate(V2.MutateRowRequest.t()) :: response() def mutate(%V2.MutateRowRequest{} = request) do diff --git a/lib/data/mutate_rows.ex b/lib/data/mutate_rows.ex index dec8c86..6794a26 100644 --- a/lib/data/mutate_rows.ex +++ b/lib/data/mutate_rows.ex @@ -1,34 +1,25 @@ defmodule Bigtable.Data.MutateRows do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.MutateRowsRequest` and submit them to Bigtable. + Provides functionality for building and submitting a `Google.Bigtable.V2.MutateRowsRequest`. """ - alias Bigtable.Request + alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @type response :: {:ok, V2.MutateRowsResponse.t()} | {:error, any()} @doc """ - Builds a `Google.Bigtable.V2.MutateRowsRequest` with a provided table name and a list of `Google.Bigtable.V2.MutateRowsRequest.Entry`. + Builds a `Google.Bigtable.V2.MutateRowsRequest` given a `Google.Bigtable.V2.MutateRowsRequest.Entry` and optional table name. """ @spec build(list(V2.MutateRowsRequest.Entry.t()), binary()) :: V2.MutateRowsRequest.t() - def build(entries, table_name) when is_binary(table_name) and is_list(entries) do + def build(entries, table_name \\ Utils.configured_table_name()) + when is_binary(table_name) and is_list(entries) do V2.MutateRowsRequest.new( table_name: table_name, entries: entries ) end - @doc """ - Builds a `Google.Bigtable.V2.MutateRowsRequest` request with a list of `Google.Bigtable.V2.MutateRowsRequest.Entry`. - - Uses the configured table name. - """ - @spec build(list(V2.MutateRowsRequest.Entry.t())) :: V2.MutateRowsRequest.t() - def build(entries) when is_list(entries) do - build(entries, Bigtable.Utils.configured_table_name()) - end - @doc """ Submits a `Google.Bigtable.V2.MutateRowsRequest` to Bigtable. diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index b69f697..09c66f1 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -1,6 +1,6 @@ defmodule Bigtable.Data.ReadModifyWriteRow do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.ReadModifyWriteRowRequest` and submit them to Bigtable. + Provides functionality for building and submitting a `Google.Bigtable.V2.ReadModifyWriteRowRequest`. """ alias Bigtable.Request alias Google.Bigtable.V2 @@ -15,7 +15,7 @@ defmodule Bigtable.Data.ReadModifyWriteRow do @type response :: {:ok, ReadModifyWriteRowResponse.t()} | {:error, binary()} @doc """ - Builds a `Google.Bigtable.V2.ReadModifyWriteRowRequest` with a provided table name and row key + Builds a `Google.Bigtable.V2.ReadModifyWriteRowRequest` given a row key and optional table name. """ @spec build(binary(), binary()) :: ReadModifyWriteRowRequest.t() def build(table_name \\ Bigtable.Utils.configured_table_name(), row_key) diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index a4e14e3..dad8a99 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -1,6 +1,6 @@ defmodule Bigtable.Data.ReadRows do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.ReadRowsRequest` and submit them to Bigtable. + Provides functionality for to building and submitting a `Google.Bigtable.V2.ReadRowsRequest`. """ alias Bigtable.Data.ChunkReader alias Bigtable.{Request, Utils} @@ -10,17 +10,19 @@ defmodule Bigtable.Data.ReadRows do @type response :: {:ok, ChunkReader.chunk_reader_result()} | {:error, any()} @doc """ - Builds a `Google.Bigtable.V2.ReadRowsRequest` with a provided table name. + Builds a `Google.Bigtable.V2.ReadRowsRequest` given an optional table name. + + Defaults to the configured table name if none is provided. ## Examples - iex> table_name = "projects/[project_id]/instances/[instnace_id]/tables/[table_name]" + iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" iex> Bigtable.Data.ReadRows.build(table_name) %Google.Bigtable.V2.ReadRowsRequest{ app_profile_id: "", filter: nil, rows: nil, rows_limit: 0, - table_name: "projects/[project_id]/instances/[instnace_id]/tables/[table_name]" + table_name: "projects/project-id/instances/instance-id/tables/table-name" } """ @spec build(binary()) :: V2.ReadRowsRequest.t() @@ -31,9 +33,7 @@ defmodule Bigtable.Data.ReadRows do @doc """ Submits a `Google.Bigtable.V2.ReadRowsRequest` to Bigtable. - Can be called with either a `Google.Bigtable.V2.ReadRowsRequest` or a table name to read all rows from a non-configured table. - - Returns a list of `{:ok, %Google.Bigtable.V2.ReadRowsResponse{}}`. + Can be called with either a `Google.Bigtable.V2.ReadRowsRequest` or an optional table name. """ @spec read(V2.ReadRowsRequest.t() | binary()) :: response() def read(table_name \\ Utils.configured_table_name()) diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index f78bd88..7aba2ce 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -1,13 +1,13 @@ defmodule Bigtable.Data.SampleRowKeys do @moduledoc """ - Provides functions to build `Google.Bigtable.V2.SampleRowKeysRequest` and submit them to Bigtable. + Provides functionality for building and submitting `Google.Bigtable.V2.SampleRowKeysRequest`. """ alias Bigtable.Request alias Google.Bigtable.V2 alias V2.Bigtable.Stub @doc """ - Builds a `Google.Bigtable.V2.SampleRowKeysRequest` given a row_key and optional custom table name. + Builds a `Google.Bigtable.V2.SampleRowKeysRequest` given a row_key and optional table name. Defaults to configured table name. @@ -21,11 +21,11 @@ defmodule Bigtable.Data.SampleRowKeys do } ### Custom Table - iex> table_name = "projects/[project_id]/instances/[instance_id]/tables/[table_name]" + iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" iex> Bigtable.Data.SampleRowKeys.build(table_name) %Google.Bigtable.V2.SampleRowKeysRequest{ app_profile_id: "", - table_name: "projects/[project_id]/instances/[instance_id]/tables/[table_name]", + table_name: "projects/project-id/instances/instance-id/tables/table-name", } """ @spec build(binary()) :: V2.SampleRowKeysRequest.t() From 148ccbb6851fe3c5d30bfb86dd4894aec8d2dda3 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Wed, 24 Apr 2019 08:43:15 +0200 Subject: [PATCH 12/15] removes data prefix --- lib/data/check_and_mutate_row.ex | 6 +- lib/data/chunk_reader.ex | 2 +- lib/data/mutate_row.ex | 2 +- lib/data/mutate_rows.ex | 2 +- lib/data/mutations.ex | 4 +- lib/data/read_modify_write_row.ex | 2 +- lib/data/read_rows.ex | 6 +- lib/data/row_filter.ex | 58 +++++++++---------- lib/data/row_set.ex | 20 +++---- lib/data/sample_row_keys.ex | 6 +- lib/grpc/client_stub.ex | 6 +- lib/supervisor.ex | 2 +- mix.exs | 26 ++++----- test/data/check_and_mutate_row_test.exs | 2 +- test/data/mutate_row_test.exs | 2 +- test/data/mutate_rows_test.exs | 2 +- test/data/mutations_test.exs | 2 +- test/data/read_modify_write_row_test.exs | 2 +- test/data/read_rows_test.exs | 2 +- test/data/row_filter_integration_test.exs | 2 +- test/data/row_filter_test.exs | 2 +- test/data/row_set_test.exs | 2 +- test/data/sample_row_keys_test.exs | 2 +- .../read_rows_acceptance_test.exs | 6 +- 24 files changed, 84 insertions(+), 84 deletions(-) diff --git a/lib/data/check_and_mutate_row.ex b/lib/data/check_and_mutate_row.ex index 195066a..4248781 100644 --- a/lib/data/check_and_mutate_row.ex +++ b/lib/data/check_and_mutate_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.CheckAndMutateRow do +defmodule Bigtable.CheckAndMutateRow do @moduledoc """ Provides functionality for building and submitting a `Google.Bigtable.V2.CheckAndMutateRowRequest`. """ @@ -16,7 +16,7 @@ defmodule Bigtable.Data.CheckAndMutateRow do ## Examples ### Default Table - iex> Bigtable.Data.CheckAndMutateRow.build("Test#123") + iex> Bigtable.CheckAndMutateRow.build("Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], @@ -28,7 +28,7 @@ defmodule Bigtable.Data.CheckAndMutateRow do ### Custom Table iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" - iex> Bigtable.Data.CheckAndMutateRow.build(table_name, "Test#123") + iex> Bigtable.CheckAndMutateRow.build(table_name, "Test#123") %Google.Bigtable.V2.CheckAndMutateRowRequest{ app_profile_id: "", false_mutations: [], diff --git a/lib/data/chunk_reader.ex b/lib/data/chunk_reader.ex index 5e25074..7375e6d 100644 --- a/lib/data/chunk_reader.ex +++ b/lib/data/chunk_reader.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.ChunkReader do +defmodule Bigtable.ChunkReader do @moduledoc """ Reads chunks from `Google.Bigtable.V2.ReadRowsResponse` and parses them into complete cells grouped by rowkey. """ diff --git a/lib/data/mutate_row.ex b/lib/data/mutate_row.ex index 93136a4..5355cfe 100644 --- a/lib/data/mutate_row.ex +++ b/lib/data/mutate_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.MutateRow do +defmodule Bigtable.MutateRow do @moduledoc """ Provides functionality for building and submitting a `Google.Bigtable.V2.MutateRowRequest`. """ diff --git a/lib/data/mutate_rows.ex b/lib/data/mutate_rows.ex index 6794a26..9800d9d 100644 --- a/lib/data/mutate_rows.ex +++ b/lib/data/mutate_rows.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.MutateRows do +defmodule Bigtable.MutateRows do @moduledoc """ Provides functionality for building and submitting a `Google.Bigtable.V2.MutateRowsRequest`. """ diff --git a/lib/data/mutations.ex b/lib/data/mutations.ex index 196807e..885570f 100644 --- a/lib/data/mutations.ex +++ b/lib/data/mutations.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.Mutations do +defmodule Bigtable.Mutations do @moduledoc """ Provides functions to build Bigtable mutations that are used when forming row mutation requests. @@ -12,7 +12,7 @@ defmodule Bigtable.Data.Mutations do ## Examples - iex> Bigtable.Data.Mutations.build("Row#123") + iex> Bigtable.Mutations.build("Row#123") %Google.Bigtable.V2.MutateRowsRequest.Entry{mutations: [], row_key: "Row#123"} """ @spec build(binary()) :: Entry.t() diff --git a/lib/data/read_modify_write_row.ex b/lib/data/read_modify_write_row.ex index 09c66f1..b8ab806 100644 --- a/lib/data/read_modify_write_row.ex +++ b/lib/data/read_modify_write_row.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.ReadModifyWriteRow do +defmodule Bigtable.ReadModifyWriteRow do @moduledoc """ Provides functionality for building and submitting a `Google.Bigtable.V2.ReadModifyWriteRowRequest`. """ diff --git a/lib/data/read_rows.ex b/lib/data/read_rows.ex index dad8a99..6819c8c 100644 --- a/lib/data/read_rows.ex +++ b/lib/data/read_rows.ex @@ -1,8 +1,8 @@ -defmodule Bigtable.Data.ReadRows do +defmodule Bigtable.ReadRows do @moduledoc """ Provides functionality for to building and submitting a `Google.Bigtable.V2.ReadRowsRequest`. """ - alias Bigtable.Data.ChunkReader + alias Bigtable.ChunkReader alias Bigtable.{Request, Utils} alias Google.Bigtable.V2 alias V2.Bigtable.Stub @@ -16,7 +16,7 @@ defmodule Bigtable.Data.ReadRows do ## Examples iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" - iex> Bigtable.Data.ReadRows.build(table_name) + iex> Bigtable.ReadRows.build(table_name) %Google.Bigtable.V2.ReadRowsRequest{ app_profile_id: "", filter: nil, diff --git a/lib/data/row_filter.ex b/lib/data/row_filter.ex index b1ced4f..ca3fee7 100644 --- a/lib/data/row_filter.ex +++ b/lib/data/row_filter.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.RowFilter do +defmodule Bigtable.RowFilter do alias Google.Bigtable.V2.{ColumnRange, ReadRowsRequest, RowFilter, TimestampRange} @type column_range :: {binary(), binary(), boolean()} | {binary(), binary()} @@ -12,8 +12,8 @@ defmodule Bigtable.Data.RowFilter do ## Examples - iex> filters = [Bigtable.Data.RowFilter.cells_per_column(2), Bigtable.Data.RowFilter.row_key_regex("^Test#\w+")] - iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowFilter.chain(filters) + iex> filters = [Bigtable.RowFilter.cells_per_column(2), Bigtable.RowFilter.row_key_regex("^Test#\w+")] + iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowFilter.chain(filters) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:chain, @@ -40,7 +40,7 @@ defmodule Bigtable.Data.RowFilter do Adds a cells per column `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_column(2) + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_column(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_column_limit_filter, 2} @@ -58,7 +58,7 @@ defmodule Bigtable.Data.RowFilter do Creates a cells per column `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.cells_per_column(2) + iex> Bigtable.RowFilter.cells_per_column(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_column_limit_filter, 2} } @@ -73,7 +73,7 @@ defmodule Bigtable.Data.RowFilter do Adds a cells per row `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_row(2) + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_row(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_limit_filter, 2} @@ -91,7 +91,7 @@ defmodule Bigtable.Data.RowFilter do Creates a cells per row `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.cells_per_row(2) + iex> Bigtable.RowFilter.cells_per_row(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_limit_filter, 2} } @@ -106,7 +106,7 @@ defmodule Bigtable.Data.RowFilter do Adds a cells per row offset `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.cells_per_row_offset(2) + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.cells_per_row_offset(2) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_offset_filter, 2} @@ -124,7 +124,7 @@ defmodule Bigtable.Data.RowFilter do Creates a cells per row offset `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.cells_per_row_offset(2) + iex> Bigtable.RowFilter.cells_per_row_offset(2) %Google.Bigtable.V2.RowFilter{ filter: {:cells_per_row_offset_filter, 2} } @@ -139,7 +139,7 @@ defmodule Bigtable.Data.RowFilter do Adds a row key regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.row_key_regex("^Test#\\w+") + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.row_key_regex("^Test#\\w+") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:row_key_regex_filter, "^Test#\\w+"} @@ -157,7 +157,7 @@ defmodule Bigtable.Data.RowFilter do Creates a row key regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.row_key_regex("^Test#\\w+") + iex> Bigtable.RowFilter.row_key_regex("^Test#\\w+") %Google.Bigtable.V2.RowFilter{ filter: {:row_key_regex_filter, "^Test#\\w+"} } @@ -172,7 +172,7 @@ defmodule Bigtable.Data.RowFilter do Adds a value regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.value_regex("^test$") + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.value_regex("^test$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:value_regex_filter, "^test$"} @@ -190,7 +190,7 @@ defmodule Bigtable.Data.RowFilter do Creates a value regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.value_regex("^test$") + iex> Bigtable.RowFilter.value_regex("^test$") %Google.Bigtable.V2.RowFilter{ filter: {:value_regex_filter, "^test$"} } @@ -205,7 +205,7 @@ defmodule Bigtable.Data.RowFilter do Adds a family name regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.family_name_regex("^testFamily$") + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.family_name_regex("^testFamily$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:family_name_regex_filter, "^testFamily$"} @@ -223,7 +223,7 @@ defmodule Bigtable.Data.RowFilter do Creates a family name regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.family_name_regex("^testFamily$") + iex> Bigtable.RowFilter.family_name_regex("^testFamily$") %Google.Bigtable.V2.RowFilter{ filter: {:family_name_regex_filter, "^testFamily$"} } @@ -238,7 +238,7 @@ defmodule Bigtable.Data.RowFilter do Adds a column qualifier regex `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.column_qualifier_regex("^testColumn$") + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.column_qualifier_regex("^testColumn$") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:column_qualifier_regex_filter, "^testColumn$"} @@ -256,7 +256,7 @@ defmodule Bigtable.Data.RowFilter do Creates a family name regex `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.column_qualifier_regex("^testColumn$") + iex> Bigtable.RowFilter.column_qualifier_regex("^testColumn$") %Google.Bigtable.V2.RowFilter{ filter: {:column_qualifier_regex_filter, "^testColumn$"} } @@ -276,7 +276,7 @@ defmodule Bigtable.Data.RowFilter do ## Examples iex> range = {"column2", "column4"} - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.column_range("family", range) + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.column_range("family", range) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: { @@ -315,7 +315,7 @@ defmodule Bigtable.Data.RowFilter do ## Examples iex> range = {"column2", "column4"} - iex> Bigtable.Data.RowFilter.column_range("family", range) + iex> Bigtable.RowFilter.column_range("family", range) %Google.Bigtable.V2.RowFilter{ filter: { :column_range_filter, @@ -345,7 +345,7 @@ defmodule Bigtable.Data.RowFilter do ## Examples iex> range = [start_timestamp: 1000, end_timestamp: 2000] - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.timestamp_range(range) + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.timestamp_range(range) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: { @@ -373,7 +373,7 @@ defmodule Bigtable.Data.RowFilter do ## Examples iex> range = [start_timestamp: 1000, end_timestamp: 2000] - iex> Bigtable.Data.RowFilter.timestamp_range(range) + iex> Bigtable.RowFilter.timestamp_range(range) %Google.Bigtable.V2.RowFilter{ filter: { :timestamp_range_filter, @@ -400,7 +400,7 @@ defmodule Bigtable.Data.RowFilter do Adds a pass all `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.pass_all() + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.pass_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:pass_all_filter, true} @@ -420,7 +420,7 @@ defmodule Bigtable.Data.RowFilter do Matches all cells, regardless of input. Functionally equivalent to leaving filter unset, but included for completeness. ## Examples - iex> Bigtable.Data.RowFilter.pass_all() + iex> Bigtable.RowFilter.pass_all() %Google.Bigtable.V2.RowFilter{ filter: {:pass_all_filter, true} } @@ -435,7 +435,7 @@ defmodule Bigtable.Data.RowFilter do Adds a block all `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.block_all() + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.block_all() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:block_all_filter, true} @@ -455,7 +455,7 @@ defmodule Bigtable.Data.RowFilter do Does not match any cells, regardless of input. Useful for temporarily disabling just part of a filter. ## Examples - iex> Bigtable.Data.RowFilter.block_all() + iex> Bigtable.RowFilter.block_all() %Google.Bigtable.V2.RowFilter{ filter: {:block_all_filter, true} } @@ -470,7 +470,7 @@ defmodule Bigtable.Data.RowFilter do Adds a strip value transformer `Google.Bigtable.V2.RowFilter` a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.strip_value_transformer() + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.strip_value_transformer() iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:strip_value_transformer, true} @@ -488,7 +488,7 @@ defmodule Bigtable.Data.RowFilter do Creates a strip value transformer `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.strip_value_transformer() + iex> Bigtable.RowFilter.strip_value_transformer() %Google.Bigtable.V2.RowFilter{ filter: {:strip_value_transformer, true} } @@ -503,7 +503,7 @@ defmodule Bigtable.Data.RowFilter do Adds an apply label transformer `Google.Bigtable.V2.RowFilter` to a `Google.Bigtable.V2.ReadRowsRequest`. ## Examples - iex> request = Bigtable.Data.ReadRows.build() |> Bigtable.Data.RowFilter.apply_label_transformer("label") + iex> request = Bigtable.ReadRows.build() |> Bigtable.RowFilter.apply_label_transformer("label") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.filter %Google.Bigtable.V2.RowFilter{ filter: {:apply_label_transformer, "label"} @@ -520,7 +520,7 @@ defmodule Bigtable.Data.RowFilter do @doc """ Creates an apply label transformer `Google.Bigtable.V2.RowFilter`. ## Examples - iex> Bigtable.Data.RowFilter.apply_label_transformer("label") + iex> Bigtable.RowFilter.apply_label_transformer("label") %Google.Bigtable.V2.RowFilter{ filter: {:apply_label_transformer, "label"} } diff --git a/lib/data/row_set.ex b/lib/data/row_set.ex index 35b12fb..997c198 100644 --- a/lib/data/row_set.ex +++ b/lib/data/row_set.ex @@ -1,8 +1,8 @@ -defmodule Bigtable.Data.RowSet do +defmodule Bigtable.RowSet do @moduledoc """ Provides functions to build a `Google.Bigtable.V2.RowSet` and apply it to a `Google.Bigtable.V2.ReadRowsRequest` """ - alias Bigtable.Data.ReadRows + alias Bigtable.ReadRows alias Google.Bigtable.V2 @doc """ @@ -13,12 +13,12 @@ defmodule Bigtable.Data.RowSet do ## Examples #### Single Key - iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_keys("Row#123") + iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_keys("Row#123") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123"], row_ranges: []} #### Multiple Keys - iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_keys(["Row#123", "Row#124"]) + iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_keys(["Row#123", "Row#124"]) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123", "Row#124"], row_ranges: []} """ @@ -42,12 +42,12 @@ defmodule Bigtable.Data.RowSet do ## Examples #### Single Key - iex> request = Bigtable.Data.RowSet.row_keys("Row#123") + iex> request = Bigtable.RowSet.row_keys("Row#123") iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123"], row_ranges: []} #### Multiple Keys - iex> request = Bigtable.Data.RowSet.row_keys(["Row#123", "Row#124"]) + iex> request = Bigtable.RowSet.row_keys(["Row#123", "Row#124"]) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{row_keys: ["Row#123", "Row#124"], row_ranges: []} """ @@ -71,7 +71,7 @@ defmodule Bigtable.Data.RowSet do ## Examples #### Single Range - iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_ranges({"start", "end"}) + iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_ranges({"start", "end"}) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -86,7 +86,7 @@ defmodule Bigtable.Data.RowSet do #### Multiple Ranges iex> ranges = [{"start1", "end1"}, {"start2", "end2", false}] - iex> request = Bigtable.Data.ReadRows.build("table") |> Bigtable.Data.RowSet.row_ranges(ranges) + iex> request = Bigtable.ReadRows.build("table") |> Bigtable.RowSet.row_ranges(ranges) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -128,7 +128,7 @@ defmodule Bigtable.Data.RowSet do ## Examples #### Single Range - iex> request = Bigtable.Data.RowSet.row_ranges({"start", "end"}) + iex> request = Bigtable.RowSet.row_ranges({"start", "end"}) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], @@ -143,7 +143,7 @@ defmodule Bigtable.Data.RowSet do #### Multiple Ranges iex> ranges = [{"start1", "end1"}, {"start2", "end2", false}] - iex> request = Bigtable.Data.RowSet.row_ranges(ranges) + iex> request = Bigtable.RowSet.row_ranges(ranges) iex> with %Google.Bigtable.V2.ReadRowsRequest{} <- request, do: request.rows %Google.Bigtable.V2.RowSet{ row_keys: [], diff --git a/lib/data/sample_row_keys.ex b/lib/data/sample_row_keys.ex index 7aba2ce..3950693 100644 --- a/lib/data/sample_row_keys.ex +++ b/lib/data/sample_row_keys.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.SampleRowKeys do +defmodule Bigtable.SampleRowKeys do @moduledoc """ Provides functionality for building and submitting `Google.Bigtable.V2.SampleRowKeysRequest`. """ @@ -14,7 +14,7 @@ defmodule Bigtable.Data.SampleRowKeys do ## Examples ### Default Table - iex> Bigtable.Data.SampleRowKeys.build() + iex> Bigtable.SampleRowKeys.build() %Google.Bigtable.V2.SampleRowKeysRequest{ app_profile_id: "", table_name: "projects/dev/instances/dev/tables/test", @@ -22,7 +22,7 @@ defmodule Bigtable.Data.SampleRowKeys do ### Custom Table iex> table_name = "projects/project-id/instances/instance-id/tables/table-name" - iex> Bigtable.Data.SampleRowKeys.build(table_name) + iex> Bigtable.SampleRowKeys.build(table_name) %Google.Bigtable.V2.SampleRowKeysRequest{ app_profile_id: "", table_name: "projects/project-id/instances/instance-id/tables/table-name", diff --git a/lib/grpc/client_stub.ex b/lib/grpc/client_stub.ex index 9a07f2f..8f40d09 100644 --- a/lib/grpc/client_stub.ex +++ b/lib/grpc/client_stub.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Data.Service do +defmodule Bigtable.Service do @moduledoc false use GRPC.Service, name: "google.bigtable.v2.Bigtable" @@ -18,7 +18,7 @@ defmodule Bigtable.Data.Service do ) end -defmodule Bigtable.Data.Stub do +defmodule Bigtable.Stub do @moduledoc false - use GRPC.Stub, service: Bigtable.Data.Service + use GRPC.Stub, service: Bigtable.Service end diff --git a/lib/supervisor.ex b/lib/supervisor.ex index e53b1f3..f4d6b5d 100644 --- a/lib/supervisor.ex +++ b/lib/supervisor.ex @@ -9,7 +9,7 @@ defmodule Bigtable.Supervisor do def init(:ok) do children = [ Bigtable.Connection, - {DynamicSupervisor, name: Bigtable.Data.ChunkReader.Supervisor, strategy: :one_for_one} + {DynamicSupervisor, name: Bigtable.ChunkReader.Supervisor, strategy: :one_for_one} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/mix.exs b/mix.exs index e4c45e3..97c35c8 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Bigtable.MixProject do use Mix.Project - alias Bigtable.{Admin, Connection, Data} + alias Bigtable.{Admin, Connection} @version "0.6.1" @@ -58,7 +58,7 @@ defmodule Bigtable.MixProject do extras: extras(), groups_for_extras: groups_for_extras(), nest_modules_by_prefix: [ - Bigtable.Data.ChunkReader + Bigtable.ChunkReader ] ] end @@ -90,17 +90,17 @@ defmodule Bigtable.MixProject do Connection ], Data: [ - Data.CheckAndMutateRow, - Data.ChunkReader, - Data.ChunkReader.ReadCell, - Data.MutateRow, - Data.MutateRows, - Data.Mutations, - Data.ReadModifyWriteRow, - Data.ReadRows, - Data.RowFilter, - Data.RowSet, - Data.SampleRowKeys + Bigtable.CheckAndMutateRow, + Bigtable.ChunkReader, + Bigtable.ChunkReader.ReadCell, + Bigtable.MutateRow, + Bigtable.MutateRows, + Bigtable.Mutations, + Bigtable.ReadModifyWriteRow, + Bigtable.ReadRows, + Bigtable.RowFilter, + Bigtable.RowSet, + Bigtable.SampleRowKeys ] ] end diff --git a/test/data/check_and_mutate_row_test.exs b/test/data/check_and_mutate_row_test.exs index 828d2d6..a2cfce3 100644 --- a/test/data/check_and_mutate_row_test.exs +++ b/test/data/check_and_mutate_row_test.exs @@ -1,6 +1,6 @@ defmodule CheckAndMutateRowTest do @moduledoc false - alias Bigtable.Data.{CheckAndMutateRow, ChunkReader, MutateRow, Mutations, ReadRows, RowFilter} + alias Bigtable.{CheckAndMutateRow, ChunkReader, MutateRow, Mutations, ReadRows, RowFilter} alias ChunkReader.ReadCell use ExUnit.Case diff --git a/test/data/mutate_row_test.exs b/test/data/mutate_row_test.exs index 38d4e8d..154fc0c 100644 --- a/test/data/mutate_row_test.exs +++ b/test/data/mutate_row_test.exs @@ -1,6 +1,6 @@ defmodule MutateRowTest do @moduledoc false - alias Bigtable.Data.{MutateRow, Mutations} + alias Bigtable.{MutateRow, Mutations} use ExUnit.Case diff --git a/test/data/mutate_rows_test.exs b/test/data/mutate_rows_test.exs index 13022a8..033a2b6 100644 --- a/test/data/mutate_rows_test.exs +++ b/test/data/mutate_rows_test.exs @@ -1,7 +1,7 @@ defmodule MutateRowsTest do @moduledoc false - alias Bigtable.Data.{MutateRows, Mutations} + alias Bigtable.{MutateRows, Mutations} use ExUnit.Case setup do diff --git a/test/data/mutations_test.exs b/test/data/mutations_test.exs index cb0f6d7..041944e 100644 --- a/test/data/mutations_test.exs +++ b/test/data/mutations_test.exs @@ -1,5 +1,5 @@ defmodule MutationsTest do - alias Bigtable.Data.Mutations + alias Bigtable.Mutations alias Google.Bigtable.V2.MutateRowsRequest.Entry use ExUnit.Case diff --git a/test/data/read_modify_write_row_test.exs b/test/data/read_modify_write_row_test.exs index d80dd3f..4634577 100644 --- a/test/data/read_modify_write_row_test.exs +++ b/test/data/read_modify_write_row_test.exs @@ -1,6 +1,6 @@ defmodule ReadModifyWriteRowTest do @moduledoc false - alias Bigtable.Data.{MutateRow, Mutations, ReadModifyWriteRow, ReadRows, RowFilter} + alias Bigtable.{MutateRow, Mutations, ReadModifyWriteRow, ReadRows, RowFilter} use ExUnit.Case doctest ReadModifyWriteRow diff --git a/test/data/read_rows_test.exs b/test/data/read_rows_test.exs index 4b21dc0..67f2484 100644 --- a/test/data/read_rows_test.exs +++ b/test/data/read_rows_test.exs @@ -1,5 +1,5 @@ defmodule ReadRowsTest do - alias Bigtable.Data.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows} + alias Bigtable.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows} use ExUnit.Case doctest ReadRows diff --git a/test/data/row_filter_integration_test.exs b/test/data/row_filter_integration_test.exs index 5f9f945..2838143 100644 --- a/test/data/row_filter_integration_test.exs +++ b/test/data/row_filter_integration_test.exs @@ -1,6 +1,6 @@ defmodule RowFilterIntegration do @moduledoc false - alias Bigtable.Data.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows, RowFilter} + alias Bigtable.{ChunkReader, MutateRow, MutateRows, Mutations, ReadRows, RowFilter} alias ChunkReader.ReadCell alias Google.Protobuf.{BytesValue, StringValue} diff --git a/test/data/row_filter_test.exs b/test/data/row_filter_test.exs index ce5f244..6619fa4 100644 --- a/test/data/row_filter_test.exs +++ b/test/data/row_filter_test.exs @@ -1,6 +1,6 @@ defmodule RowFilterTest do @moduledoc false - alias Bigtable.Data.RowFilter + alias Bigtable.RowFilter use ExUnit.Case diff --git a/test/data/row_set_test.exs b/test/data/row_set_test.exs index 1f5533b..6fa5ba3 100644 --- a/test/data/row_set_test.exs +++ b/test/data/row_set_test.exs @@ -1,5 +1,5 @@ defmodule RowSetTest do - alias Bigtable.Data.RowSet + alias Bigtable.RowSet use ExUnit.Case diff --git a/test/data/sample_row_keys_test.exs b/test/data/sample_row_keys_test.exs index 1a7cf7d..f733685 100644 --- a/test/data/sample_row_keys_test.exs +++ b/test/data/sample_row_keys_test.exs @@ -1,6 +1,6 @@ defmodule SampleRowKeysTest do @moduledoc false - alias Bigtable.Data.SampleRowKeys + alias Bigtable.SampleRowKeys use ExUnit.Case diff --git a/test/google_acceptance/read_rows_acceptance_test.exs b/test/google_acceptance/read_rows_acceptance_test.exs index 507d4b5..6ae3038 100644 --- a/test/google_acceptance/read_rows_acceptance_test.exs +++ b/test/google_acceptance/read_rows_acceptance_test.exs @@ -1,5 +1,5 @@ defmodule TestResult do - alias Bigtable.Data.ChunkReader.ReadCell + alias Bigtable.ChunkReader.ReadCell alias Google.Bigtable.V2.ReadRowsResponse.CellChunk def from_chunk(row_key, %ReadCell{} = ri) do @@ -16,7 +16,7 @@ defmodule TestResult do end defmodule GoogleAcceptanceTest do - alias Bigtable.Data.ChunkReader + alias Bigtable.ChunkReader defmacro __using__(json: json) do json @@ -53,7 +53,7 @@ defmodule GoogleAcceptanceTest do end defmodule ReadRowsAcceptanceTest do - alias Bigtable.Data.ChunkReader + alias Bigtable.ChunkReader alias Google.Bigtable.V2.ReadRowsResponse.CellChunk use ExUnit.Case From 3176ca247cc5f6824acba413f71bafc12001cff3 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Wed, 24 Apr 2019 09:08:49 +0200 Subject: [PATCH 13/15] refactors connection and auth modules --- lib/{connection => }/auth.ex | 2 +- lib/connection.ex | 77 +++++++++++++++++++++ lib/connection/connection.ex | 102 ---------------------------- lib/request.ex | 12 +++- test/connection/connection_test.exs | 2 +- 5 files changed, 88 insertions(+), 107 deletions(-) rename lib/{connection => }/auth.ex (97%) create mode 100644 lib/connection.ex delete mode 100644 lib/connection/connection.ex diff --git a/lib/connection/auth.ex b/lib/auth.ex similarity index 97% rename from lib/connection/auth.ex rename to lib/auth.ex index d0f59cb..b3ca805 100644 --- a/lib/connection/auth.ex +++ b/lib/auth.ex @@ -1,4 +1,4 @@ -defmodule Bigtable.Connection.Auth do +defmodule Bigtable.Auth do @moduledoc false @scopes [ diff --git a/lib/connection.ex b/lib/connection.ex new file mode 100644 index 0000000..a482ac4 --- /dev/null +++ b/lib/connection.ex @@ -0,0 +1,77 @@ +defmodule Bigtable.Connection do + @moduledoc false + + use GenServer + @default_endpoint "bigtable.googleapis.com:443" + + ## Client API + def start_link(_opts) do + GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + end + + @doc """ + Connects to Bigtable and returns a `GRPC.Channel`. + """ + @spec connect() :: GRPC.Channel.t() + def connect do + GenServer.call(__MODULE__, :connect) + end + + @doc """ + Disconnects from the provided `GRPC.Channel`. + """ + @spec disconnect(GRPC.Channel.t()) :: :ok + def disconnect(channel) do + GenServer.cast(__MODULE__, {:disconnect, channel}) + end + + # Server Callbacks + @spec init(:ok) :: {:ok, map()} + def init(:ok) do + {:ok, %{endpoint: get_endpoint(), opts: build_opts()}} + end + + def handle_call(:connect, _from, %{endpoint: endpoint, opts: opts} = state) do + {:ok, channel} = + GRPC.Stub.connect( + endpoint, + opts + ) + + {:reply, channel, state} + end + + def handle_cast({:disconnect, channel}, state) do + GRPC.Stub.disconnect(channel) + {:noreply, state} + end + + def handle_info(_msg, state) do + {:noreply, state} + end + + @spec build_opts() :: list() + defp build_opts do + if Application.get_env(:bigtable, :ssl, true) do + [ + cred: %GRPC.Credential{ + ssl: [] + } + ] + else + [] + end + end + + @spec get_endpoint() :: binary() + def get_endpoint do + emulator = System.get_env("BIGTABLE_EMULATOR_HOST") + endpoint = Application.get_env(:bigtable, :endpoint, @default_endpoint) + + if emulator != nil do + emulator + else + endpoint + end + end +end diff --git a/lib/connection/connection.ex b/lib/connection/connection.ex deleted file mode 100644 index ebed6b2..0000000 --- a/lib/connection/connection.ex +++ /dev/null @@ -1,102 +0,0 @@ -defmodule Bigtable.Connection do - @moduledoc """ - Holds the configured gRPC connection to Bigtable - """ - use GenServer - - alias Bigtable.Connection.Auth - - @default_host "bigtable.googleapis.com:443" - @default_opts [] - - ## Client API - @doc false - - def start_link(_opts) do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) - end - - @doc """ - Returns the configured `GRPC.Channel` - """ - @spec connect() :: GRPC.Channel.t() - def connect do - GenServer.call(__MODULE__, :connect) - end - - def disconnect(channel) do - GenServer.cast(__MODULE__, {:disconnect, channel}) - end - - @spec get_metadata() :: Keyword.t() - def get_metadata do - token = Auth.get_token() - metadata = %{authorization: "Bearer #{token.token}"} - [metadata: metadata, content_type: "application/grpc", return_headers: true] - end - - # Server Callbacks - @doc false - @spec init(:ok) :: {:ok, map()} - def init(:ok) do - # Fetches the url to use for Bigtable gRPC connection - endpoint = - case get_custom_endpoint() do - nil -> @default_host - custom -> custom - end - - opts = build_opts() - - # Connects the stub to the Bigtable gRPC server - - {:ok, %{endpoint: endpoint, opts: opts}} - end - - def handle_call(:connect, _from, %{endpoint: endpoint, opts: opts} = state) do - {:ok, channel} = - GRPC.Stub.connect( - endpoint, - opts - ) - - {:reply, channel, state} - end - - def handle_cast({:disconnect, channel}, state) do - GRPC.Stub.disconnect(channel) - {:noreply, state} - end - - def handle_info(_msg, state) do - {:noreply, state} - end - - defp build_opts do - case Application.get_env(:bigtable, :ssl, true) do - true -> - @default_opts ++ - [ - cred: %GRPC.Credential{ - ssl: [] - } - ] - - false -> - @default_opts - end - end - - def get_custom_endpoint do - env = System.get_env("BIGTABLE_EMULATOR_HOST") - custom = Application.get_env(:bigtable, :endpoint, nil) - - case env do - nil -> - custom - - endpoint -> - endpoint - end - end -end diff --git a/lib/request.ex b/lib/request.ex index 736be3d..f7da3df 100644 --- a/lib/request.ex +++ b/lib/request.ex @@ -1,6 +1,6 @@ defmodule Bigtable.Request do @moduledoc false - alias Bigtable.Connection + alias Bigtable.{Auth, Connection} alias Connection.Worker @spec process_request(any(), function(), list()) :: {:ok, any()} | {:error, any()} @@ -9,11 +9,11 @@ defmodule Bigtable.Request do :poolboy.transaction( :connection_pool, fn pid -> - metadata = Connection.get_metadata() + token = Auth.get_token() pid |> Worker.get_connection() - |> request_fn.(request, metadata) + |> request_fn.(request, get_metadata(token)) end, 10_000 ) @@ -53,4 +53,10 @@ defmodule Bigtable.Request do @spec remaining_resp?({:ok | :error | :trailers, any()}) :: boolean() defp remaining_resp?({status, _}), do: status != :trailers + + @spec get_metadata(map()) :: Keyword.t() + defp get_metadata(%{token: token}) do + metadata = %{authorization: "Bearer #{token}"} + [metadata: metadata, content_type: "application/grpc", return_headers: true] + end end diff --git a/test/connection/connection_test.exs b/test/connection/connection_test.exs index 88c030e..f060516 100644 --- a/test/connection/connection_test.exs +++ b/test/connection/connection_test.exs @@ -7,7 +7,7 @@ defmodule ConnectionTest do describe "Connection.get_connection() " do test "should return a Channel struct" do [host, port] = - Connection.get_custom_endpoint() + Connection.get_endpoint() |> String.split(":") expected = %GRPC.Channel{ From f649bdba1bbbb7814d81de74fdd1f5398da7fe46 Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Wed, 24 Apr 2019 09:11:08 +0200 Subject: [PATCH 14/15] refactors utils --- lib/utils.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/utils.ex b/lib/utils.ex index edc9617..29d5aab 100644 --- a/lib/utils.ex +++ b/lib/utils.ex @@ -1,6 +1,7 @@ defmodule Bigtable.Utils do @moduledoc false + @spec configured_table_name() :: binary() def configured_table_name do instance = configured_instance_name() table = Application.get_env(:bigtable, :table) @@ -8,16 +9,19 @@ defmodule Bigtable.Utils do "#{instance}/tables/#{table}" end + @spec configured_instance_name() :: binary() def configured_instance_name do project = get_project() instance = get_instance() "projects/#{project}/instances/#{instance}" end + @spec get_project() :: binary() defp get_project do Application.get_env(:bigtable, :project) end + @spec get_instance() :: binary() defp get_instance do Application.get_env(:bigtable, :instance) end From 05fa618d4ab4f348cbfd9117b9fc982cd452a71d Mon Sep 17 00:00:00 2001 From: Jason Scott Date: Wed, 24 Apr 2019 09:25:54 +0200 Subject: [PATCH 15/15] bumps version and updates installation guide --- guides/introduction/installation.md | 20 ++++++++++++++------ mix.exs | 4 +--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/guides/introduction/installation.md b/guides/introduction/installation.md index 58cc527..51d91ae 100644 --- a/guides/introduction/installation.md +++ b/guides/introduction/installation.md @@ -6,7 +6,7 @@ #mix.exs def deps do [ - {:bigtable, "~> 0.1.0"}, + {:bigtable, "~> 0.7.0"}, ] end ``` @@ -18,11 +18,14 @@ end ```elixir #dev.exs config :bigtable, - project: "project_id", - instance: "instance_id", - table: "table_name", - endpoint: "localhost:8086" + project: "project", + instance: "instance", + table: "table_name", # Default table name to use in requests + endpoint: "localhost:9035", ssl: false + +config :goth, + disabled: true ``` #### Production Configuration @@ -32,5 +35,10 @@ config :bigtable, config :bigtable, project: "project_id", instance: "instance_id", - table: "table_name" + # Default table name to use in requests + table: "table_name", + # Optional connection pool size. Defaults to 128 + pool_size: 128, + # Optional connection pool overflow when size is exceeded + pool_overflow: 128 ``` diff --git a/mix.exs b/mix.exs index 97c35c8..77fa1f7 100644 --- a/mix.exs +++ b/mix.exs @@ -3,7 +3,7 @@ defmodule Bigtable.MixProject do alias Bigtable.{Admin, Connection} - @version "0.6.1" + @version "0.7.0" def project do [ @@ -67,14 +67,12 @@ defmodule Bigtable.MixProject do [ "guides/introduction/overview.md", "guides/introduction/installation.md" - # "guides/operations/read_rows.md" ] end defp groups_for_extras do [ Introduction: ~r/guides\/introduction\/.?/ - # Operations: ~r/guides\/operations\/.?/ ] end