From 552847e58131c56cbaa1e3962255c4236cce99c8 Mon Sep 17 00:00:00 2001 From: Peter Hartman Date: Sun, 29 Oct 2023 12:44:58 +0000 Subject: [PATCH] Fix for /issues/5 --- .vscode/settings.json | 2 - lib/data_layer/data_layer.ex | 29 +++++---- lib/paginator/continuation_property.ex | 4 ++ lib/paginator/paginator.ex | 3 + mix.exs | 5 +- mix.lock | 3 +- test/custom_pagination_test.exs | 82 +++++++++++++++++++++++--- test/test_helper.exs | 1 + 8 files changed, 104 insertions(+), 25 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 7a73a41..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file diff --git a/lib/data_layer/data_layer.ex b/lib/data_layer/data_layer.ex index b019df0..03e097f 100644 --- a/lib/data_layer/data_layer.ex +++ b/lib/data_layer/data_layer.ex @@ -617,24 +617,31 @@ defmodule AshJsonApiWrapper.DataLayer do Logger.warning( "ash_json_api_wrapper does not support limits with offsets yet, and so they will both be applied after." ) + end - query - else - case endpoint.limit_with do - {:param, param} -> - %{ - query - | query_params: Map.put(query.query_params || %{}, param, query.limit) - } - - _ -> + case endpoint.limit_with do + {:param, param} -> + %{ query - end + | query_params: Map.put(query.query_params || %{}, param, query.limit) + } + + _ -> + query end else query end + query = + if endpoint.paginator do + %{paginator: {module, opts}} = endpoint + {:ok, instructions} = module.start(opts) + apply_instructions(query, instructions) + else + query + end + with {:ok, %{status: status} = response} when status >= 200 and status < 300 <- make_request(resource, query), {:ok, body} <- Jason.decode(response.body), diff --git a/lib/paginator/continuation_property.ex b/lib/paginator/continuation_property.ex index 9dcad4d..a257f13 100644 --- a/lib/paginator/continuation_property.ex +++ b/lib/paginator/continuation_property.ex @@ -2,6 +2,10 @@ defmodule AshJsonApiWrapper.Paginator.ContinuationProperty do @moduledoc "A paginator that uses a continuation property to paginate" use AshJsonApiWrapper.Paginator + def start(_opts) do + {:ok, %{}} + end + def continue(_response, [], _), do: :halt def continue(response, _entities, opts) do diff --git a/lib/paginator/paginator.ex b/lib/paginator/paginator.ex index 74bce6a..3f2df37 100644 --- a/lib/paginator/paginator.ex +++ b/lib/paginator/paginator.ex @@ -11,6 +11,9 @@ defmodule AshJsonApiWrapper.Paginator do end end + @callback start(opts :: Keyword.t()) :: + {:ok, %{optional(:params) => map, optional(:headers) => map}} + @callback continue( response :: term, entities :: [Ash.Resource.record()], diff --git a/mix.exs b/mix.exs index 4c4eae7..018056d 100644 --- a/mix.exs +++ b/mix.exs @@ -134,7 +134,7 @@ defmodule AshJsonApiWrapper.MixProject do defp deps do [ {:ash, ash_version("~> 2.15")}, - {:tesla, "~> 1.7"}, + {:tesla, "~> 1.8"}, {:exjsonpath, "~> 0.1"}, # Dev/Test dependencies {:ex_doc, "~> 0.22", only: :dev, runtime: false}, @@ -145,7 +145,8 @@ defmodule AshJsonApiWrapper.MixProject do {:git_ops, "~> 2.5", only: :dev}, {:excoveralls, "~> 0.13.0", only: [:dev, :test]}, {:mix_test_watch, "~> 1.0", only: :dev, runtime: false}, - {:parse_trans, "3.3.0", only: [:dev, :test], override: true} + {:parse_trans, "3.3.0", only: [:dev, :test], override: true}, + {:mox, "~> 1.0", only: :test} ] end diff --git a/mix.lock b/mix.lock index 18b618e..f749fe4 100644 --- a/mix.lock +++ b/mix.lock @@ -29,6 +29,7 @@ "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mix_test_watch": {:hex, :mix_test_watch, "1.1.1", "eee6fc570d77ad6851c7bc08de420a47fd1e449ef5ccfa6a77ef68b72e7e51ad", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "f82262b54dee533467021723892e15c3267349849f1f737526523ecba4e6baae"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, @@ -39,7 +40,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "stream_data": {:hex, :stream_data, "0.6.0", "e87a9a79d7ec23d10ff83eb025141ef4915eeb09d4491f79e52f2562b73e5f47", [:mix], [], "hexpm", "b92b5031b650ca480ced047578f1d57ea6dd563f5b57464ad274718c9c29501c"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "tesla": {:hex, :tesla, "1.7.0", "a62dda2f80d4f8a925eb7b8c5b78c461e0eb996672719fe1a63b26321a5f8b4e", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "2e64f01ebfdb026209b47bc651a0e65203fcff4ae79c11efb73c4852b00dc313"}, + "tesla": {:hex, :tesla, "1.8.0", "d511a4f5c5e42538d97eef7c40ec4f3e44effdc5068206f42ed859e09e51d1fd", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.13", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, ">= 1.0.0", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.2", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:msgpax, "~> 2.3", [hex: :msgpax, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "10501f360cd926a309501287470372af1a6e1cbed0f43949203a4c13300bc79f"}, "typable": {:hex, :typable, "0.3.0", "0431e121d124cd26f312123e313d2689b9a5322b15add65d424c07779eaa3ca1", [:mix], [], "hexpm", "880a0797752da1a4c508ac48f94711e04c86156f498065a83d160eef945858f8"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } diff --git a/test/custom_pagination_test.exs b/test/custom_pagination_test.exs index 8c44512..a888273 100644 --- a/test/custom_pagination_test.exs +++ b/test/custom_pagination_test.exs @@ -1,14 +1,36 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do use ExUnit.Case require Ash.Query + import Mox @moduletag :custom_pagination + # Make sure mocks are verified when the test exits + setup :verify_on_exit! + + defmodule TestingTesla do + use Tesla + + adapter(AshJsonApiWrapper.MockAdapter) + # plug(Tesla.Middleware.Logger) + + plug(Tesla.Middleware.Retry, + delay: 2000, + max_retries: 5, + max_delay: 4_000, + should_retry: fn + {:ok, %{status: status}} when status in [429] -> true + {:ok, _} -> false + {:error, _} -> true + end + ) + end + # ── Custom paginator ── defmodule CustomPaginator do use AshJsonApiWrapper.Paginator - def cursor do + defp cursor do case :ets.whereis(:cursor) do :undefined -> :ets.new(:cursor, [:set, :protected, :named_table]) @@ -22,22 +44,27 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do end end - def increment_cursor do + defp increment_cursor do :ets.insert(:cursor, {self(), cursor() + 1}) end - def reset_cursor do + defp reset_cursor do + cursor() :ets.insert(:cursor, {self(), 1}) end - def continue(_response, [], _) do + def start(_opts) do reset_cursor() + {:ok, %{params: %{"p" => 1}}} + end + + def continue(_response, [], _) do :halt end def continue(_response, _entities, _opts) do increment_cursor() - {:ok, %{params: %{:p => cursor()}}} + {:ok, %{params: %{"p" => cursor()}}} end end @@ -49,7 +76,7 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do validate_api_inclusion?: false json_api_wrapper do - tesla(Tesla) + tesla(TestingTesla) endpoints do base("https://65383945a543859d1bb1528e.mockapi.io/api/v1") @@ -99,13 +126,50 @@ defmodule AshJsonApiWrapper.CustomPagination.Test do Application.put_env(:ash, :validate_api_resource_inclusion?, false) Application.put_env(:ash, :validate_api_config_inclusion?, false) + AshJsonApiWrapper.MockAdapter + |> expect(:call, 4, fn env, _options -> + case env.query do + %{"l" => 3, "p" => 2} -> + {:ok, %Tesla.Env{env | status: 200, body: "[]"}} + + %{"l" => 3, "p" => 1} -> + {:ok, + %Tesla.Env{ + env + | status: 200, + body: """ + [ + {"name": "Kendra Ernser", "id":"1"}, + {"name": "Max Hartman", "id":"2"}, + {"name": "John Benton", "id":"3"} + ] + """ + }} + + query -> + {:ok, + %Tesla.Env{ + env + | status: 500, + body: "Unexpected parameters: #{query |> Kernel.inspect()}" + }} + end + end) + users = Users |> Ash.Query.for_read(:list_users) - |> Api.read!(page: [limit: 99]) + # |> Ash.Query.limit(2) + |> Api.read!(page: [limit: 2, offset: 0]) + + users2 = + Users + |> Ash.Query.for_read(:list_users) + |> Api.read!(page: [limit: 2, offset: 1]) - user_count = users.results |> Enum.count() + users_count = users.results |> Enum.count() + users2_count = users2.results |> Enum.count() - assert(user_count == 99) + assert(users_count == users2_count) end end diff --git a/test/test_helper.exs b/test/test_helper.exs index e778860..bd174f4 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,3 @@ ExUnit.start() +Mox.defmock(AshJsonApiWrapper.MockAdapter, for: Tesla.Adapter) ExUnit.configure(exclude: [:hackernews])