From e02dde7ee954c6d6f9e75ce8a6fbe7573b706826 Mon Sep 17 00:00:00 2001 From: Maxim Filonov <53992153+sl1depengwyn@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:22:01 +0300 Subject: [PATCH] feat: Advanced Filters (#9769) * feat: Advanced Filters * Fix query performance * Fix timestamp filtering; Fix query construction * Add csv export * Frontend integration Add search_params to response Add limit to tokens endpoint Add fee in api response Add exclusion/inclusion of from/to addresses Remove raw_input from api response * Remove comment * Add methods search; Optimize internal txs query * Fix `method_id_to_name_from_params` * Fix filtering by amount; add filter by native * Fix review comments * Handle all token types * Optimize query * Process review comments * Process review comments --------- Co-authored-by: Viktor Baranov --- .../lib/block_scout_web/api_router.ex | 6 + .../api/v2/advanced_filter_controller.ex | 372 +++++++ .../controllers/api/v2/fallback_controller.ex | 7 + .../controllers/api/v2/token_controller.ex | 12 +- .../views/api/v2/advanced_filter_view.ex | 172 ++++ .../v2/advanced_filter_controller_test.exs | 945 ++++++++++++++++++ .../api/v2/validator_controller_test.exs | 21 +- .../lib/explorer/chain/advanced_filter.ex | 706 +++++++++++++ .../lib/explorer/chain/contract_method.ex | 49 +- .../address_transaction_csv_exporter.ex | 28 +- apps/explorer/lib/explorer/chain/token.ex | 8 + .../lib/explorer/chain/token_transfer.ex | 6 + .../lib/explorer/chain/transaction.ex | 16 +- apps/explorer/lib/explorer/helper.ex | 15 + .../lib/explorer/market/market_history.ex | 16 + cspell.json | 228 ++--- 16 files changed, 2421 insertions(+), 186 deletions(-) create mode 100644 apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex create mode 100644 apps/block_scout_web/lib/block_scout_web/views/api/v2/advanced_filter_view.ex create mode 100644 apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs create mode 100644 apps/explorer/lib/explorer/chain/advanced_filter.ex diff --git a/apps/block_scout_web/lib/block_scout_web/api_router.ex b/apps/block_scout_web/lib/block_scout_web/api_router.ex index 24e323db8c5b..e58618a34c3c 100644 --- a/apps/block_scout_web/lib/block_scout_web/api_router.ex +++ b/apps/block_scout_web/lib/block_scout_web/api_router.ex @@ -349,6 +349,12 @@ defmodule BlockScoutWeb.ApiRouter do get("/batches/:batch_number", V2.ArbitrumController, :batch) end end + + scope "/advanced-filters" do + get("/", V2.AdvancedFilterController, :list) + get("/csv", V2.AdvancedFilterController, :list_csv) + get("/methods", V2.AdvancedFilterController, :list_methods) + end end scope "/v1/graphql" do diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex new file mode 100644 index 000000000000..a3c2f332a26f --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/advanced_filter_controller.ex @@ -0,0 +1,372 @@ +defmodule BlockScoutWeb.API.V2.AdvancedFilterController do + use BlockScoutWeb, :controller + + import BlockScoutWeb.Chain, only: [default_paging_options: 0, split_list_by_page: 1, next_page_params: 4] + + alias BlockScoutWeb.API.V2.{AdvancedFilterView, CSVExportController, TransactionView} + alias Explorer.{Chain, PagingOptions} + alias Explorer.Chain.{AdvancedFilter, ContractMethod, Data, Token, Transaction} + alias Explorer.Chain.CSVExport.Helper, as: CSVHelper + alias Plug.Conn + + action_fallback(BlockScoutWeb.API.V2.FallbackController) + + @api_true [api?: true] + + @methods [ + %{method_id: "0xa9059cbb", name: "transfer"}, + %{method_id: "0xa0712d68", name: "mint"}, + %{method_id: "0x095ea7b3", name: "approve"}, + %{method_id: "0x40993b26", name: "buy"}, + %{method_id: "0x3593564c", name: "execute"}, + %{method_id: "0x3ccfd60b", name: "withdraw"}, + %{method_id: "0xd0e30db0", name: "deposit"}, + %{method_id: "0x0a19b14a", name: "trade"}, + %{method_id: "0x4420e486", name: "register"}, + %{method_id: "0x5f575529", name: "swap"}, + %{method_id: "0xd9627aa4", name: "sellToUniswap"}, + %{method_id: "0xe9e05c42", name: "depositTransaction"}, + %{method_id: "0x23b872dd", name: "transferFrom"}, + %{method_id: "0xa22cb465", name: "setApprovalForAll"}, + %{method_id: "0x2e7ba6ef", name: "claim"}, + %{method_id: "0x0502b1c5", name: "unoswap"}, + %{method_id: "0xb2267a7b", name: "sendMessage"}, + %{method_id: "0x9871efa4", name: "unxswapByOrderId"}, + %{method_id: "0xbf6eac2f", name: "stake"}, + %{method_id: "0x3ce33bff", name: "bridge"}, + %{method_id: "0xeb672419", name: "requestL2Transaction"}, + %{method_id: "0xe449022e", name: "uniswapV3Swap"}, + %{method_id: "0x0162e2d0", name: "swapETHForExactTokens"} + ] + + @methods_id_to_name_map Map.new(@methods, fn %{method_id: method_id, name: name} -> {method_id, name} end) + @methods_name_to_id_map Map.new(@methods, fn %{method_id: method_id, name: name} -> {name, method_id} end) + + @methods_filter_limit 20 + @tokens_filter_limit 20 + + @doc """ + Function responsible for `api/v2/advanced-filters/` endpoint. + """ + @spec list(Plug.Conn.t(), map()) :: Plug.Conn.t() + def list(conn, params) do + full_options = params |> extract_filters() |> Keyword.merge(paging_options(params)) |> Keyword.merge(@api_true) + + advanced_filters_plus_one = AdvancedFilter.list(full_options) + + {advanced_filters, next_page} = split_list_by_page(advanced_filters_plus_one) + + {decoded_transactions, _abi_acc, methods_acc} = + advanced_filters + |> Enum.map(fn af -> %Transaction{to_address: af.to_address, input: af.input, hash: af.hash} end) + |> TransactionView.decode_transactions(true) + + next_page_params = + next_page |> next_page_params(advanced_filters, Map.take(params, ["items_count"]), &paging_params/1) + + render(conn, :advanced_filters, + advanced_filters: advanced_filters, + decoded_transactions: decoded_transactions, + search_params: %{ + method_ids: method_id_to_name_from_params(full_options[:methods] || [], methods_acc), + tokens: contract_address_hash_to_token_from_params(full_options[:token_contract_address_hashes]) + }, + next_page_params: next_page_params + ) + end + + @doc """ + Function responsible for `api/v2/advanced-filters/csv` endpoint. + """ + @spec list_csv(Plug.Conn.t(), map()) :: Plug.Conn.t() + def list_csv(conn, params) do + with {:recaptcha, true} <- + {:recaptcha, + Application.get_env(:block_scout_web, :recaptcha)[:is_disabled] || + CSVHelper.captcha_helper().recaptcha_passed?(params["recaptcha_response"])} do + full_options = + params + |> extract_filters() + |> Keyword.merge(paging_options(params)) + |> Keyword.update(:paging_options, %PagingOptions{page_size: CSVHelper.limit()}, fn paging_options -> + %PagingOptions{paging_options | page_size: CSVHelper.limit()} + end) + + full_options + |> AdvancedFilter.list() + |> AdvancedFilterView.to_csv_format() + |> CSVHelper.dump_to_stream() + |> Enum.reduce_while(CSVExportController.put_resp_params(conn), fn chunk, conn -> + case Conn.chunk(conn, chunk) do + {:ok, conn} -> + {:cont, conn} + + {:error, :closed} -> + {:halt, conn} + end + end) + end + end + + @doc """ + Function responsible for `api/v2/advanced-filters/methods` endpoint, + including `api/v2/advanced-filters/methods/?q=:search_string`. + """ + @spec list_methods(Plug.Conn.t(), map()) :: {:method, nil | Explorer.Chain.ContractMethod.t()} | Plug.Conn.t() + def list_methods(conn, %{"q" => query}) do + case {@methods_id_to_name_map[query], @methods_name_to_id_map[query]} do + {name, _} when is_binary(name) -> + render(conn, :methods, methods: [%{method_id: query, name: name}]) + + {_, id} when is_binary(id) -> + render(conn, :methods, methods: [%{method_id: id, name: query}]) + + _ -> + mb_contract_method = + case Data.cast(query) do + {:ok, %Data{bytes: <<_::bytes-size(4)>> = binary_method_id}} -> + ContractMethod.find_contract_method_by_selector_id(binary_method_id, @api_true) + + _ -> + ContractMethod.find_contract_method_by_name(query, @api_true) + end + + with {:method, %ContractMethod{abi: %{"name" => name}, identifier: identifier}} <- {:method, mb_contract_method} do + render(conn, :methods, methods: [%{method_id: "0x" <> Base.encode16(identifier, case: :lower), name: name}]) + end + end + end + + def list_methods(conn, _params) do + render(conn, :methods, methods: @methods) + end + + defp method_id_to_name_from_params(prepared_method_ids, methods_acc) do + {decoded_method_ids, method_ids_to_find} = + Enum.reduce(prepared_method_ids, {%{}, []}, fn method_id, {decoded, to_decode} -> + {:ok, method_id_hash} = Data.cast(method_id) + + case {Map.get(@methods_id_to_name_map, method_id), + methods_acc + |> Map.get(method_id_hash.bytes, []) + |> Enum.find( + &match?(%ContractMethod{abi: %{"type" => "function", "name" => name}} when is_binary(name), &1) + )} do + {name, _} when is_binary(name) -> + {Map.put(decoded, method_id, name), to_decode} + + {_, %ContractMethod{abi: %{"type" => "function", "name" => name}}} when is_binary(name) -> + {Map.put(decoded, method_id, name), to_decode} + + {nil, nil} -> + {decoded, [method_id_hash.bytes | to_decode]} + end + end) + + method_ids_to_find + |> ContractMethod.find_contract_methods(@api_true) + |> Enum.reduce(%{}, fn contract_method, acc -> + case contract_method do + %ContractMethod{abi: %{"name" => name}, identifier: identifier} when is_binary(name) -> + Map.put(acc, "0x" <> Base.encode16(identifier, case: :lower), name) + + _ -> + acc + end + end) + |> Map.merge(decoded_method_ids) + end + + defp contract_address_hash_to_token_from_params(tokens) do + token_contract_address_hashes_to_include = tokens[:include] || [] + + token_contract_address_hashes_to_exclude = tokens[:exclude] || [] + + token_contract_address_hashes_to_include + |> Kernel.++(token_contract_address_hashes_to_exclude) + |> Enum.reject(&(&1 == "native")) + |> Enum.uniq() + |> Enum.take(@tokens_filter_limit) + |> Token.get_by_contract_address_hashes(@api_true) + |> Map.new(fn token -> {token.contract_address_hash, token} end) + end + + defp extract_filters(params) do + [ + tx_types: prepare_tx_types(params["tx_types"]), + methods: params["methods"] |> prepare_methods(), + age: prepare_age(params["age_from"], params["age_to"]), + from_address_hashes: + prepare_include_exclude_address_hashes( + params["from_address_hashes_to_include"], + params["from_address_hashes_to_exclude"], + &prepare_address_hash/1 + ), + to_address_hashes: + prepare_include_exclude_address_hashes( + params["to_address_hashes_to_include"], + params["to_address_hashes_to_exclude"], + &prepare_address_hash/1 + ), + address_relation: prepare_address_relation(params["address_relation"]), + amount: prepare_amount(params["amount_from"], params["amount_to"]), + token_contract_address_hashes: + params["token_contract_address_hashes_to_include"] + |> prepare_include_exclude_address_hashes( + params["token_contract_address_hashes_to_exclude"], + &prepare_token_address_hash/1 + ) + |> Enum.map(fn + {key, value} when is_list(value) -> {key, Enum.take(value, @tokens_filter_limit)} + key_value -> key_value + end) + ] + end + + @allowed_tx_types ~w(COIN_TRANSFER ERC-20 ERC-404 ERC-721 ERC-1155) + + defp prepare_tx_types(tx_types) when is_binary(tx_types) do + tx_types + |> String.upcase() + |> String.split(",") + |> Enum.filter(&(&1 in @allowed_tx_types)) + end + + defp prepare_tx_types(_), do: nil + + defp prepare_methods(methods) when is_binary(methods) do + methods + |> String.downcase() + |> String.split(",") + |> Enum.filter(fn + "0x" <> method_id when byte_size(method_id) == 8 -> + case Base.decode16(method_id, case: :mixed) do + {:ok, _} -> true + _ -> false + end + + _ -> + false + end) + |> Enum.uniq() + |> Enum.take(@methods_filter_limit) + end + + defp prepare_methods(_), do: nil + + defp prepare_age(from, to), do: [from: parse_date(from), to: parse_date(to)] + + defp parse_date(string_date) do + case string_date && DateTime.from_iso8601(string_date) do + {:ok, date, _utc_offset} -> date + _ -> nil + end + end + + defp prepare_address_hashes(address_hashes, map_filter_function) + when is_binary(address_hashes) do + address_hashes + |> String.split(",") + |> Enum.flat_map(&map_filter_function.(&1)) + end + + defp prepare_address_hashes(_, _), do: nil + + defp prepare_address_hash(maybe_address_hash) do + case Chain.string_to_address_hash(maybe_address_hash) do + {:ok, address_hash} -> [address_hash] + _ -> [] + end + end + + defp prepare_token_address_hash(token_address_hash) do + case String.downcase(token_address_hash) do + "native" -> ["native"] + _ -> prepare_address_hash(token_address_hash) + end + end + + defp prepare_address_relation(relation) do + case relation && String.downcase(relation) do + r when r in [nil, "or"] -> :or + "and" -> :and + _ -> nil + end + end + + defp prepare_amount(from, to), do: [from: parse_decimal(from), to: parse_decimal(to)] + + defp parse_decimal(string_decimal) do + case string_decimal && Decimal.parse(string_decimal) do + {decimal, ""} -> decimal + _ -> nil + end + end + + defp prepare_include_exclude_address_hashes(include, exclude, map_filter_function) do + [ + include: prepare_address_hashes(include, map_filter_function), + exclude: prepare_address_hashes(exclude, map_filter_function) + ] + end + + # Paging + + defp paging_options(%{ + "block_number" => block_number_string, + "transaction_index" => tx_index_string, + "internal_transaction_index" => internal_tx_index_string, + "token_transfer_index" => token_transfer_index_string, + "token_transfer_batch_index" => token_transfer_batch_index_string + }) do + with {block_number, ""} <- block_number_string && Integer.parse(block_number_string), + {tx_index, ""} <- tx_index_string && Integer.parse(tx_index_string), + {:ok, internal_tx_index} <- parse_nullable_integer_paging_parameter(internal_tx_index_string), + {:ok, token_transfer_index} <- parse_nullable_integer_paging_parameter(token_transfer_index_string), + {:ok, token_transfer_batch_index} <- parse_nullable_integer_paging_parameter(token_transfer_batch_index_string) do + [ + paging_options: %{ + default_paging_options() + | key: %{ + block_number: block_number, + transaction_index: tx_index, + internal_transaction_index: internal_tx_index, + token_transfer_index: token_transfer_index, + token_transfer_batch_index: token_transfer_batch_index + } + } + ] + else + _ -> [paging_options: default_paging_options()] + end + end + + defp paging_options(_), do: [paging_options: default_paging_options()] + + defp parse_nullable_integer_paging_parameter(""), do: {:ok, nil} + + defp parse_nullable_integer_paging_parameter(string) when is_binary(string) do + case Integer.parse(string) do + {integer, ""} -> {:ok, integer} + _ -> {:error, :invalid_paging_parameter} + end + end + + defp parse_nullable_integer_paging_parameter(_), do: {:error, :invalid_paging_parameter} + + defp paging_params(%AdvancedFilter{ + block_number: block_number, + transaction_index: tx_index, + internal_transaction_index: internal_tx_index, + token_transfer_index: token_transfer_index, + token_transfer_batch_index: token_transfer_batch_index + }) do + %{ + block_number: block_number, + transaction_index: tx_index, + internal_transaction_index: internal_tx_index, + token_transfer_index: token_transfer_index, + token_transfer_batch_index: token_transfer_batch_index + } + end +end diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex index 373704ccb03a..eb6a7447e10a 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/fallback_controller.ex @@ -278,6 +278,13 @@ defmodule BlockScoutWeb.API.V2.FallbackController do |> render(:message, %{message: @unverified_smart_contract}) end + def call(conn, {:method, _}) do + conn + |> put_status(:not_found) + |> put_view(ApiView) + |> render(:message, %{message: @not_found}) + end + def call(conn, {:is_empty_response, true}) do conn |> put_status(500) diff --git a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex index 53180c925798..fbe4bbf998cb 100644 --- a/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex +++ b/apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_controller.ex @@ -1,9 +1,10 @@ defmodule BlockScoutWeb.API.V2.TokenController do + alias Explorer.PagingOptions use BlockScoutWeb, :controller alias BlockScoutWeb.AccessHelper alias BlockScoutWeb.API.V2.{AddressView, TransactionView} - alias Explorer.{Chain, Repo} + alias Explorer.{Chain, Helper, Repo} alias Explorer.Chain.{Address, BridgedToken, Token, Token.Instance} alias Indexer.Fetcher.OnDemand.TokenTotalSupply, as: TokenTotalSupplyOnDemand @@ -14,7 +15,8 @@ defmodule BlockScoutWeb.API.V2.TokenController do next_page_params: 3, token_transfers_next_page_params: 3, unique_tokens_paging_options: 1, - unique_tokens_next_page: 3 + unique_tokens_next_page: 3, + default_paging_options: 0 ] import BlockScoutWeb.PagingHelper, @@ -300,6 +302,12 @@ defmodule BlockScoutWeb.API.V2.TokenController do options = params |> paging_options() + |> Keyword.update(:paging_options, default_paging_options(), fn %PagingOptions{ + page_size: page_size + } = paging_options -> + mb_parsed_limit = Helper.parse_integer(params["limit"]) + %PagingOptions{paging_options | page_size: min(page_size, mb_parsed_limit && abs(mb_parsed_limit))} + end) |> Keyword.merge(token_transfers_types_options(params)) |> Keyword.merge(tokens_sorting(params)) |> Keyword.merge(@api_true) diff --git a/apps/block_scout_web/lib/block_scout_web/views/api/v2/advanced_filter_view.ex b/apps/block_scout_web/lib/block_scout_web/views/api/v2/advanced_filter_view.ex new file mode 100644 index 000000000000..f54781551508 --- /dev/null +++ b/apps/block_scout_web/lib/block_scout_web/views/api/v2/advanced_filter_view.ex @@ -0,0 +1,172 @@ +defmodule BlockScoutWeb.API.V2.AdvancedFilterView do + use BlockScoutWeb, :view + + alias BlockScoutWeb.API.V2.{Helper, TokenView, TransactionView} + alias Explorer.Chain.{Address, Data, Transaction} + alias Explorer.Market + alias Explorer.Market.MarketHistory + + def render("advanced_filters.json", %{ + advanced_filters: advanced_filters, + decoded_transactions: decoded_transactions, + search_params: %{ + method_ids: method_ids, + tokens: tokens + }, + next_page_params: next_page_params + }) do + %{ + items: + advanced_filters + |> Enum.zip(decoded_transactions) + |> Enum.map(fn {af, decoded_input} -> prepare_advanced_filter(af, decoded_input) end), + search_params: prepare_search_params(method_ids, tokens), + next_page_params: next_page_params + } + end + + def render("methods.json", %{methods: methods}) do + methods + end + + def to_csv_format(advanced_filters) do + exchange_rate = Market.get_coin_exchange_rate() + + date_to_prices = + Enum.reduce(advanced_filters, %{}, fn af, acc -> + date = DateTime.to_date(af.timestamp) + + if Map.has_key?(acc, date) do + acc + else + market_history = MarketHistory.price_at_date(date) + + Map.put( + acc, + date, + {market_history && market_history.opening_price, market_history && market_history.closing_price} + ) + end + end) + + row_names = [ + "TxHash", + "Type", + "MethodId", + "UtcTimestamp", + "FromAddress", + "ToAddress", + "Value", + "TokenContractAddressHash", + "TokenDecimals", + "TokenSymbol", + "BlockNumber", + "Fee", + "CurrentPrice", + "TxDateOpeningPrice", + "TxDateClosingPrice" + ] + + af_lists = + advanced_filters + |> Stream.map(fn advanced_filter -> + method_id = + case advanced_filter.input do + %{bytes: <>} -> method_id + _ -> nil + end + + {opening_price, closing_price} = date_to_prices[DateTime.to_date(advanced_filter.timestamp)] + + [ + to_string(advanced_filter.hash), + advanced_filter.type, + method_id, + advanced_filter.timestamp, + Address.checksum(advanced_filter.from_address.hash), + Address.checksum(advanced_filter.to_address.hash), + advanced_filter.value, + if(advanced_filter.type != "coin_transfer", + do: advanced_filter.token_transfer.token.contract_address_hash, + else: nil + ), + if(advanced_filter.type != "coin_transfer", do: advanced_filter.token_transfer.token.decimals, else: nil), + if(advanced_filter.type != "coin_transfer", do: advanced_filter.token_transfer.token.symbol, else: nil), + advanced_filter.block_number, + advanced_filter.fee, + exchange_rate.usd_value, + opening_price, + closing_price + ] + end) + + Stream.concat([row_names], af_lists) + end + + defp prepare_advanced_filter(advanced_filter, decoded_input) do + %{ + hash: advanced_filter.hash, + type: advanced_filter.type, + method: + if(advanced_filter.type != "coin_transfer", + do: + TransactionView.method_name( + %Transaction{ + to_address: %Address{ + hash: advanced_filter.token_transfer.token.contract_address_hash, + contract_code: "0x" |> Data.cast() |> elem(1) + }, + input: advanced_filter.input + }, + decoded_input + ), + else: + TransactionView.method_name( + %Transaction{to_address: advanced_filter.to_address, input: advanced_filter.input}, + decoded_input + ) + ), + from: + Helper.address_with_info( + nil, + advanced_filter.from_address, + advanced_filter.from_address.hash, + false + ), + to: + Helper.address_with_info( + nil, + advanced_filter.to_address, + advanced_filter.to_address.hash, + false + ), + value: advanced_filter.value, + total: + if(advanced_filter.type != "coin_transfer", + do: TransactionView.prepare_token_transfer_total(advanced_filter.token_transfer), + else: nil + ), + token: + if(advanced_filter.type != "coin_transfer", + do: TokenView.render("token.json", %{token: advanced_filter.token_transfer.token}), + else: nil + ), + timestamp: advanced_filter.timestamp, + block_number: advanced_filter.block_number, + transaction_index: advanced_filter.transaction_index, + internal_transaction_index: advanced_filter.internal_transaction_index, + token_transfer_index: advanced_filter.token_transfer_index, + token_transfer_batch_index: advanced_filter.token_transfer_batch_index, + fee: advanced_filter.fee + } + end + + defp prepare_search_params(method_ids, tokens) do + tokens_map = + Map.new(tokens, fn {contract_address_hash, token} -> + {contract_address_hash, TokenView.render("token.json", %{token: token})} + end) + + %{methods: method_ids, tokens: tokens_map} + end +end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs new file mode 100644 index 000000000000..4e14bc95970f --- /dev/null +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/advanced_filter_controller_test.exs @@ -0,0 +1,945 @@ +defmodule BlockScoutWeb.API.V2.AdvancedFilterControllerTest do + use BlockScoutWeb.ConnCase + + import Mox + + alias Explorer.Chain.{AdvancedFilter, Data} + alias Explorer.{Factory, TestHelper} + + describe "/advanced_filters" do + test "empty list", %{conn: conn} do + request = get(conn, "/api/v2/advanced-filters") + assert response = json_response(request, 200) + assert response["items"] == [] + assert response["next_page_params"] == nil + end + + test "get and paginate advanced filter (transactions split between pages)", %{conn: conn} do + first_tx = :transaction |> insert() |> with_block() + insert_list(3, :token_transfer, transaction: first_tx) + + for i <- 0..2 do + insert(:internal_transaction, + transaction: first_tx, + block_hash: first_tx.block_hash, + index: i, + block_index: i + ) + end + + insert_list(51, :transaction) |> with_block() + + request = get(conn, "/api/v2/advanced-filters") + assert response = json_response(request, 200) + request_2nd_page = get(conn, "/api/v2/advanced-filters", response["next_page_params"]) + + assert response_2nd_page = json_response(request_2nd_page, 200) + check_paginated_response(AdvancedFilter.list(), response["items"], response_2nd_page["items"]) + end + + test "get and paginate advanced filter (token transfers split between pages)", %{conn: conn} do + first_tx = :transaction |> insert() |> with_block() + insert_list(3, :token_transfer, transaction: first_tx) + + for i <- 0..2 do + insert(:internal_transaction, + transaction: first_tx, + block_hash: first_tx.block_hash, + index: i, + block_index: i + ) + end + + second_tx = :transaction |> insert() |> with_block() + insert_list(50, :token_transfer, transaction: second_tx, block_number: second_tx.block_number) + + request = get(conn, "/api/v2/advanced-filters") + assert response = json_response(request, 200) + request_2nd_page = get(conn, "/api/v2/advanced-filters", response["next_page_params"]) + assert response_2nd_page = json_response(request_2nd_page, 200) + + check_paginated_response(AdvancedFilter.list(), response["items"], response_2nd_page["items"]) + end + + test "get and paginate advanced filter (batch token transfers split between pages)", %{conn: conn} do + first_tx = :transaction |> insert() |> with_block() + insert_list(3, :token_transfer, transaction: first_tx) + + for i <- 0..2 do + insert(:internal_transaction, + transaction: first_tx, + block_hash: first_tx.block_hash, + index: i, + block_index: i + ) + end + + second_tx = :transaction |> insert() |> with_block() + + insert_list(5, :token_transfer, + transaction: second_tx, + block_number: second_tx.block_number, + token_type: "ERC-1155", + token_ids: 0..10 |> Enum.to_list(), + amounts: 10..20 |> Enum.to_list() + ) + + request = get(conn, "/api/v2/advanced-filters") + assert response = json_response(request, 200) + request_2nd_page = get(conn, "/api/v2/advanced-filters", response["next_page_params"]) + assert response_2nd_page = json_response(request_2nd_page, 200) + + check_paginated_response(AdvancedFilter.list(), response["items"], response_2nd_page["items"]) + end + + test "get and paginate advanced filter (internal transactions split between pages)", %{conn: conn} do + first_tx = :transaction |> insert() |> with_block() + insert_list(3, :token_transfer, transaction: first_tx) + + for i <- 0..2 do + insert(:internal_transaction, + transaction: first_tx, + block_hash: first_tx.block_hash, + index: i, + block_index: i + ) + end + + second_tx = :transaction |> insert() |> with_block() + + for i <- 0..49 do + insert(:internal_transaction, + transaction: second_tx, + block_hash: second_tx.block_hash, + index: i, + block_index: i + ) + end + + request = get(conn, "/api/v2/advanced-filters") + assert response = json_response(request, 200) + request_2nd_page = get(conn, "/api/v2/advanced-filters", response["next_page_params"]) + + assert response_2nd_page = json_response(request_2nd_page, 200) + check_paginated_response(AdvancedFilter.list(), response["items"], response_2nd_page["items"]) + end + + test "filter by tx_type", %{conn: conn} do + 30 |> insert_list(:transaction) |> with_block() + + tx = insert(:transaction) |> with_block() + + for token_type <- ~w(ERC-20 ERC-404 ERC-721 ERC-1155), + _ <- 0..4 do + insert(:token_transfer, transaction: tx, token_type: token_type) + end + + tx = :transaction |> insert() |> with_block() + + for i <- 0..29 do + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + end + + for tx_type_filter_string <- + ~w(COIN_TRANSFER COIN_TRANSFER,ERC-404 ERC-721,ERC-1155 ERC-20,COIN_TRANSFER,ERC-1155) do + tx_type_filter = tx_type_filter_string |> String.split(",") + request = get(conn, "/api/v2/advanced-filters", %{"tx_types" => tx_type_filter_string}) + assert response = json_response(request, 200) + + assert Enum.all?(response["items"], fn item -> String.upcase(item["type"]) in tx_type_filter end) + + if response["next_page_params"] do + request_2nd_page = + get( + conn, + "/api/v2/advanced-filters", + Map.merge(%{"tx_types" => tx_type_filter_string}, response["next_page_params"]) + ) + + assert response_2nd_page = json_response(request_2nd_page, 200) + + assert Enum.all?(response_2nd_page["items"], fn item -> String.upcase(item["type"]) in tx_type_filter end) + + check_paginated_response( + AdvancedFilter.list(tx_types: tx_type_filter), + response["items"], + response_2nd_page["items"] + ) + end + end + end + + test "filter by methods", %{conn: conn} do + TestHelper.get_eip1967_implementation_zero_addresses() + + tx = :transaction |> insert() |> with_block() + + smart_contract = build(:smart_contract) + + contract_address = + insert(:address, + hash: address_hash(), + verified: true, + contract_code: Factory.contract_code_info().bytecode, + smart_contract: smart_contract + ) + + method_id1_string = "0xa9059cbb" + method_id2_string = "0xa0712d68" + method_id3_string = "0x095ea7b3" + method_id4_string = "0x40993b26" + + {:ok, method1} = Data.cast(method_id1_string <> "ab0ba0") + {:ok, method2} = Data.cast(method_id2_string <> "ab0ba0") + {:ok, method3} = Data.cast(method_id3_string <> "ab0ba0") + {:ok, method4} = Data.cast(method_id4_string <> "ab0ba0") + + for i <- 0..4 do + insert(:internal_transaction, + transaction: tx, + to_address_hash: contract_address.hash, + to_address: contract_address, + block_hash: tx.block_hash, + index: i, + block_index: i, + input: method1 + ) + end + + for i <- 5..9 do + insert(:internal_transaction, + transaction: tx, + to_address_hash: contract_address.hash, + to_address: contract_address, + block_hash: tx.block_hash, + index: i, + block_index: i, + input: method2 + ) + end + + 5 + |> insert_list(:transaction, to_address_hash: contract_address.hash, to_address: contract_address, input: method2) + |> with_block() + + 5 + |> insert_list(:transaction, to_address_hash: contract_address.hash, to_address: contract_address, input: method3) + |> with_block() + + method3_transaction = + :transaction + |> insert(to_address_hash: contract_address.hash, to_address: contract_address, input: method3) + |> with_block() + + method4_transaction = + :transaction + |> insert(to_address_hash: contract_address.hash, to_address: contract_address, input: method4) + |> with_block() + + 5 |> insert_list(:token_transfer, transaction: method3_transaction) + 5 |> insert_list(:token_transfer, transaction: method4_transaction) + + request = get(conn, "/api/v2/advanced-filters", %{"methods" => "0xa0712d68,0x095ea7b3"}) + assert response = json_response(request, 200) + + assert Enum.all?(response["items"], fn item -> + String.slice(item["method"], 0..9) in [method_id2_string, method_id3_string] + end) + + assert Enum.count(response["items"]) == 21 + end + + test "filter by age", %{conn: conn} do + first_timestamp = ~U[2023-12-12 00:00:00.000000Z] + + for i <- 0..4 do + tx = :transaction |> insert() |> with_block(block_timestamp: Timex.shift(first_timestamp, days: i)) + + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "age_from" => "2023-12-14T00:00:00Z", + "age_to" => "2023-12-16T00:00:00Z" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 9 + end + + test "filter by from address include", %{conn: conn} do + address = insert(:address) + + for i <- 0..4 do + tx = :transaction |> insert() |> with_block() + + if i < 2 do + :transaction |> insert(from_address_hash: address.hash, from_address: address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + from_address_hash: address.hash, + from_address: address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + from_address_hash: address.hash, + from_address: address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = get(conn, "/api/v2/advanced-filters", %{"from_address_hashes_to_include" => to_string(address.hash)}) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 6 + end + + test "filter by from address exclude", %{conn: conn} do + address = insert(:address) + + for i <- 0..4 do + tx = :transaction |> insert() |> with_block() + + if i < 4 do + :transaction |> insert(from_address_hash: address.hash, from_address: address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + from_address_hash: address.hash, + from_address: address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + from_address_hash: address.hash, + from_address: address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = get(conn, "/api/v2/advanced-filters", %{"from_address_hashes_to_exclude" => to_string(address.hash)}) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 7 + end + + test "filter by from address include and exclude", %{conn: conn} do + address_to_include = insert(:address) + address_to_exclude = insert(:address) + + for i <- 0..2 do + tx = + :transaction + |> insert(from_address_hash: address_to_exclude.hash, from_address: address_to_exclude) + |> with_block() + + if i < 4 do + :transaction + |> insert(from_address_hash: address_to_include.hash, from_address: address_to_include) + |> with_block() + + insert(:internal_transaction, + transaction: tx, + from_address_hash: address_to_include.hash, + from_address: address_to_include, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + from_address_hash: address_to_include.hash, + from_address: address_to_include, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "from_address_hashes_to_include" => to_string(address_to_include.hash), + "from_address_hashes_to_exclude" => to_string(address_to_exclude.hash) + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 9 + end + + test "filter by to address include", %{conn: conn} do + address = insert(:address) + + for i <- 0..4 do + tx = :transaction |> insert() |> with_block() + + if i < 2 do + :transaction |> insert(to_address_hash: address.hash, to_address: address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: address.hash, + to_address: address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: address.hash, + to_address: address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = get(conn, "/api/v2/advanced-filters", %{"to_address_hashes_to_include" => to_string(address.hash)}) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 6 + end + + test "filter by to address exclude", %{conn: conn} do + address = insert(:address) + + for i <- 0..4 do + tx = :transaction |> insert() |> with_block() + + if i < 4 do + :transaction |> insert(to_address_hash: address.hash, to_address: address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: address.hash, + to_address: address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: address.hash, + to_address: address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = get(conn, "/api/v2/advanced-filters", %{"to_address_hashes_to_exclude" => to_string(address.hash)}) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 7 + end + + test "filter by to address include and exclude", %{conn: conn} do + address_to_include = insert(:address) + address_to_exclude = insert(:address) + + for i <- 0..2 do + tx = + :transaction + |> insert(to_address_hash: address_to_exclude.hash, to_address: address_to_exclude) + |> with_block() + + if i < 4 do + :transaction + |> insert(to_address_hash: address_to_include.hash, to_address: address_to_include) + |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: address_to_include.hash, + to_address: address_to_include, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: address_to_include.hash, + to_address: address_to_include, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + else + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "to_address_hashes_to_include" => to_string(address_to_include.hash), + "to_address_hashes_to_exclude" => to_string(address_to_exclude.hash) + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 9 + end + + test "filter by from and to address", %{conn: conn} do + from_address = insert(:address) + to_address = insert(:address) + + for i <- 0..8 do + tx = :transaction |> insert() |> with_block() + + cond do + i < 2 -> + :transaction |> insert(from_address_hash: from_address.hash, from_address: from_address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + from_address_hash: from_address.hash, + from_address: from_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + from_address_hash: from_address.hash, + from_address: from_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + i < 4 -> + :transaction |> insert(to_address_hash: to_address.hash, to_address: to_address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: to_address.hash, + to_address: to_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: to_address.hash, + to_address: to_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + i < 6 -> + :transaction + |> insert( + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address + ) + |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + true -> + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "from_address_hashes_to_include" => to_string(from_address.hash), + "to_address_hashes_to_include" => to_string(to_address.hash), + "address_relation" => "AnD" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 6 + end + + test "filter by from or to address", %{conn: conn} do + from_address = insert(:address) + to_address = insert(:address) + + for i <- 0..8 do + tx = :transaction |> insert() |> with_block() + + cond do + i < 2 -> + :transaction |> insert(from_address_hash: from_address.hash, from_address: from_address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + from_address_hash: from_address.hash, + from_address: from_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + from_address_hash: from_address.hash, + from_address: from_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + i < 4 -> + :transaction |> insert(to_address_hash: to_address.hash, to_address: to_address) |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: to_address.hash, + to_address: to_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: to_address.hash, + to_address: to_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + i < 6 -> + :transaction + |> insert( + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address + ) + |> with_block() + + insert(:internal_transaction, + transaction: tx, + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, + to_address_hash: to_address.hash, + to_address: to_address, + from_address_hash: from_address.hash, + from_address: from_address, + transaction: tx, + block_number: tx.block_number, + log_index: i + ) + + true -> + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: i, + block_index: i + ) + + insert(:token_transfer, transaction: tx, block_number: tx.block_number, log_index: i) + end + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "from_address_hashes_to_include" => to_string(from_address.hash), + "to_address_hashes_to_include" => to_string(to_address.hash) + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 18 + end + + test "filter by amount", %{conn: conn} do + for i <- 0..4 do + tx = :transaction |> insert(value: i * 10 ** 18) |> with_block() + + insert(:internal_transaction, + transaction: tx, + block_hash: tx.block_hash, + index: 0, + block_index: 0, + value: i * 10 ** 18 + ) + + token = insert(:token, decimals: 10) + + insert(:token_transfer, + amount: i * 10 ** 10, + token_contract_address: token.contract_address, + transaction: tx, + block_number: tx.block_number, + log_index: 0 + ) + end + + request = get(conn, "/api/v2/advanced-filters", %{"amount_from" => "0.5", "amount_to" => "2.99"}) + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 6 + end + + test "filter by token contract address include", %{conn: conn} do + token_a = insert(:token) + token_b = insert(:token) + token_c = insert(:token) + + tx = :transaction |> insert() |> with_block() + + for token <- [token_a, token_b, token_c, token_a, token_b, token_c, token_a, token_b, token_c] do + insert(:token_transfer, + token_contract_address: token.contract_address, + transaction: tx, + block_number: tx.block_number, + log_index: 0 + ) + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "token_contract_address_hashes_to_include" => + "#{token_b.contract_address_hash},#{token_c.contract_address_hash}" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 6 + end + + test "filter by token contract address exclude", %{conn: conn} do + token_a = insert(:token) + token_b = insert(:token) + token_c = insert(:token) + + tx = :transaction |> insert() |> with_block() + + for token <- [token_a, token_b, token_c, token_a, token_b, token_c, token_a, token_b, token_c] do + insert(:token_transfer, + token_contract_address: token.contract_address, + transaction: tx, + block_number: tx.block_number, + log_index: 0 + ) + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "token_contract_address_hashes_to_exclude" => + "#{token_b.contract_address_hash},#{token_c.contract_address_hash}" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 4 + end + + test "filter by token contract address include with native", %{conn: conn} do + token_a = insert(:token) + token_b = insert(:token) + token_c = insert(:token) + + tx = :transaction |> insert() |> with_block() + + for token <- [token_a, token_b, token_c, token_a, token_b, token_c, token_a, token_b, token_c] do + insert(:token_transfer, + token_contract_address: token.contract_address, + transaction: tx, + block_number: tx.block_number, + log_index: 0 + ) + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "token_contract_address_hashes_to_include" => + "#{token_b.contract_address_hash},#{token_c.contract_address_hash},native" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 7 + end + + test "filter by token contract address exclude with native", %{conn: conn} do + token_a = insert(:token) + token_b = insert(:token) + token_c = insert(:token) + + tx = :transaction |> insert() |> with_block() + + for token <- [token_a, token_b, token_c, token_a, token_b, token_c, token_a, token_b, token_c] do + insert(:token_transfer, + token_contract_address: token.contract_address, + transaction: tx, + block_number: tx.block_number, + log_index: 0 + ) + end + + request = + get(conn, "/api/v2/advanced-filters", %{ + "token_contract_address_hashes_to_exclude" => + "#{token_b.contract_address_hash},#{token_c.contract_address_hash},native" + }) + + assert response = json_response(request, 200) + + assert Enum.count(response["items"]) == 3 + end + end + + describe "/advanced_filters/methods?q=" do + test "returns 404 if method does not exist", %{conn: conn} do + request = get(conn, "/api/v2/advanced-filters/methods", %{"q" => "foo"}) + assert response = json_response(request, 404) + assert response["message"] == "Not found" + end + + test "finds method by name", %{conn: conn} do + insert(:contract_method) + request = get(conn, "/api/v2/advanced-filters/methods", %{"q" => "set"}) + assert response = json_response(request, 200) + assert response == [%{"method_id" => "0x60fe47b1", "name" => "set"}] + end + + test "finds method by id", %{conn: conn} do + insert(:contract_method) + request = get(conn, "/api/v2/advanced-filters/methods", %{"q" => "0x60fe47b1"}) + assert response = json_response(request, 200) + assert response == [%{"method_id" => "0x60fe47b1", "name" => "set"}] + end + end + + defp check_paginated_response(all_advanced_filters, first_page, second_page) do + assert all_advanced_filters + |> Enum.map( + &{&1.block_number, &1.transaction_index, &1.internal_transaction_index, &1.token_transfer_index, + &1.token_transfer_batch_index} + ) == + Enum.map( + first_page ++ second_page, + &{&1["block_number"], &1["transaction_index"], &1["internal_transaction_index"], + &1["token_transfer_index"], &1["token_transfer_batch_index"]} + ) + end +end diff --git a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/validator_controller_test.exs b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/validator_controller_test.exs index ac3f66320b1e..8b38069ffbe1 100644 --- a/apps/block_scout_web/test/block_scout_web/controllers/api/v2/validator_controller_test.exs +++ b/apps/block_scout_web/test/block_scout_web/controllers/api/v2/validator_controller_test.exs @@ -5,6 +5,7 @@ defmodule BlockScoutWeb.API.V2.ValidatorControllerTest do alias Explorer.Chain.Address alias Explorer.Chain.Cache.StabilityValidatorsCounters alias Explorer.Chain.Stability.Validator, as: ValidatorStability + alias Explorer.Helper defp check_paginated_response(first_page_resp, second_page_resp, list) do assert Enum.count(first_page_resp["items"]) == 50 @@ -19,12 +20,12 @@ defmodule BlockScoutWeb.API.V2.ValidatorControllerTest do defp compare_default_sorting_for_asc({validator_1, blocks_count_1}, {validator_2, blocks_count_2}) do case { - compare(blocks_count_1, blocks_count_2), - compare( + Helper.compare(blocks_count_1, blocks_count_2), + Helper.compare( Keyword.fetch!(ValidatorStability.state_enum(), validator_1.state), Keyword.fetch!(ValidatorStability.state_enum(), validator_2.state) ), - compare(validator_1.address_hash.bytes, validator_2.address_hash.bytes) + Helper.compare(validator_1.address_hash.bytes, validator_2.address_hash.bytes) } do {:lt, _, _} -> false {:eq, :lt, _} -> false @@ -35,12 +36,12 @@ defmodule BlockScoutWeb.API.V2.ValidatorControllerTest do defp compare_default_sorting_for_desc({validator_1, blocks_count_1}, {validator_2, blocks_count_2}) do case { - compare(blocks_count_1, blocks_count_2), - compare( + Helper.compare(blocks_count_1, blocks_count_2), + Helper.compare( Keyword.fetch!(ValidatorStability.state_enum(), validator_1.state), Keyword.fetch!(ValidatorStability.state_enum(), validator_2.state) ), - compare(validator_1.address_hash.bytes, validator_2.address_hash.bytes) + Helper.compare(validator_1.address_hash.bytes, validator_2.address_hash.bytes) } do {:gt, _, _} -> false {:eq, :lt, _} -> false @@ -59,14 +60,6 @@ defmodule BlockScoutWeb.API.V2.ValidatorControllerTest do assert compare_item(validator, json) end - defp compare(a, b) do - cond do - a < b -> :lt - a > b -> :gt - true -> :eq - end - end - describe "/validators/stability" do test "get paginated list of the validators", %{conn: conn} do validators = diff --git a/apps/explorer/lib/explorer/chain/advanced_filter.ex b/apps/explorer/lib/explorer/chain/advanced_filter.ex new file mode 100644 index 000000000000..6ad943fc2616 --- /dev/null +++ b/apps/explorer/lib/explorer/chain/advanced_filter.ex @@ -0,0 +1,706 @@ +defmodule Explorer.Chain.AdvancedFilter do + @moduledoc """ + Models an advanced filter. + """ + + use Explorer.Schema + + import Ecto.Query + + alias Explorer.{Chain, Helper, PagingOptions} + alias Explorer.Chain.{Address, Data, Hash, InternalTransaction, TokenTransfer, Transaction} + + @primary_key false + typed_embedded_schema null: false do + field(:hash, Hash.Full) + field(:type, :string) + field(:input, Data) + field(:timestamp, :utc_datetime_usec) + + belongs_to( + :from_address, + Address, + foreign_key: :from_address_hash, + references: :hash, + type: Hash.Address + ) + + belongs_to( + :to_address, + Address, + foreign_key: :to_address_hash, + references: :hash, + type: Hash.Address + ) + + field(:value, :decimal, null: true) + + has_one(:token_transfer, TokenTransfer, foreign_key: :transaction_hash, references: :hash, null: true) + + field(:fee, :decimal) + + field(:block_number, :integer) + field(:transaction_index, :integer) + field(:internal_transaction_index, :integer, null: true) + field(:token_transfer_index, :integer, null: true) + field(:token_transfer_batch_index, :integer, null: true) + end + + @typep tx_types :: {:tx_types, [String.t()] | nil} + @typep methods :: {:methods, [String.t()] | nil} + @typep age :: {:age, [{:from, DateTime.t() | nil} | {:to, DateTime.t() | nil}] | nil} + @typep from_address_hashes :: {:from_address_hashes, [Hash.Address.t()] | nil} + @typep to_address_hashes :: {:to_address_hashes, [Hash.Address.t()] | nil} + @typep address_relation :: {:address_relation, :or | :and | nil} + @typep amount :: {:amount, [{:from, Decimal.t()} | {:to, Decimal.t()}] | nil} + @typep token_contract_address_hashes :: + {:token_contract_address_hashes, [{:include, [Hash.Address.t()]} | {:include, [Hash.Address.t()]}] | nil} + @type options :: [ + tx_types() + | methods() + | age() + | from_address_hashes() + | to_address_hashes() + | address_relation() + | amount() + | token_contract_address_hashes() + | Chain.paging_options() + | Chain.api?() + ] + + @spec list(options()) :: [__MODULE__.t()] + def list(options \\ []) do + paging_options = Keyword.get(options, :paging_options) + + tasks = + options + |> queries(paging_options) + |> Enum.map(fn query -> Task.async(fn -> Chain.select_repo(options).all(query) end) end) + + tasks + |> Task.yield_many(:timer.seconds(60)) + |> Enum.flat_map(fn {_task, res} -> + case res do + {:ok, result} -> + result + + {:exit, reason} -> + raise "Query fetching advanced filters terminated: #{inspect(reason)}" + + nil -> + raise "Query fetching advanced filters timed out." + end + end) + |> Enum.map(&to_advanced_filter/1) + |> Enum.sort(&sort_function/2) + |> take_page_size(paging_options) + end + + defp queries(options, paging_options) do + cond do + only_transactions?(options) -> + [transactions_query(paging_options, options), internal_transactions_query(paging_options, options)] + + only_token_transfers?(options) -> + [token_transfers_query(paging_options, options)] + + true -> + [ + transactions_query(paging_options, options), + internal_transactions_query(paging_options, options), + token_transfers_query(paging_options, options) + ] + end + end + + defp only_transactions?(options) do + transaction_types = options[:tx_types] + tokens_to_include = options[:token_contract_address_hashes][:include] + + transaction_types == ["COIN_TRANSFER"] or tokens_to_include == ["native"] + end + + defp only_token_transfers?(options) do + transaction_types = options[:tx_types] + tokens_to_include = options[:token_contract_address_hashes][:include] + tokens_to_exclude = options[:token_contract_address_hashes][:exclude] + + (is_list(transaction_types) and length(transaction_types) > 0 and "COIN_TRANSFER" not in transaction_types) or + (is_list(tokens_to_include) and length(tokens_to_include) > 0 and "native" not in tokens_to_include) or + (is_list(tokens_to_exclude) and "native" in tokens_to_exclude) + end + + defp to_advanced_filter(%Transaction{} = transaction) do + %__MODULE__{ + hash: transaction.hash, + type: "coin_transfer", + input: transaction.input, + timestamp: transaction.block_timestamp, + from_address: transaction.from_address, + to_address: transaction.to_address, + value: transaction.value.value, + fee: transaction |> Transaction.fee(:wei) |> elem(1), + block_number: transaction.block_number, + transaction_index: transaction.index + } + end + + defp to_advanced_filter(%InternalTransaction{} = internal_transaction) do + %__MODULE__{ + hash: internal_transaction.transaction.hash, + type: "coin_transfer", + input: internal_transaction.input, + timestamp: internal_transaction.transaction.block_timestamp, + from_address: internal_transaction.from_address, + to_address: internal_transaction.to_address, + value: internal_transaction.value.value, + fee: + internal_transaction.transaction.gas_price && internal_transaction.gas_used && + Decimal.mult(internal_transaction.transaction.gas_price.value, internal_transaction.gas_used), + block_number: internal_transaction.transaction.block_number, + transaction_index: internal_transaction.transaction.index, + internal_transaction_index: internal_transaction.index + } + end + + defp to_advanced_filter(%TokenTransfer{} = token_transfer) do + %__MODULE__{ + hash: token_transfer.transaction.hash, + type: token_transfer.token_type, + input: token_transfer.transaction.input, + timestamp: token_transfer.transaction.block_timestamp, + from_address: token_transfer.from_address, + to_address: token_transfer.to_address, + fee: token_transfer.transaction |> Transaction.fee(:wei) |> elem(1), + token_transfer: %TokenTransfer{ + token_transfer + | amounts: [token_transfer.amount], + token_ids: token_transfer.token_id && [token_transfer.token_id] + }, + block_number: token_transfer.block_number, + transaction_index: token_transfer.transaction.index, + token_transfer_index: token_transfer.log_index, + token_transfer_batch_index: token_transfer.reverse_index_in_batch + } + end + + defp sort_function(a, b) do + case { + Helper.compare(a.block_number, b.block_number), + Helper.compare(a.transaction_index, b.transaction_index), + Helper.compare(a.token_transfer_index, b.token_transfer_index), + Helper.compare(a.token_transfer_batch_index, b.token_transfer_batch_index), + Helper.compare(a.internal_transaction_index, b.internal_transaction_index) + } do + {:lt, _, _, _, _} -> + false + + {:eq, :lt, _, _, _} -> + false + + {:eq, :eq, _, _, _} -> + case {a.token_transfer_index, a.token_transfer_batch_index, a.internal_transaction_index, + b.token_transfer_index, b.token_transfer_batch_index, b.internal_transaction_index} do + {nil, _, nil, _, _, _} -> + true + + {a_tt_index, a_tt_batch_index, nil, b_tt_index, b_tt_batch_index, _} when not is_nil(b_tt_index) -> + {a_tt_index, a_tt_batch_index} > {b_tt_index, b_tt_batch_index} + + {nil, _, a_it_index, _, _, b_it_index} -> + a_it_index > b_it_index + + {_, _, _, _, _, _} -> + false + end + + _ -> + true + end + end + + defp take_page_size(list, %PagingOptions{page_size: page_size}) when is_integer(page_size) do + Enum.take(list, page_size) + end + + defp take_page_size(list, _), do: list + + defp transactions_query(paging_options, options) do + query = + from(transaction in Transaction, + as: :transaction, + preload: [ + :block, + from_address: [:names, :smart_contract, :proxy_implementations], + to_address: [:names, :smart_contract, :proxy_implementations] + ], + order_by: [ + desc: transaction.block_number, + desc: transaction.index + ] + ) + + query + |> page_transactions(paging_options) + |> limit_query(paging_options) + |> apply_transactions_filters(options) + end + + defp page_transactions(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: tx_index + } + }) do + dynamic_condition = + dynamic(^page_block_number_dynamic(:transaction, block_number) or ^page_tx_index_dynamic(block_number, tx_index)) + + query |> where(^dynamic_condition) + end + + defp page_transactions(query, _), do: query + + defp internal_transactions_query(paging_options, options) do + query = + from(internal_transaction in InternalTransaction, + as: :internal_transaction, + join: transaction in assoc(internal_transaction, :transaction), + as: :transaction, + preload: [ + from_address: [:names, :smart_contract, :proxy_implementations], + to_address: [:names, :smart_contract, :proxy_implementations], + transaction: transaction + ], + order_by: [ + desc: transaction.block_number, + desc: transaction.index, + desc: internal_transaction.index + ] + ) + + query + |> page_internal_transactions(paging_options) + |> limit_query(paging_options) + |> apply_transactions_filters(options) + end + + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: tx_index, + internal_transaction_index: nil + } + }) do + case {block_number, tx_index} do + {0, 0} -> + query |> where(as(:transaction).block_number == ^block_number and as(:transaction).index == ^tx_index) + + {0, tx_index} -> + query + |> where(as(:transaction).block_number == ^block_number and as(:transaction).index <= ^tx_index) + + {block_number, 0} -> + query |> where(as(:transaction).block_number < ^block_number) + + _ -> + query + |> where( + as(:transaction).block_number < ^block_number or + (as(:transaction).block_number == ^block_number and as(:transaction).index <= ^tx_index) + ) + end + end + + defp page_internal_transactions(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: tx_index, + internal_transaction_index: it_index + } + }) do + dynamic_condition = + dynamic( + ^page_block_number_dynamic(:transaction, block_number) or ^page_tx_index_dynamic(block_number, tx_index) or + ^page_it_index_dynamic(block_number, tx_index, it_index) + ) + + query + |> where(^dynamic_condition) + end + + defp page_internal_transactions(query, _), do: query + + defp token_transfers_query(paging_options, options) do + token_transfer_query = + from(token_transfer in TokenTransfer, + as: :token_transfer, + join: transaction in assoc(token_transfer, :transaction), + as: :transaction, + join: token in assoc(token_transfer, :token), + as: :token, + select: %TokenTransfer{ + token_transfer + | token_id: fragment("UNNEST(?)", token_transfer.token_ids), + amount: + fragment("UNNEST(COALESCE(?, ARRAY[COALESCE(?, 1)]))", token_transfer.amounts, token_transfer.amount), + reverse_index_in_batch: + fragment("GENERATE_SERIES(COALESCE(ARRAY_LENGTH(?, 1), 1), 1, -1)", token_transfer.amounts), + token_decimals: token.decimals + }, + order_by: [ + desc: token_transfer.block_number, + desc: token_transfer.log_index + ] + ) + + token_transfer_query + |> apply_token_transfers_filters(options) + |> page_token_transfers(paging_options) + |> filter_token_transfers_by_amount(options[:amount][:from], options[:amount][:to]) + |> make_token_transfer_query_unnested() + |> limit_query(paging_options) + end + + defp page_token_transfers(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: tx_index, + token_transfer_index: nil, + internal_transaction_index: nil + } + }) do + case {block_number, tx_index} do + {0, 0} -> + query |> where(as(:transaction).block_number == ^block_number and as(:transaction).index == ^tx_index) + + {0, tx_index} -> + query + |> where([token_transfer], token_transfer.block_number == ^block_number and as(:transaction).index < ^tx_index) + + {block_number, 0} -> + query |> where([token_transfer], token_transfer.block_number < ^block_number) + + {block_number, tx_index} -> + query + |> where( + [token_transfer], + token_transfer.block_number < ^block_number or + (token_transfer.block_number == ^block_number and as(:transaction).index <= ^tx_index) + ) + end + end + + defp page_token_transfers(query, %PagingOptions{ + key: %{ + block_number: block_number, + transaction_index: tx_index, + token_transfer_index: nil + } + }) do + dynamic_condition = + dynamic( + ^page_block_number_dynamic(:token_transfer, block_number) or ^page_tx_index_dynamic(block_number, tx_index) + ) + + query |> where(^dynamic_condition) + end + + defp page_token_transfers(query, %PagingOptions{ + key: %{ + block_number: block_number, + token_transfer_index: tt_index, + token_transfer_batch_index: tt_batch_index + } + }) do + dynamic_condition = + dynamic( + ^page_block_number_dynamic(:token_transfer, block_number) or + ^page_tt_index_dynamic(:token_transfer, block_number, tt_index, tt_batch_index) + ) + + paged_query = query |> where(^dynamic_condition) + + paged_query + |> make_token_transfer_query_unnested() + |> where( + ^page_tt_batch_index_dynamic( + block_number, + tt_index, + tt_batch_index + ) + ) + end + + defp page_token_transfers(query, _), do: query + + defp page_block_number_dynamic(binding, block_number) when block_number > 0 do + dynamic(as(^binding).block_number < ^block_number) + end + + defp page_block_number_dynamic(_, _) do + dynamic(false) + end + + defp page_tx_index_dynamic(block_number, tx_index) when tx_index > 0 do + dynamic([transaction: tx], tx.block_number == ^block_number and tx.index < ^tx_index) + end + + defp page_tx_index_dynamic(_, _) do + dynamic(false) + end + + defp page_it_index_dynamic(block_number, tx_index, it_index) when it_index > 0 do + dynamic( + [transaction: tx, internal_transaction: it], + tx.block_number == ^block_number and tx.index == ^tx_index and + it.index < ^it_index + ) + end + + defp page_it_index_dynamic(_, _, _) do + dynamic(false) + end + + defp page_tt_index_dynamic(binding, block_number, tt_index, tt_batch_index) + when tt_index > 0 and tt_batch_index > 1 do + dynamic(as(^binding).block_number == ^block_number and as(^binding).log_index <= ^tt_index) + end + + defp page_tt_index_dynamic(binding, block_number, tt_index, _tt_batch_index) when tt_index > 0 do + dynamic(as(^binding).block_number == ^block_number and as(^binding).log_index < ^tt_index) + end + + defp page_tt_index_dynamic(_, _, _, _) do + dynamic(false) + end + + defp page_tt_batch_index_dynamic(block_number, tt_index, tt_batch_index) when tt_batch_index > 1 do + dynamic( + [unnested_token_transfer: tt], + ^page_block_number_dynamic(:unnested_token_transfer, block_number) or + ^page_tt_index_dynamic( + :unnested_token_transfer, + block_number, + tt_index, + 0 + ) or + (tt.block_number == ^block_number and tt.log_index == ^tt_index and tt.reverse_index_in_batch < ^tt_batch_index) + ) + end + + defp page_tt_batch_index_dynamic(_, _, _) do + dynamic(true) + end + + defp limit_query(query, %PagingOptions{page_size: limit}) when is_integer(limit), do: limit(query, ^limit) + + defp limit_query(query, _), do: query + + defp apply_token_transfers_filters(query, options) do + query + |> filter_by_tx_type(options[:tx_types]) + |> filter_token_transfers_by_methods(options[:methods]) + |> filter_by_token(options[:token_contract_address_hashes][:include], :include) + |> filter_by_token(options[:token_contract_address_hashes][:exclude], :exclude) + |> apply_common_filters(options) + end + + defp apply_transactions_filters(query, options) do + query + |> filter_transactions_by_amount(options[:amount][:from], options[:amount][:to]) + |> filter_transactions_by_methods(options[:methods]) + |> apply_common_filters(options) + end + + defp apply_common_filters(query, options) do + query + |> only_collated_transactions() + |> filter_by_timestamp(options[:age][:from], options[:age][:to]) + |> filter_by_addresses(options[:from_address_hashes], options[:to_address_hashes], options[:address_relation]) + end + + defp only_collated_transactions(query) do + query |> where(not is_nil(as(:transaction).block_number) and not is_nil(as(:transaction).index)) + end + + defp filter_by_tx_type(query, [_ | _] = tx_types) do + query |> where([token_transfer], token_transfer.token_type in ^tx_types) + end + + defp filter_by_tx_type(query, _), do: query + + defp filter_transactions_by_methods(query, [_ | _] = methods) do + prepared_methods = prepare_methods(methods) + + query |> where([t], fragment("substring(? FOR 4)", t.input) in ^prepared_methods) + end + + defp filter_transactions_by_methods(query, _), do: query + + defp filter_token_transfers_by_methods(query, [_ | _] = methods) do + prepared_methods = prepare_methods(methods) + + query |> where(fragment("substring(? FOR 4)", as(:transaction).input) in ^prepared_methods) + end + + defp filter_token_transfers_by_methods(query, _), do: query + + defp prepare_methods(methods) do + methods + |> Enum.flat_map(fn + method -> + case Data.cast(method) do + {:ok, method} -> [method.bytes] + _ -> [] + end + end) + end + + defp filter_by_timestamp(query, %DateTime{} = from, %DateTime{} = to) do + query |> where(as(:transaction).block_timestamp >= ^from and as(:transaction).block_timestamp <= ^to) + end + + defp filter_by_timestamp(query, %DateTime{} = from, _to) do + query |> where(as(:transaction).block_timestamp >= ^from) + end + + defp filter_by_timestamp(query, _from, %DateTime{} = to) do + query |> where(as(:transaction).block_timestamp <= ^to) + end + + defp filter_by_timestamp(query, _, _), do: query + + defp filter_by_addresses(query, from_addresses, to_addresses, relation) do + to_address_dynamic = do_filter_by_addresses(:to_address_hash, to_addresses) + + from_address_dynamic = do_filter_by_addresses(:from_address_hash, from_addresses) + + final_condition = + case {to_address_dynamic, from_address_dynamic} do + {not_nil_to_address, not_nil_from_address} when nil not in [not_nil_to_address, not_nil_from_address] -> + combine_filter_by_addresses(not_nil_to_address, not_nil_from_address, relation) + + _ -> + to_address_dynamic || from_address_dynamic + end + + case final_condition do + not_nil when not is_nil(not_nil) -> query |> where(^not_nil) + _ -> query + end + end + + defp do_filter_by_addresses(field, addresses) do + to_include_dynamic = do_filter_by_addresses_inclusion(field, addresses && Keyword.get(addresses, :include)) + to_exclude_dynamic = do_filter_by_addresses_exclusion(field, addresses && Keyword.get(addresses, :exclude)) + + case {to_include_dynamic, to_exclude_dynamic} do + {not_nil_include, not_nil_exclude} when nil not in [not_nil_include, not_nil_exclude] -> + dynamic([t], ^not_nil_include and ^not_nil_exclude) + + _ -> + to_include_dynamic || to_exclude_dynamic + end + end + + defp do_filter_by_addresses_inclusion(field, [_ | _] = addresses) do + dynamic([t], field(t, ^field) in ^addresses) + end + + defp do_filter_by_addresses_inclusion(_, _), do: nil + + defp do_filter_by_addresses_exclusion(field, [_ | _] = addresses) do + dynamic([t], field(t, ^field) not in ^addresses) + end + + defp do_filter_by_addresses_exclusion(_, _), do: nil + + defp combine_filter_by_addresses(from_addresses_dynamic, to_addresses_dynamic, :or) do + dynamic([t], ^from_addresses_dynamic or ^to_addresses_dynamic) + end + + defp combine_filter_by_addresses(from_addresses_dynamic, to_addresses_dynamic, _) do + dynamic([t], ^from_addresses_dynamic and ^to_addresses_dynamic) + end + + @eth_decimals 1000_000_000_000_000_000 + + defp filter_transactions_by_amount(query, from, to) when not is_nil(from) and not is_nil(to) and from < to do + query |> where([t], t.value / @eth_decimals >= ^from and t.value / @eth_decimals <= ^to) + end + + defp filter_transactions_by_amount(query, _from, to) when not is_nil(to) do + query |> where([t], t.value / @eth_decimals <= ^to) + end + + defp filter_transactions_by_amount(query, from, _to) when not is_nil(from) do + query |> where([t], t.value / @eth_decimals >= ^from) + end + + defp filter_transactions_by_amount(query, _, _), do: query + + defp filter_token_transfers_by_amount(query, from, to) when not is_nil(from) and not is_nil(to) and from < to do + unnested_query = make_token_transfer_query_unnested(query) + + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from and + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + ) + end + + defp filter_token_transfers_by_amount(query, _from, to) when not is_nil(to) do + unnested_query = make_token_transfer_query_unnested(query) + + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) <= ^to + ) + end + + defp filter_token_transfers_by_amount(query, from, _to) when not is_nil(from) do + unnested_query = make_token_transfer_query_unnested(query) + + unnested_query + |> where( + [unnested_token_transfer: tt], + tt.amount / fragment("10 ^ COALESCE(?, 0)", tt.token_decimals) >= ^from + ) + end + + defp filter_token_transfers_by_amount(query, _, _), do: query + + defp make_token_transfer_query_unnested(query) do + if has_named_binding?(query, :unnested_token_transfer) do + query + else + from(token_transfer in subquery(query), + as: :unnested_token_transfer, + preload: [ + :transaction, + :token, + from_address: [:names, :smart_contract, :proxy_implementations], + to_address: [:names, :smart_contract, :proxy_implementations] + ], + select_merge: %{ + token_ids: [token_transfer.token_id], + amounts: [token_transfer.amount] + } + ) + end + end + + defp filter_by_token(query, [_ | _] = token_contract_address_hashes, :include) do + filtered = token_contract_address_hashes |> Enum.reject(&(&1 == "native")) + query |> where([token_transfer], token_transfer.token_contract_address_hash in ^filtered) + end + + defp filter_by_token(query, [_ | _] = token_contract_address_hashes, :exclude) do + filtered = token_contract_address_hashes |> Enum.reject(&(&1 == "native")) + query |> where([token_transfer], token_transfer.token_contract_address_hash not in ^filtered) + end + + defp filter_by_token(query, _, _), do: query +end diff --git a/apps/explorer/lib/explorer/chain/contract_method.ex b/apps/explorer/lib/explorer/chain/contract_method.ex index e23c7811f1aa..c4716cde0043 100644 --- a/apps/explorer/lib/explorer/chain/contract_method.ex +++ b/apps/explorer/lib/explorer/chain/contract_method.ex @@ -9,7 +9,7 @@ defmodule Explorer.Chain.ContractMethod do use Explorer.Schema alias Explorer.Chain.{Hash, MethodIdentifier, SmartContract} - alias Explorer.Repo + alias Explorer.{Chain, Repo} typed_schema "contract_methods" do field(:identifier, MethodIdentifier) @@ -65,7 +65,7 @@ defmodule Explorer.Chain.ContractMethod do end @doc """ - Finds limited number of contract methods by selector id + Query that finds limited number of contract methods by selector id """ @spec find_contract_method_query(binary(), integer()) :: Ecto.Query.t() def find_contract_method_query(method_id, limit) do @@ -76,6 +76,51 @@ defmodule Explorer.Chain.ContractMethod do ) end + @doc """ + Finds contract method by selector id + """ + @spec find_contract_method_by_selector_id(binary(), [Chain.api?()]) :: __MODULE__.t() | nil + def find_contract_method_by_selector_id(method_id, options) do + query = + from( + contract_method in __MODULE__, + where: contract_method.abi["type"] == "function", + where: contract_method.identifier == ^method_id, + limit: 1 + ) + + Chain.select_repo(options).one(query) + end + + @spec find_contract_method_by_name(String.t(), [Chain.api?()]) :: __MODULE__.t() | nil + def find_contract_method_by_name(name, options) do + query = + from( + contract_method in __MODULE__, + where: contract_method.abi["type"] == "function", + where: contract_method.abi["name"] == ^name, + limit: 1 + ) + + Chain.select_repo(options).one(query) + end + + @doc """ + Finds contract methods by selector id + """ + @spec find_contract_methods(binary(), [Chain.api?()]) :: [__MODULE__.t()] + def find_contract_methods(method_ids, options) do + query = + from( + contract_method in __MODULE__, + distinct: contract_method.identifier, + where: contract_method.abi["type"] == "function", + where: contract_method.identifier in ^method_ids + ) + + Chain.select_repo(options).all(query) + end + defp abi_element_to_contract_method(element) do case ABI.parse_specification([element], include_events?: true) do [selector] -> diff --git a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex index f7263c89c0e7..bfe1e92cfd0e 100644 --- a/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex +++ b/apps/explorer/lib/explorer/chain/csv_export/address_transaction_csv_exporter.ex @@ -3,12 +3,7 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do Exports transactions to a csv file. """ - import Ecto.Query, - only: [ - from: 2 - ] - - alias Explorer.{Market, PagingOptions, Repo} + alias Explorer.{Market, PagingOptions} alias Explorer.Market.MarketHistory alias Explorer.Chain.{Address, DenormalizationHelper, Hash, Transaction, Wei} alias Explorer.Chain.CSVExport.Helper @@ -67,7 +62,13 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do if Map.has_key?(acc, date) do acc else - Map.put(acc, date, price_at_date(date)) + market_history = MarketHistory.price_at_date(date) + + Map.put( + acc, + date, + {market_history && market_history.opening_price, market_history && market_history.closing_price} + ) end end) @@ -111,17 +112,4 @@ defmodule Explorer.Chain.CSVExport.AddressTransactionCsvExporter do {:maximum, value} -> "Max of #{value}" end end - - defp price_at_date(date) do - query = - from( - mh in MarketHistory, - where: mh.date == ^date - ) - - case Repo.one(query) do - nil -> {nil, nil} - price -> {price.opening_price, price.closing_price} - end - end end diff --git a/apps/explorer/lib/explorer/chain/token.ex b/apps/explorer/lib/explorer/chain/token.ex index d5d8838d06df..c89e07788142 100644 --- a/apps/explorer/lib/explorer/chain/token.ex +++ b/apps/explorer/lib/explorer/chain/token.ex @@ -238,6 +238,14 @@ defmodule Explorer.Chain.Token do Chain.select_repo(options).get_by(__MODULE__, contract_address_hash: hash) end + @doc """ + Gets tokens with given contract address hashes. + """ + @spec get_by_contract_address_hashes([Hash.Address.t()], [Chain.api?()]) :: [Token.t()] + def get_by_contract_address_hashes(hashes, options) do + Chain.select_repo(options).all(from(t in __MODULE__, where: t.contract_address_hash in ^hashes)) + end + @doc """ For usage in Indexer.Fetcher.TokenInstance.LegacySanitizeERC721 """ diff --git a/apps/explorer/lib/explorer/chain/token_transfer.ex b/apps/explorer/lib/explorer/chain/token_transfer.ex index 12edc67d89de..c86be9a8cc28 100644 --- a/apps/explorer/lib/explorer/chain/token_transfer.ex +++ b/apps/explorer/lib/explorer/chain/token_transfer.ex @@ -61,6 +61,9 @@ defmodule Explorer.Chain.TokenTransfer do * `:log_index` - Index of the corresponding `t:Explorer.Chain.Log.t/0` in the block. * `:amounts` - Tokens transferred amounts in case of batched transfer in ERC-1155 * `:token_ids` - IDs of the tokens (applicable to ERC-1155 tokens) + * `:token_id` - virtual field, ID of token, used to unnest ERC-1155 batch transfers + * `:index_in_batch` - Index of the token transfer in the ERC-1155 batch transfer + * `:reverse_index_in_batch` - Reverse index of the token transfer in the ERC-1155 batch transfer, last element index is 1 * `:block_consensus` - Consensus of the block that the transfer took place """ @primary_key false @@ -70,7 +73,10 @@ defmodule Explorer.Chain.TokenTransfer do field(:log_index, :integer, primary_key: true, null: false) field(:amounts, {:array, :decimal}) field(:token_ids, {:array, :decimal}) + field(:token_id, :decimal, virtual: true) field(:index_in_batch, :integer, virtual: true) + field(:reverse_index_in_batch, :integer, virtual: true) + field(:token_decimals, :decimal, virtual: true) field(:token_type, :string) field(:block_consensus, :boolean) diff --git a/apps/explorer/lib/explorer/chain/transaction.ex b/apps/explorer/lib/explorer/chain/transaction.ex index b943b9febd47..9c0e2c029906 100644 --- a/apps/explorer/lib/explorer/chain/transaction.ex +++ b/apps/explorer/lib/explorer/chain/transaction.ex @@ -263,7 +263,7 @@ defmodule Explorer.Chain.Transaction do alias ABI.FunctionSelector alias Ecto.Association.NotLoaded alias Ecto.Changeset - alias Explorer.{Chain, PagingOptions, Repo, SortingHelper} + alias Explorer.{Chain, Helper, PagingOptions, Repo, SortingHelper} alias Explorer.Chain.{ Block, @@ -1529,10 +1529,10 @@ defmodule Explorer.Chain.Transaction do defp compare_default_sorting(a, b) do case { - compare(a.block_number, b.block_number), - compare(a.index, b.index), + Helper.compare(a.block_number, b.block_number), + Helper.compare(a.index, b.index), DateTime.compare(a.inserted_at, b.inserted_at), - compare(Hash.to_integer(a.hash), Hash.to_integer(b.hash)) + Helper.compare(Hash.to_integer(a.hash), Hash.to_integer(b.hash)) } do {:lt, _, _, _} -> false {:eq, :lt, _, _} -> false @@ -1542,14 +1542,6 @@ defmodule Explorer.Chain.Transaction do end end - defp compare(a, b) do - cond do - a < b -> :lt - a > b -> :gt - true -> :eq - end - end - @doc """ Creates a query to fetch transactions taking into account paging_options (possibly nil), from_block (may be nil), to_block (may be nil) and boolean `with_pending?` that indicates if pending transactions should be included diff --git a/apps/explorer/lib/explorer/helper.ex b/apps/explorer/lib/explorer/helper.ex index 92d8652f40e3..bc09fc0bc422 100644 --- a/apps/explorer/lib/explorer/helper.ex +++ b/apps/explorer/lib/explorer/helper.ex @@ -166,4 +166,19 @@ defmodule Explorer.Helper do end def valid_url?(_), do: false + + @doc """ + Compare two values and returns either :lt, :eq or :gt. + + Please be careful: this function compares arguments using `<` and `>`, + hence it should not be used to compare structures (for instance %DateTime{} or %Decimal{}). + """ + @spec compare(term(), term()) :: :lt | :eq | :gt + def compare(a, b) do + cond do + a < b -> :lt + a > b -> :gt + true -> :eq + end + end end diff --git a/apps/explorer/lib/explorer/market/market_history.ex b/apps/explorer/lib/explorer/market/market_history.ex index d8ddc3ad5a33..bfcef2da37c3 100644 --- a/apps/explorer/lib/explorer/market/market_history.ex +++ b/apps/explorer/lib/explorer/market/market_history.ex @@ -5,6 +5,8 @@ defmodule Explorer.Market.MarketHistory do use Explorer.Schema + alias Explorer.Chain + @typedoc """ The recorded values of the configured coin to USD for a single day. @@ -22,4 +24,18 @@ defmodule Explorer.Market.MarketHistory do field(:tvl, :decimal) field(:secondary_coin, :boolean) end + + @doc """ + Returns the market history (for the secondary coin if specified) for the given date. + """ + @spec price_at_date(Date.t(), boolean(), [Chain.api?()]) :: t() | nil + def price_at_date(date, secondary_coin? \\ false, options \\ []) do + query = + from( + mh in __MODULE__, + where: mh.date == ^date and mh.secondary_coin == ^secondary_coin? + ) + + Chain.select_repo(options).one(query) + end end diff --git a/cspell.json b/cspell.json index aa1edcf326aa..c19fa922c259 100644 --- a/cspell.json +++ b/cspell.json @@ -9,108 +9,15 @@ "apps/block_scout_web/assets/js/lib/ace/src-min/*.js" ], "words": [ - "AION", - "AIRTABLE", - "ARGMAX", - "Aiubo", - "Asfpp", - "Asfpp", - "Autodetection", - "Autonity", - "Averify", - "bitmask", - "Blockchair", - "CALLCODE", - "CBOR", - "Celestia", - "Cldr", - "Consolas", - "Cyclomatic", - "DATETIME", - "DELEGATECALL", - "Decompiler", - "DefiLlama", - "DefiLlama", - "Denormalization", - "Denormalized", - "ECTO", - "EDCSA", - "Ebhwp", - "Encryptor", - "Erigon", - "Ethash", - "Faileddi", - "Filesize", - "Floki", - "Fuov", - "Hazkne", - "Hodl", - "Iframe", - "Iframes", - "Incrementer", - "Instrumenter", - "Karnaugh", - "Keepalive", - "LUKSO", - "Limegreen", - "MARKETCAP", - "Mobula", - "MDWW", - "Mainnets", - "Mendonça", - "Menlo", - "Merkle", - "Mixfile", - "NOTOK", - "Nerg", - "Nerg", - "Nethermind", - "Neue", - "Njhr", - "Nodealus", - "NovesFi", - "Numbe", - "Nunito", - "PGDATABASE", - "PGHOST", - "PGPASSWORD", - "PGPORT", - "PGUSER", - "POSDAO", - "Posix", - "Postrge", - "Qebz", - "Qmbgk", - "REINDEX", - "RPC's", - "RPCs", - "SENDGRID", - "SJONRPC", - "SOLIDITYSCAN", - "SOLIDITYSCAN", - "STATICCALL", - "Secon", - "Segoe", - "Sokol", - "Synthereum", - "Sérgio", - "Tcnwg", - "Testinit", - "Testit", - "Testname", - "Txns", - "UUPS", - "Unitarion", - "Unitorius", - "Unitorus", - "Utqn", - "Wanchain", "aave", "absname", "acbs", "accs", "actb", "addedfile", + "AION", + "AIRTABLE", + "Aiubo", "alloc", "amzootyukbugmx", "apikey", @@ -120,10 +27,14 @@ "ARGMAX", "arounds", "asda", + "Asfpp", "atoken", "autodetectfalse", + "Autodetection", "autodetecttrue", + "Autonity", "autoplay", + "Averify", "backoff", "badhash", "badnumber", @@ -137,11 +48,13 @@ "bignumber", "bigserial", "binwrite", + "bitmask", "bizbuz", + "Blockchair", "blockheight", "blockless", - "blocknum", "blockno", + "blocknum", "blockreward", "blockscout", "blockscoutuser", @@ -149,6 +62,7 @@ "bridgedtokenlist", "browserconfig", "bsdr", + "Btvk", "buildcache", "buildin", "buildx", @@ -159,10 +73,13 @@ "bzzr", "cacerts", "callcode", + "CALLCODE", "calltracer", "callvalue", "capturelog", "cattributes", + "CBOR", + "Celestia", "cellspacing", "certifi", "cfasync", @@ -175,6 +92,7 @@ "checkverifystatus", "childspec", "citext", + "Cldr", "clearfix", "clickover", "codeformat", @@ -193,6 +111,7 @@ "compilerversion", "concache", "cond", + "Consolas", "contractaddress", "contractaddresses", "contractname", @@ -205,23 +124,30 @@ "ctbs", "ctid", "cumalative", + "Cyclomatic", "cypherpunk", "czilladx", "datapoint", "datepicker", + "DATETIME", "deae", "decamelize", "decompiled", "decompiler", + "Decompiler", "dedup", + "DefiLlama", "defmock", "defsupervisor", "dejob", "dejobio", "delegatecall", + "DELEGATECALL", "delegators", "demonitor", "denormalization", + "Denormalization", + "Denormalized", "descr", "describedby", "differenceby", @@ -229,17 +155,22 @@ "dropzone", "dxgd", "dyntsrohg", + "Ebhwp", "econnrefused", + "ECTO", + "EDCSA", "edhygl", "efkuga", + "Encryptor", "endregion", "enetunreach", "enoent", "epns", + "Erigon", "errora", "errorb", "erts", - "erts", + "Ethash", "etherchain", "ethprice", "ethsupply", @@ -255,18 +186,20 @@ "extname", "extremums", "exvcr", + "Faileddi", "falala", + "feelin", "FEVM", "filecoin", "Filecoin", "Filesize", - "Filecoin", - "fkey", "fkey", + "Floki", "fontawesome", "fortawesome", "fsym", "fullwidth", + "Fuov", "fvdskvjglav", "fwrite", "fwupv", @@ -294,6 +227,7 @@ "gtag", "happygokitty", "haspopup", + "Hazkne", "histoday", "hljs", "Hodl", @@ -302,11 +236,15 @@ "hyperledger", "ifdef", "ifeq", + "Iframe", "iframes", + "Iframes", "ilike", "illustr", "inapp", + "Incrementer", "insertable", + "Instrumenter", "intersectionby", "ints", "invalidend", @@ -322,19 +260,22 @@ "johnnny", "jsons", "juon", + "Karnaugh", "keccak", + "Keepalive", "keyout", "kittencream", + "KnxbUejwY", "labeledby", "labelledby", "lastmod", - "lastmod", "lastname", "lastword", "lformat", "libraryaddress", "libraryname", "libsecp", + "Limegreen", "linecap", "linejoin", "listaccounts", @@ -342,27 +283,35 @@ "lkve", "llhauc", "loggable", + "LUKSO", "luxon", "mabi", + "Mainnets", "malihu", "mallowance", + "MARKETCAP", "maxlength", "mbot", "mcap", "mconst", "mdef", + "MDWW", "meer", - "meer", + "Mendonça", + "Menlo", "mergeable", + "Merkle", "metatags", "microsecs", "millis", "mintings", "mistmatches", "miterlimit", + "Mixfile", "mmem", "mname", "mnot", + "Mobula", "moxed", "moxing", "mpayable", @@ -382,12 +331,17 @@ "mykey", "nanomorph", "nbsp", + "Nerg", + "Nethermind", + "Neue", "newkey", "nftproduct", "ngettext", "nillifies", + "Njhr", "nlmyzui", "nocheck", + "Nodealus", "nohighlight", "nolink", "nonconsensus", @@ -397,9 +351,12 @@ "noreply", "NOTOK", "noves", + "NovesFi", "nowarn", "nowrap", "ntoa", + "Numbe", + "Nunito", "nxdomain", "offchain", "omni", @@ -415,7 +372,13 @@ "peekers", "pendingtxlist", "perc", + "permissionless", "persistable", + "PGDATABASE", + "PGHOST", + "PGPASSWORD", + "PGPORT", + "PGUSER", "phash", "pikaday", "pkey", @@ -428,6 +391,9 @@ "pocc", "polyline", "poolboy", + "POSDAO", + "Posix", + "Postrge", "prederive", "prederived", "progressbar", @@ -435,15 +401,15 @@ "psql", "purrstige", "qdai", + "Qebz", "qitmeer", - "qitmeer", + "Qmbgk", "qrcode", "queriable", "questiona", "questionb", "qwertyufhgkhiop", "qwertyuioiuytrewertyuioiuytrertyuio", - "qwertyuioiuytrewertyuioiuytrertyuio", "racecar", "raisedbrow", "rangeright", @@ -478,8 +444,9 @@ "RPCs", "safelow", "savechives", + "Secon", "secp", - "secp", + "Segoe", "seindexed", "selfdestruct", "selfdestructed", @@ -491,10 +458,13 @@ "shibarium", "shortdoc", "shortify", + "SJONRPC", "smallint", "smth", "snapshotted", "snapshotting", + "Sokol", + "SOLIDITYSCAN", "soljson", "someout", "sourcecode", @@ -505,6 +475,7 @@ "stakers", "stateroot", "staticcall", + "STATICCALL", "strftime", "strhash", "stringly", @@ -525,13 +496,18 @@ "sushiswap", "swal", "sweetalert", + "Synthereum", "tabindex", "tablist", "tabpanel", "tarekraafat", "tbody", "tbrf", + "Tcnwg", "tems", + "Testinit", + "Testit", + "Testname", "testpassword", "testtest", "testuser", @@ -556,6 +532,7 @@ "tsym", "txid", "txlistinternal", + "Txns", "txpool", "txreceipt", "ueberauth", @@ -565,21 +542,28 @@ "unfetched", "unfinalized", "unindexed", + "Unitarion", + "Unitorius", + "Unitorus", "unknownc", "unknowne", "unmarshal", "unmatching", "unnest", + "unnested", + "unoswap", "unpadded", "unprefixed", "unstaged", + "unxswap", "upsert", "upserted", "upserting", "upserts", "urijs", "urlset", - "urlset", + "Utqn", + "UUPS", "valign", "valuemax", "valuemin", @@ -592,6 +576,7 @@ "volumeto", "vyper", "walletconnect", + "Wanchain", "warninga", "warningb", "watchlist", @@ -622,36 +607,7 @@ "zkatana", "zkbob", "zkevm", - "erts", - "Asfpp", - "Nerg", - "secp", - "qwertyuioiuytrewertyuioiuytrertyuio", - "urlset", - "lastmod", - "qitmeer", - "meer", - "DefiLlama", - "SOLIDITYSCAN", - "fkey", - "getcontractcreation", - "contractaddresses", - "tokennfttx", - "libraryname", - "libraryaddress", - "evmversion", - "verifyproxycontract", - "checkproxyverification", - "NOTOK", - "sushiswap", - "zetachain", - "zksync", - "filecoin", - "Filecoin", - "permissionless", - "feelin", - "KnxbUejwY", - "Btvk" + "zksync" ], "enableFiletypes": [ "dotenv",