diff --git a/lib/ae_mdw/accounts.ex b/lib/ae_mdw/accounts.ex new file mode 100644 index 000000000..a2e741a08 --- /dev/null +++ b/lib/ae_mdw/accounts.ex @@ -0,0 +1,30 @@ +defmodule AeMdw.Accounts do + @moduledoc """ + Module for account related operations + """ + alias AeMdw.Blocks + alias AeMdw.Db.Model + alias AeMdw.Db.State + alias AeMdw.Db.Sync.Stats, as: SyncStats + alias AeMdw.Node.Db + + require Model + + @spec maybe_increase_creation_statistics(State.t(), Db.pubkey(), Blocks.time()) :: State.t() + def maybe_increase_creation_statistics(state, pubkey, time) do + state + |> State.get(Model.AccountCreation, pubkey) + |> case do + :not_found -> + state + |> State.put( + Model.AccountCreation, + Model.account_creation(index: pubkey, creation_time: time) + ) + |> SyncStats.increment_statistics(:total_accounts, time, 1) + + _account_creation -> + state + end + end +end diff --git a/lib/ae_mdw/db/model.ex b/lib/ae_mdw/db/model.ex index 1ef054ab1..e7ab9c63e 100644 --- a/lib/ae_mdw/db/model.ex +++ b/lib/ae_mdw/db/model.ex @@ -111,6 +111,16 @@ defmodule AeMdw.Db.Model do count: non_neg_integer() ) + @account_creation_defaults [index: nil, creation_time: 0] + defrecord :account_creation, @account_creation_defaults + + @type account_creation_index() :: pubkey() + @type account_creation() :: + record(:account_creation, + index: account_creation_index(), + creation_time: non_neg_integer() + ) + # txs block index : # index = {kb_index (0..), mb_index}, tx_index = tx_index, hash = block (header) hash # On keyblock boundary: mb_index = -1} @@ -1379,6 +1389,7 @@ defmodule AeMdw.Db.Model do [ AeMdw.Db.Model.BalanceAccount, AeMdw.Db.Model.AccountBalance, + AeMdw.Db.Model.AccountCreation, AeMdw.Db.Model.AsyncTask, AeMdw.Db.Model.Migrations ] @@ -1387,6 +1398,7 @@ defmodule AeMdw.Db.Model do @spec record(atom()) :: atom() def record(AeMdw.Db.Model.BalanceAccount), do: :balance_account def record(AeMdw.Db.Model.AccountBalance), do: :account_balance + def record(AeMdw.Db.Model.AccountCreation), do: :account_creation def record(AeMdw.Db.Model.AsyncTask), do: :async_task def record(AeMdw.Db.Model.Migrations), do: :migrations def record(AeMdw.Db.Model.Tx), do: :tx diff --git a/lib/ae_mdw/db/mutations/account_creation_mutation.ex b/lib/ae_mdw/db/mutations/account_creation_mutation.ex new file mode 100644 index 000000000..6d045db9b --- /dev/null +++ b/lib/ae_mdw/db/mutations/account_creation_mutation.ex @@ -0,0 +1,34 @@ +defmodule AeMdw.Db.AccountCreationMutation do + @moduledoc """ + Mark account creation time + """ + + alias AeMdw.Accounts + alias AeMdw.Blocks + alias AeMdw.Db.State + alias AeMdw.Node.Db + + @derive AeMdw.Db.Mutation + defstruct [:account_pk, :time] + + @opaque t() :: %__MODULE__{ + account_pk: Db.pubkey(), + time: Blocks.time() + } + + @spec new(Db.pubkey(), Blocks.time()) :: t() + def new(account_pk, time) do + %__MODULE__{account_pk: account_pk, time: time} + end + + @spec execute(t(), State.t()) :: State.t() + def execute( + %__MODULE__{ + account_pk: account_pk, + time: time + }, + state + ) do + Accounts.maybe_increase_creation_statistics(state, account_pk, time) + end +end diff --git a/lib/ae_mdw/stats.ex b/lib/ae_mdw/stats.ex index eb653e4e3..5bbafb655 100644 --- a/lib/ae_mdw/stats.ex +++ b/lib/ae_mdw/stats.ex @@ -50,6 +50,7 @@ defmodule AeMdw.Stats do | :difficulty | :hashrate | :contracts + | :total_accounts @type interval_by() :: :day | :week | :month @type interval_start() :: non_neg_integer() @@ -280,6 +281,14 @@ defmodule AeMdw.Stats do end end + @spec fetch_total_accounts_stats(State.t(), pagination(), query(), range(), cursor()) :: + {:ok, {pagination_cursor(), [statistic()], pagination_cursor()}} | {:error, reason()} + def fetch_total_accounts_stats(state, pagination, query, range, cursor) do + with {:ok, filters} <- Util.convert_params(query, &convert_param/1) do + fetch_statistics(state, pagination, filters, range, cursor, :total_accounts) + end + end + defp fetch_statistics(state, pagination, filters, range, cursor, tag) do with {:ok, cursor} <- deserialize_statistic_cursor(cursor) do paginated_statistics = diff --git a/lib/ae_mdw/sync/server.ex b/lib/ae_mdw/sync/server.ex index 3f76b6463..1cf511742 100644 --- a/lib/ae_mdw/sync/server.ex +++ b/lib/ae_mdw/sync/server.ex @@ -36,9 +36,11 @@ defmodule AeMdw.Sync.Server do alias AeMdw.Db.State alias AeMdw.Db.Status alias AeMdw.Db.Sync.Block + alias AeMdw.Db.AccountCreationMutation alias AeMdw.Db.UpdateBalanceAccountMutation alias AeMdw.Db.RollbackMutation alias AeMdw.Log + alias AeMdw.Node.Db alias AeMdw.Sync.AsyncTasks.WealthRankAccounts alias AeMdw.Sync.MemStoreCreator alias AeMdwWeb.Websocket.Broadcaster @@ -357,7 +359,14 @@ defmodule AeMdw.Sync.Server do [{block_index, mblock, UpdateBalanceAccountMutation.new(account_pk, balance)} | acc] end) - gen_mutations ++ account_balances_mutations + time = Db.get_block_time(mb_hash) + + account_creation_mutations = + Enum.map(accounts_set, fn pubkey -> + {block_index, mblock, AccountCreationMutation.new(pubkey, time)} + end) + + gen_mutations ++ account_balances_mutations ++ account_creation_mutations else gen_mutations end diff --git a/lib/ae_mdw_web/controllers/stats_controller.ex b/lib/ae_mdw_web/controllers/stats_controller.ex index d0a50f15b..64602bdbf 100644 --- a/lib/ae_mdw_web/controllers/stats_controller.ex +++ b/lib/ae_mdw_web/controllers/stats_controller.ex @@ -10,11 +10,13 @@ defmodule AeMdwWeb.StatsController do @stats_limit 1_000 - plug PaginatedPlug when action not in ~w(transactions_stats blocks_stats names_stats)a + plug(PaginatedPlug when action not in ~w(transactions_stats blocks_stats names_stats)a) - plug PaginatedPlug, - [max_limit: @stats_limit] - when action in ~w(transactions_stats blocks_stats names_stats)a + plug( + PaginatedPlug, + [max_limit: @stats_limit] + when action in ~w(transactions_stats blocks_stats names_stats)a + ) action_fallback(FallbackController) @@ -134,4 +136,14 @@ defmodule AeMdwWeb.StatsController do Util.render(conn, paginated_stats) end end + + @spec total_accounts_stats(Conn.t(), map()) :: Conn.t() + def total_accounts_stats(%Conn{assigns: assigns} = conn, _params) do + %{state: state, pagination: pagination, query: query, scope: scope, cursor: cursor} = assigns + + with {:ok, paginated_stats} <- + Stats.fetch_total_accounts_stats(state, pagination, query, scope, cursor) do + Util.render(conn, paginated_stats) + end + end end diff --git a/lib/ae_mdw_web/router.ex b/lib/ae_mdw_web/router.ex index dc1bc2da0..76ce462b1 100644 --- a/lib/ae_mdw_web/router.ex +++ b/lib/ae_mdw_web/router.ex @@ -69,6 +69,7 @@ defmodule AeMdwWeb.Router do get "/stats/blocks", StatsController, :blocks_stats get "/stats/difficulty", StatsController, :difficulty_stats get "/stats/hashrate", StatsController, :hashrate_stats + get "/stats/total-accounts", StatsController, :total_accounts_stats get "/stats/names", StatsController, :names_stats get "/stats/total", StatsController, :total_stats get "/stats/delta", StatsController, :delta_stats diff --git a/priv/migrations/20241016112036_add_account_creation_table.ex b/priv/migrations/20241016112036_add_account_creation_table.ex new file mode 100644 index 000000000..55b6f6eed --- /dev/null +++ b/priv/migrations/20241016112036_add_account_creation_table.ex @@ -0,0 +1,83 @@ +defmodule AeMdw.Migrations.AddAccountCreationTable do + @moduledoc """ + Add account creation table and update account creation statistics. + """ + + alias AeMdw.Collection + alias AeMdw.Db.State + alias AeMdw.Db.Model + alias AeMdw.Db.WriteMutation + alias AeMdw.Db.DeleteKeysMutation + alias AeMdw.Db.StatisticsMutation + alias AeMdw.Db.Sync.Stats, as: SyncStats + alias AeMdw.Db.RocksDbCF + alias AeMdw.Sync.Transaction + + require Model + + @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} + def run(state, _from_start?) do + keys_to_delete = state |> Collection.stream(Model.AccountCreation, nil) |> Enum.to_list() + clear_mutation = DeleteKeysMutation.new(%{Model.AccountCreation => keys_to_delete}) + state = State.commit_db(state, [clear_mutation]) + + protocol_accounts = + for {protocol, height} <- :aec_hard_forks.protocols(), + protocol <= :aec_hard_forks.protocol_vsn(:lima), + {account, _balance} <- :aec_fork_block_settings.accounts(protocol), + into: %{} do + {account, height} + end + + Model.Tx + |> RocksDbCF.stream() + |> Task.async_stream(fn Model.tx(id: tx_hash, time: time) -> + tx_hash + |> :aec_db.get_signed_tx() + |> Transaction.get_ids_from_tx() + |> Enum.reduce(%{}, fn + {:id, :account, pubkey}, acc -> + Map.put_new(acc, pubkey, time) + + _other, acc -> + acc + end) + end) + |> Enum.reduce(protocol_accounts, fn {:ok, new_map}, acc_times -> + Map.merge(acc_times, new_map, fn _k, v1, v2 -> + min(v1, v2) + end) + end) + |> Enum.reduce({%{}, []}, fn {pubkey, time}, {statistics, mutations} -> + new_statistics = + time + |> SyncStats.time_intervals() + |> Enum.map(fn {interval_by, interval_start} -> + {:total_accounts, interval_by, interval_start} + end) + |> Enum.reduce(statistics, fn key, statistics -> + Map.update(statistics, key, 1, &(&1 + 1)) + end) + + {new_statistics, + [ + WriteMutation.new( + Model.AccountCreation, + Model.account_creation(index: pubkey, creation_time: time) + ) + | mutations + ]} + end) + |> then(fn {statistics, mutations} -> + Stream.concat(mutations, [StatisticsMutation.new(statistics)]) + end) + |> Stream.chunk_every(1000) + |> Enum.reduce({state, 0}, fn mutations, {acc_state, count} -> + { + State.commit_db(acc_state, mutations), + count + length(mutations) + } + end) + |> then(fn {_state, count} -> {:ok, count} end) + end +end diff --git a/test/ae_mdw/db/account_creation_mutation_test.exs b/test/ae_mdw/db/account_creation_mutation_test.exs new file mode 100644 index 000000000..00bd751ec --- /dev/null +++ b/test/ae_mdw/db/account_creation_mutation_test.exs @@ -0,0 +1,44 @@ +defmodule AeMdw.Db.AccountCreationMutationTest do + use AeMdw.Db.MutationCase + + alias AeMdw.Collection + alias AeMdw.Db.Model + alias AeMdw.Db.AccountCreationMutation + alias AeMdw.Db.State + + require Model + + test "account creation mutation" do + state = empty_state() + + state = AccountCreationMutation.execute(AccountCreationMutation.new(<<1::256>>, 1), state) + + key_boundary = {{:total_accounts, :day, -1}, {:total_accounts, :week, nil}} + + all_active_account_statistics = + state + |> Collection.stream(Model.Statistic, :forward, key_boundary, nil) + |> Enum.to_list() + + [_all_account_creations] = + state + |> Collection.stream(Model.AccountCreation, nil) + |> Enum.to_list() + + assert 3 = Enum.count(all_active_account_statistics) + + state = State.commit(state, [AccountCreationMutation.new(<<1::256>>, 5)]) + + all_active_account_statistics = + state + |> Collection.stream(Model.Statistic, :forward, key_boundary, nil) + |> Enum.to_list() + + [_all_account_creations] = + state + |> Collection.stream(Model.AccountCreation, nil) + |> Enum.to_list() + + assert 3 = Enum.count(all_active_account_statistics) + end +end diff --git a/test/ae_mdw_web/controllers/stats_controller_test.exs b/test/ae_mdw_web/controllers/stats_controller_test.exs index 62b806b04..2830fb2a2 100644 --- a/test/ae_mdw_web/controllers/stats_controller_test.exs +++ b/test/ae_mdw_web/controllers/stats_controller_test.exs @@ -73,6 +73,29 @@ defmodule AeMdwWeb.StatsControllerTest do end end + describe "total_accounts_stats" do + test "it returns total_accounts stats", %{conn: conn, store: store} do + store = + store + |> Store.put( + Model.Statistic, + Model.statistic(index: {:total_accounts, :week, 15_552}, count: 1) + ) + + assert %{"prev" => nil, "data" => [stat1], "next" => nil} = + conn + |> with_store(store) + |> get("/v3/stats/total-accounts") + |> json_response(200) + + assert %{ + "count" => 0, + "start_date" => "2018-12-11", + "end_date" => "2018-12-12" + } = stat1 + end + end + describe "transactions_stats" do test "it returns the count of transactions for the latest daily periods", %{ conn: conn,