feat: add total accounts stats

lib/ae_mdw/accounts.ex

@@ -0,0 +1,30 @@
+defmodule AeMdw.Accounts do
+  @moduledoc """
+    Module for account related operations
+  """
+  alias AeMdw.Blocks
+  alias AeMdw.Db.Model
+  alias AeMdw.Db.State
+  alias AeMdw.Db.Sync.Stats, as: SyncStats
+  alias AeMdw.Node.Db
+
+  require Model
+
+  @spec maybe_increase_creation_statistics(State.t(), Db.pubkey(), Blocks.time()) :: State.t()
+  def maybe_increase_creation_statistics(state, pubkey, time) do
+    state
+    |> State.get(Model.AccountCreation, pubkey)
+    |> case do
+      :not_found ->
+        state
+        |> State.put(
+          Model.AccountCreation,
+          Model.account_creation(index: pubkey, creation_time: time)
+        )
+        |> SyncStats.increment_statistics(:total_accounts, time, 1)
+
+      _account_creation ->
+        state
+    end
+  end
+end

lib/ae_mdw/db/model.ex

@@ -111,6 +111,16 @@ defmodule AeMdw.Db.Model do
             count: non_neg_integer()
           )
 
+  @account_creation_defaults [index: nil, creation_time: 0]
+  defrecord :account_creation, @account_creation_defaults
+
+  @type account_creation_index() :: pubkey()
+  @type account_creation() ::
+          record(:account_creation,
+            index: account_creation_index(),
+            creation_time: non_neg_integer()
+          )
+
   # txs block index :
   #     index = {kb_index (0..), mb_index}, tx_index = tx_index, hash = block (header) hash
   #     On keyblock boundary: mb_index = -1}
@@ -1379,6 +1389,7 @@ defmodule AeMdw.Db.Model do
     [
       AeMdw.Db.Model.BalanceAccount,
       AeMdw.Db.Model.AccountBalance,
+      AeMdw.Db.Model.AccountCreation,
       AeMdw.Db.Model.AsyncTask,
       AeMdw.Db.Model.Migrations
     ]
@@ -1387,6 +1398,7 @@ defmodule AeMdw.Db.Model do
   @spec record(atom()) :: atom()
   def record(AeMdw.Db.Model.BalanceAccount), do: :balance_account
   def record(AeMdw.Db.Model.AccountBalance), do: :account_balance
+  def record(AeMdw.Db.Model.AccountCreation), do: :account_creation
   def record(AeMdw.Db.Model.AsyncTask), do: :async_task
   def record(AeMdw.Db.Model.Migrations), do: :migrations
   def record(AeMdw.Db.Model.Tx), do: :tx

lib/ae_mdw/db/mutations/account_creation_mutation.ex

@@ -0,0 +1,34 @@
+defmodule AeMdw.Db.AccountCreationMutation do
+  @moduledoc """
+    Mark account creation time
+  """
+
+  alias AeMdw.Accounts
+  alias AeMdw.Blocks
+  alias AeMdw.Db.State
+  alias AeMdw.Node.Db
+
+  @derive AeMdw.Db.Mutation
+  defstruct [:account_pk, :time]
+
+  @opaque t() :: %__MODULE__{
+            account_pk: Db.pubkey(),
+            time: Blocks.time()
+          }
+
+  @spec new(Db.pubkey(), Blocks.time()) :: t()
+  def new(account_pk, time) do
+    %__MODULE__{account_pk: account_pk, time: time}
+  end
+
+  @spec execute(t(), State.t()) :: State.t()
+  def execute(
+        %__MODULE__{
+          account_pk: account_pk,
+          time: time
+        },
+        state
+      ) do
+    Accounts.maybe_increase_creation_statistics(state, account_pk, time)
+  end
+end

lib/ae_mdw/db/temp_store.ex

@@ -0,0 +1,89 @@
+defmodule AeMdw.Db.TempStore do
+  @moduledoc """
+  Store implementation with operations reading/writing in-memory.
+
+  Used to store temporary data before it is written to the main store.
+  """
+  alias AeMdw.Database
+  alias AeMdw.Db.WriteMutation
+
+  @derive AeMdw.Db.Store
+  defstruct [:tables]
+
+  @typep key() :: Database.key()
+  @typep record() :: Database.record()
+  @typep table() :: Database.table()
+
+  @opaque t() :: %__MODULE__{}
+
+  @spec new() :: t()
+  def new, do: %__MODULE__{tables: %{}}
+
+  @spec put(t(), table(), record()) :: t()
+  def put(%__MODULE__{tables: tables}, table, record) do
+    key = elem(record, 1)
+    store_table = Map.get(tables, table, :gb_trees.empty())
+    store_table = :gb_trees.enter(key, record, store_table)
+    %__MODULE__{tables: Map.put(tables, table, store_table)}
+  end
+
+  @spec get(t(), table(), key()) :: {:ok, record()} | :not_found
+  def get(%__MODULE__{tables: tables}, table, key) do
+    store_table = Map.get(tables, table, :gb_trees.empty())
+
+    case :gb_trees.lookup(key, store_table) do
+      {:value, record} -> {:ok, record}
+      :none -> :not_found
+    end
+  end
+
+  @spec delete(t(), table(), key()) :: t()
+  def delete(%__MODULE__{tables: tables}, table, key) do
+    store_table = Map.get(tables, table, :gb_trees.empty())
+    store_table = :gb_trees.delete_any(key, store_table)
+    %__MODULE__{tables: Map.put(tables, table, store_table)}
+  end
+
+  @spec count_keys(t(), table()) :: non_neg_integer()
+  def count_keys(%__MODULE__{tables: tables}, table) do
+    store_table = Map.get(tables, table, :gb_trees.empty())
+    :gb_trees.size(store_table)
+  end
+
+  @spec next(t(), table(), key() | nil) :: {:ok, key()} | :none
+  def next(%__MODULE__{tables: tables}, table, key) do
+    store_table = Map.get(tables, table, :gb_trees.empty())
+    iterator = :gb_trees.iterator_from(key, store_table)
+
+    with {next_key, _value, next_iterator} <- :gb_trees.next(iterator),
+         {true, _next_key} <- {next_key == key, next_key},
+         {actual_next_key, _value, _next_iterator} <- :gb_trees.next(next_iterator) do
+      {:ok, actual_next_key}
+    else
+      {false, next_key} -> {:ok, next_key}
+      :none -> :none
+    end
+  end
+
+  @spec prev(t(), table(), key() | nil) :: {:ok, key()} | :none
+  def prev(_store, _table, _key) do
+    raise "Not implemented"
+  end
+
+  @spec to_mutations(t()) :: Enumerable.t()
+  def to_mutations(%__MODULE__{tables: tables}) do
+    tables
+    |> Stream.flat_map(fn {table, store_table} ->
+      Stream.resource(
+        fn -> :gb_trees.iterator(store_table) end,
+        fn it ->
+          case :gb_trees.next(it) do
+            {_key, record, next_it} -> {[WriteMutation.new(table, record)], next_it}
+            :none -> {:halt, nil}
+          end
+        end,
+        fn _it -> :ok end
+      )
+    end)
+  end
+end

lib/ae_mdw/stats.ex

@@ -50,6 +50,7 @@ defmodule AeMdw.Stats do
           | :difficulty
           | :hashrate
           | :contracts
+          | :total_accounts
   @type interval_by() :: :day | :week | :month
   @type interval_start() :: non_neg_integer()
 
@@ -280,6 +281,14 @@ defmodule AeMdw.Stats do
     end
   end
 
+  @spec fetch_total_accounts_stats(State.t(), pagination(), query(), range(), cursor()) ::
+          {:ok, {pagination_cursor(), [statistic()], pagination_cursor()}} | {:error, reason()}
+  def fetch_total_accounts_stats(state, pagination, query, range, cursor) do
+    with {:ok, filters} <- Util.convert_params(query, &convert_param/1) do
+      fetch_statistics(state, pagination, filters, range, cursor, :total_accounts)
+    end
+  end
+
   defp fetch_statistics(state, pagination, filters, range, cursor, tag) do
     with {:ok, cursor} <- deserialize_statistic_cursor(cursor) do
       paginated_statistics =

lib/ae_mdw/sync/server.ex

@@ -36,9 +36,11 @@ defmodule AeMdw.Sync.Server do
   alias AeMdw.Db.State
   alias AeMdw.Db.Status
   alias AeMdw.Db.Sync.Block
+  alias AeMdw.Db.AccountCreationMutation
   alias AeMdw.Db.UpdateBalanceAccountMutation
   alias AeMdw.Db.RollbackMutation
   alias AeMdw.Log
+  alias AeMdw.Node.Db
   alias AeMdw.Sync.AsyncTasks.WealthRankAccounts
   alias AeMdw.Sync.MemStoreCreator
   alias AeMdwWeb.Websocket.Broadcaster
@@ -357,7 +359,14 @@ defmodule AeMdw.Sync.Server do
           [{block_index, mblock, UpdateBalanceAccountMutation.new(account_pk, balance)} | acc]
         end)
 
-      gen_mutations ++ account_balances_mutations
+      time = Db.get_block_time(mb_hash)
+
+      account_creation_mutations =
+        Enum.map(accounts_set, fn pubkey ->
+          {block_index, mblock, AccountCreationMutation.new(pubkey, time)}
+        end)
+
+      gen_mutations ++ account_balances_mutations ++ account_creation_mutations
     else
       gen_mutations
     end

lib/ae_mdw_web/controllers/stats_controller.ex

@@ -10,11 +10,13 @@ defmodule AeMdwWeb.StatsController do
 
   @stats_limit 1_000
 
-  plug PaginatedPlug when action not in ~w(transactions_stats blocks_stats names_stats)a
+  plug(PaginatedPlug when action not in ~w(transactions_stats blocks_stats names_stats)a)
 
-  plug PaginatedPlug,
-       [max_limit: @stats_limit]
-       when action in ~w(transactions_stats blocks_stats names_stats)a
+  plug(
+    PaginatedPlug,
+    [max_limit: @stats_limit]
+    when action in ~w(transactions_stats blocks_stats names_stats)a
+  )
 
   action_fallback(FallbackController)
 
@@ -134,4 +136,14 @@ defmodule AeMdwWeb.StatsController do
       Util.render(conn, paginated_stats)
     end
   end
+
+  @spec total_accounts_stats(Conn.t(), map()) :: Conn.t()
+  def total_accounts_stats(%Conn{assigns: assigns} = conn, _params) do
+    %{state: state, pagination: pagination, query: query, scope: scope, cursor: cursor} = assigns
+
+    with {:ok, paginated_stats} <-
+           Stats.fetch_total_accounts_stats(state, pagination, query, scope, cursor) do
+      Util.render(conn, paginated_stats)
+    end
+  end
 end

lib/ae_mdw_web/router.ex

@@ -69,6 +69,7 @@ defmodule AeMdwWeb.Router do
       get "/stats/blocks", StatsController, :blocks_stats
       get "/stats/difficulty", StatsController, :difficulty_stats
       get "/stats/hashrate", StatsController, :hashrate_stats
+      get "/stats/total-accounts", StatsController, :total_accounts_stats
       get "/stats/names", StatsController, :names_stats
       get "/stats/total", StatsController, :total_stats
       get "/stats/delta", StatsController, :delta_stats

priv/migrations/20241016112036_add_account_creation_table.ex

@@ -0,0 +1,83 @@
+defmodule AeMdw.Migrations.AddAccountCreationTable do
+  @moduledoc """
+  Add account creation table and update account creation statistics.
+  """
+
+  alias AeMdw.Collection
+  alias AeMdw.Db.State
+  alias AeMdw.Db.Model
+  alias AeMdw.Db.WriteMutation
+  alias AeMdw.Db.DeleteKeysMutation
+  alias AeMdw.Db.StatisticsMutation
+  alias AeMdw.Db.Sync.Stats, as: SyncStats
+  alias AeMdw.Db.RocksDbCF
+  alias AeMdw.Sync.Transaction
+
+  require Model
+
+  @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()}
+  def run(state, _from_start?) do
+    keys_to_delete = state |> Collection.stream(Model.AccountCreation, nil) |> Enum.to_list()
+    clear_mutation = DeleteKeysMutation.new(%{Model.AccountCreation => keys_to_delete})
+    state = State.commit_db(state, [clear_mutation])
+
+    protocol_accounts =
+      for {protocol, height} <- :aec_hard_forks.protocols(),
+          protocol <= :aec_hard_forks.protocol_vsn(:lima),
+          {account, _balance} <- :aec_fork_block_settings.accounts(protocol),
+          into: %{} do
+        {account, height}
+      end
+
+    Model.Tx
+    |> RocksDbCF.stream()
+    |> Task.async_stream(fn Model.tx(id: tx_hash, time: time) ->
+      tx_hash
+      |> :aec_db.get_signed_tx()
+      |> Transaction.get_ids_from_tx()
+      |> Enum.reduce(%{}, fn
+        {:id, :account, pubkey}, acc ->
+          Map.put_new(acc, pubkey, time)
+
+        _other, acc ->
+          acc
+      end)
+    end)
+    |> Enum.reduce(protocol_accounts, fn {:ok, new_map}, acc_times ->
+      Map.merge(acc_times, new_map, fn _k, v1, v2 ->
+        min(v1, v2)
+      end)
+    end)
+    |> Enum.reduce({%{}, []}, fn {pubkey, time}, {statistics, mutations} ->
+      new_statistics =
+        time
+        |> SyncStats.time_intervals()
+        |> Enum.map(fn {interval_by, interval_start} ->
+          {:total_accounts, interval_by, interval_start}
+        end)
+        |> Enum.reduce(statistics, fn key, statistics ->
+          Map.update(statistics, key, 1, &(&1 + 1))
+        end)
+
+      {new_statistics,
+       [
+         WriteMutation.new(
+           Model.AccountCreation,
+           Model.account_creation(index: pubkey, creation_time: time)
+         )
+         | mutations
+       ]}
+    end)
+    |> then(fn {statistics, mutations} ->
+      Stream.concat(mutations, [StatisticsMutation.new(statistics)])
+    end)
+    |> Stream.chunk_every(1000)
+    |> Enum.reduce({state, 0}, fn mutations, {acc_state, count} ->
+      {
+        State.commit_db(acc_state, mutations),
+        count + length(mutations)
+      }
+    end)
+    |> then(fn {_state, count} -> {:ok, count} end)
+  end
+end