feat: return transaction index in response for submitting a transaction

apps/api/lib/api/v1/controllers/transaction_controller.ex

@@ -17,8 +17,11 @@ defmodule API.V1.Controller.TransactionController do
   def submit(hex_tx_bytes) do
     with {:ok, binary} <- Encoding.to_binary(hex_tx_bytes),
          {:ok, changeset} <- Transaction.decode(binary),
-         {:ok, transaction} <- Transaction.insert(changeset) do
-      {:ok, TransactionView.serialize_hash(transaction)}
+         {:ok, %{"new-transaction" => transaction}} <- Transaction.insert(changeset) do
+      {:ok, TransactionView.serialize(transaction)}
+    else
+      {:error, _, changeset, _} -> {:error, changeset}
+      error -> error
     end
   end
 end

apps/api/lib/api/v1/views/transaction_view.ex

@@ -5,10 +5,14 @@ defmodule API.V1.View.TransactionView do
 
   alias ExPlasma.Encoding
 
-  @type serialized_hash() :: %{required(:tx_hash) => String.t()}
+  @type serialized_transaction() :: %{
+          required(:tx_hash) => String.t(),
+          required(:blknum) => pos_integer(),
+          required(:tx_index) => non_neg_integer()
+        }
 
-  @spec serialize_hash(map()) :: serialized_hash()
-  def serialize_hash(transaction) do
-    %{tx_hash: Encoding.to_hex(transaction.tx_hash)}
+  @spec serialize(map()) :: serialized_transaction()
+  def serialize(transaction) do
+    %{tx_hash: Encoding.to_hex(transaction.tx_hash), blknum: transaction.block.blknum, tx_index: transaction.tx_index}
   end
 end

apps/api/test/api/v1/controllers/block_controller_test.exs

@@ -9,8 +9,13 @@ defmodule API.V1.Controller.BlockControllerTest do
   describe "get_by_hash/1" do
     test "it returns a matching block" do
       _ = insert(:fee, hash: "22", type: :merged_fees)
-      %{id: id} = insert(:payment_v1_transaction)
-      Block.form()
+
+      {:ok, %{"new-transaction" => %{id: id}}} =
+        :payment_v1_transaction
+        |> build()
+        |> Transaction.insert()
+
+      _ = Block.form()
       transaction = Transaction |> Repo.get(id) |> Repo.preload(:block)
 
       hash = Encoding.to_hex(transaction.block.hash)

apps/api/test/api/v1/controllers/transaction_controller_test.exs

@@ -17,7 +17,7 @@ defmodule API.V1.Controllere.TransactionControllerTest do
       tx_hash = Encoding.to_hex(txn.tx_hash)
       tx_bytes = Encoding.to_hex(txn.tx_bytes)
 
-      assert TransactionController.submit(tx_bytes) == {:ok, %{tx_hash: tx_hash}}
+      assert TransactionController.submit(tx_bytes) == {:ok, %{tx_hash: tx_hash, blknum: 1_000, tx_index: 0}}
     end
 
     test "it raises an error if the tranasaction is invalid" do

apps/api/test/api/v1/router_test.exs

@@ -165,8 +165,12 @@ defmodule API.V1.RouterTest do
 
   describe "block.get" do
     test "that it returns a block" do
-      %{id: id} = insert(:payment_v1_transaction)
-      Block.form()
+      {:ok, %{"new-transaction" => %{id: id}}} =
+        :payment_v1_transaction
+        |> build()
+        |> Transaction.insert()
+
+      _ = Block.form()
       transaction = Transaction |> Repo.get(id) |> Repo.preload(:block)
 
       tx_bytes = Encoding.to_hex(transaction.tx_bytes)
@@ -212,7 +216,7 @@ defmodule API.V1.RouterTest do
       tx_hash = Encoding.to_hex(txn.tx_hash)
       {:ok, payload} = post("transaction.submit", %{transaction: tx_bytes})
 
-      assert_payload_data(payload, %{"tx_hash" => tx_hash})
+      assert_payload_data(payload, %{"tx_hash" => tx_hash, "blknum" => 1_000, "tx_index" => 0})
     end
 
     test "that it returns an error if missing transaction params" do

apps/api/test/api/v1/views/transaction_view_test.exs

@@ -2,16 +2,20 @@ defmodule API.V1.View.TransactionViewTest do
   use Engine.DB.DataCase, async: true
 
   alias API.V1.View.TransactionView
+  alias Engine.DB.Transaction
   alias ExPlasma.Encoding
 
   describe "serialize/1" do
-    test "serialize a transaction" do
+    test "serializes a transaction" do
       _ = insert(:fee, hash: "55", term: :no_fees_required, type: :merged_fees)
 
-      transaction = build(:payment_v1_transaction)
+      transaction_changeset = build(:payment_v1_transaction)
+      {:ok, %{"new-transaction" => transaction}} = Transaction.insert(transaction_changeset)
 
-      assert TransactionView.serialize_hash(transaction) == %{
-               tx_hash: Encoding.to_hex(transaction.tx_hash)
+      assert TransactionView.serialize(transaction) == %{
+               tx_hash: Encoding.to_hex(transaction.tx_hash),
+               blknum: 1_000,
+               tx_index: 0
              }
     end
   end

apps/engine/lib/engine/db/block.ex

@@ -13,6 +13,12 @@ defmodule Engine.DB.Block do
   - submitted_at_ethereum_height: The rootchain height at wish the block was submitted
   - gas: The gas price used for the submission
   - attempts_counter: The number of submission attempts
+  - txcount: The number of transaction in the block
+  - state:
+      - :forming - block accepts transactions, at most one forming block is allowed in the database
+      - :pending_submission - block no longer accepts for transaction and is waiting for being submitted to the root chain
+      - :submitted - block was submitted to the root chain
+      - :confirmed - block is confirmed on the root chain
   """
 
   use Ecto.Schema
@@ -29,13 +35,25 @@ defmodule Engine.DB.Block do
 
   require Logger
 
-  @optional_fields [:hash, :tx_hash, :formed_at_ethereum_height, :submitted_at_ethereum_height, :gas, :attempts_counter]
-  @required_fields [:nonce, :blknum]
+  @max_transaction_in_block 65_000
+
+  @optional_fields [
+    :hash,
+    :tx_hash,
+    :formed_at_ethereum_height,
+    :submitted_at_ethereum_height,
+    :gas,
+    :attempts_counter,
+    :txcount
+  ]
+  @required_fields [:nonce, :blknum, :state]
 
   @type t() :: %{
           hash: binary(),
+          state: :forming | :pending_submission | :submitted | :confirmed,
           nonce: pos_integer(),
           blknum: pos_integer() | nil,
+          txcount: non_neg_integer(),
           tx_hash: binary() | nil,
           formed_at_ethereum_height: pos_integer() | nil,
           id: pos_integer(),
@@ -54,12 +72,13 @@ defmodule Engine.DB.Block do
   @timestamps_opts [inserted_at: :node_inserted_at, updated_at: :node_updated_at]
 
   schema "blocks" do
-    # Extracted from `output_id`
     field(:hash, :binary)
+    field(:state, Ecto.Atom)
     # nonce = max(nonce) + 1
     field(:nonce, :integer)
     # blknum = nonce * 1000
     field(:blknum, :integer)
+    field(:txcount, :integer)
     field(:tx_hash, :binary)
     field(:formed_at_ethereum_height, :integer)
     field(:submitted_at_ethereum_height, :integer)
@@ -77,6 +96,11 @@ defmodule Engine.DB.Block do
     timestamps()
   end
 
+  def state_forming(), do: :forming
+  def state_pending_submission(), do: :pending_submission
+  def state_submitted(), do: :submitted
+  def state_confirmed(), do: :confirmed
+
   def changeset(struct, params) do
     struct
     |> cast(params, @required_fields ++ @optional_fields)
@@ -96,16 +120,13 @@ defmodule Engine.DB.Block do
   end
 
   @doc """
-  Forms a pending block record based on the existing pending transactions. This
-  attaches free transactions into a new block, awaiting for submission to the contract
-  later on.
+  Forms a block awaiting submission.
   """
   @decorate trace(service: :ecto, type: :backend)
   def form() do
     Multi.new()
-    |> Multi.run("new-block", &insert_block/2)
-    |> Multi.run("form-block", &attach_transactions_to_block/2)
-    |> Multi.run("hash-block", &generate_block_hash/2)
+    |> Multi.run("block", &get_or_insert_forming_block/2)
+    |> Multi.run("block-for-submission", &prepare_for_submission/2)
     |> Repo.transaction()
   end
 
@@ -123,12 +144,42 @@ defmodule Engine.DB.Block do
     end
   end
 
+  @doc """
+  Get a forming block or inserts a new one if current forming block hit it's transaction limit
+  """
+  def get_or_insert_forming_block(repo, params) do
+    {:ok, forming_block} = get_forming_block_for_update(repo, params)
+
+    case forming_block do
+      nil -> insert_block(repo)
+      block -> {:ok, block}
+    end
+  end
+
+  @doc """
+  Increases and returns number of transaction in block
+  """
+  def increase_txcount(repo, %{"block" => block}) do
+    {1, [{txcount}]} =
+      repo.update_all(
+        from(b in __MODULE__, where: b.id == ^block.id, select: {b.txcount}),
+        inc: [txcount: 1]
+      )
+
+    {:ok, txcount}
+  end
+
+  defp get_forming_block_for_update(repo, _params) do
+    block = repo.one(from(block in __MODULE__, where: block.state == ^:forming, lock: "FOR UPDATE"))
+    {:ok, block}
+  end
+
   defp get_all(repo, _changeset, new_height, mined_child_block) do
     query =
-      from(p in __MODULE__,
+      from(b in __MODULE__,
         where:
-          (p.submitted_at_ethereum_height < ^new_height or is_nil(p.submitted_at_ethereum_height)) and
-            p.blknum > ^mined_child_block,
+          (b.submitted_at_ethereum_height < ^new_height or is_nil(b.submitted_at_ethereum_height)) and
+            b.blknum > ^mined_child_block,
         order_by: [asc: :nonce]
       )
 
@@ -154,7 +205,8 @@ defmodule Engine.DB.Block do
         |> change(
           gas: gas,
           attempts_counter: plasma_block.attempts_counter + 1,
-          submitted_at_ethereum_height: new_height
+          submitted_at_ethereum_height: new_height,
+          state: :submitted
         )
         |> repo.update!([])
 
@@ -168,7 +220,18 @@ defmodule Engine.DB.Block do
     end
   end
 
-  defp insert_block(repo, _params) do
+  def insert_block_if_transaction_limit_exceeded(repo, %{"current-forming-block" => block}) do
+    case block.txcount >= @max_transaction_in_block do
+      true ->
+        {:ok, _} = prepare_for_submission(repo, %{"block" => block})
+        insert_block(repo)
+
+      false ->
+        {:ok, block}
+    end
+  end
+
+  defp insert_block(repo) do
     nonce =
       query_max_nonce()
       |> Repo.one()
@@ -179,7 +242,7 @@ defmodule Engine.DB.Block do
 
     blknum = nonce * Configuration.child_block_interval()
 
-    params = %{nonce: nonce, blknum: blknum}
+    params = %{state: :forming, nonce: nonce, blknum: blknum, txcount: 0}
 
     %__MODULE__{}
     |> changeset(params)
@@ -188,25 +251,18 @@ defmodule Engine.DB.Block do
 
   defp query_max_nonce(), do: from(block in __MODULE__, select: max(block.nonce))
 
-  defp attach_transactions_to_block(repo, %{"new-block" => block}) do
-    updates = [block_id: block.id, updated_at: NaiveDateTime.utc_now()]
-    {total, _} = repo.update_all(Transaction.query_pending(), set: updates)
-
-    {:ok, total}
-  end
-
-  defp generate_block_hash(repo, %{"new-block" => block}) do
+  defp prepare_for_submission(repo, %{"block" => block}) do
     hash =
       block.id
       |> fetch_tx_bytes_in_block()
       |> Merkle.root_hash()
 
-    changeset = change(block, hash: hash)
+    changeset = change(block, hash: hash, state: :pending_submission)
     repo.update(changeset)
   end
 
   defp fetch_tx_bytes_in_block(block_id) do
-    query = from(transaction in Transaction, where: transaction.block_id == ^block_id)
+    query = from(transaction in Transaction, where: transaction.block_id == ^block_id, order_by: transaction.tx_index)
 
     query
     |> Repo.all()

apps/engine/lib/engine/db/output.ex

@@ -40,7 +40,7 @@ defmodule Engine.DB.Output do
           position: pos_integer() | nil,
           spending_transaction: Transaction.t() | nil,
           spending_transaction_id: pos_integer() | nil,
-          state: String.t(),
+          state: :forming | :pending_submission | :submitted | :confirmed,
           updated_at: DateTime.t()
         }
 

apps/engine/lib/engine/db/transaction.ex

@@ -28,6 +28,7 @@ defmodule Engine.DB.Transaction do
   import Ecto.Changeset, only: [cast: 3, cast_assoc: 2, validate_required: 2]
   import Ecto.Query, only: [from: 2]
 
+  alias Ecto.Multi
   alias Engine.DB.Block
   alias Engine.DB.Output
   alias Engine.DB.Transaction.Validator
@@ -39,6 +40,7 @@ defmodule Engine.DB.Transaction do
   @type t() :: %{
           block: Block.t(),
           block_id: pos_integer(),
+          tx_index: non_neg_integer(),
           id: pos_integer(),
           inputs: list(Output.t()),
           inserted_at: DateTime.t(),
@@ -60,6 +62,7 @@ defmodule Engine.DB.Transaction do
     field(:tx_bytes, :binary)
     field(:tx_hash, :binary)
     field(:tx_type, :integer)
+    field(:tx_index, :integer)
 
     # Virtual fields used for convenience and validation
     # Avoid decoding/parsing signatures mutiple times along validation process
@@ -127,7 +130,25 @@ defmodule Engine.DB.Transaction do
     |> Validator.validate_statefully(params)
   end
 
-  def insert(changeset), do: Repo.insert(changeset)
+  @doc """
+  Inserts new transaction and associates it with currently forming block.
+  If including a new transaction in a block would violate maximum number of transaction per block
+  then the new transaction is associated with a newly inserted block.
+  """
+  def insert(changeset) do
+    Multi.new()
+    |> Multi.run("current-forming-block", &Block.get_or_insert_forming_block/2)
+    |> Multi.run("block", &Block.insert_block_if_transaction_limit_exceeded/2)
+    |> Multi.run("block-txcount", &Block.increase_txcount/2)
+    |> Multi.run("new-transaction", fn repo, params -> insert_transaction(repo, params, changeset) end)
+    |> Repo.transaction()
+  end
+
+  defp insert_transaction(repo, %{"block-txcount" => block_txcount, "block" => block}, changeset) do
+    changeset
+    |> Ecto.Changeset.change(%{tx_index: block_txcount - 1, block: block})
+    |> repo.insert()
+  end
 
   defp recovered_to_map(transaction) do
     inputs = Enum.map(transaction.inputs, &Map.from_struct/1)