test: performance checking

apps/engine/lib/engine/db/block.ex

@@ -13,7 +13,6 @@ defmodule Engine.DB.Block do
   - submitted_at_ethereum_height: The rootchain height at wish the block was submitted
   - gas: The gas price used for the submission
   - attempts_counter: The number of submission attempts
-  - txcount: The number of transaction in the block
   - state:
       - :forming - block accepts transactions, at most one forming block is allowed in the database
       - :pending_submission - block no longer accepts for transaction and is waiting for being submitted to the root chain
@@ -45,14 +44,13 @@ defmodule Engine.DB.Block do
     :gas,
     :attempts_counter
   ]
-  @required_fields [:nonce, :blknum, :state, :txcount]
+  @required_fields [:nonce, :blknum, :state]
 
   @type t() :: %{
           hash: binary(),
-          state: :forming | :pending_submission | :submitted | :confirmed,
+          state: :forming | :finalizing | :pending_submission | :submitted | :confirmed,
           nonce: pos_integer(),
           blknum: pos_integer() | nil,
-          txcount: non_neg_integer(),
           tx_hash: binary() | nil,
           formed_at_ethereum_height: pos_integer() | nil,
           id: pos_integer(),
@@ -77,7 +75,6 @@ defmodule Engine.DB.Block do
     field(:nonce, :integer)
     # blknum = nonce * 1000
     field(:blknum, :integer)
-    field(:txcount, :integer)
     field(:tx_hash, :binary)
     field(:formed_at_ethereum_height, :integer)
     field(:submitted_at_ethereum_height, :integer)
@@ -124,8 +121,8 @@ defmodule Engine.DB.Block do
   @decorate trace(service: :ecto, type: :backend)
   def form() do
     Multi.new()
-    |> Multi.run("block", &get_or_insert_forming_block/2)
-    |> Multi.run("block-for-submission", &prepare_for_submission/2)
+    |> Multi.run(:block, &get_or_insert_forming_block/2)
+    |> Multi.run(:block_for_submission, &prepare_for_submission/2)
     |> Repo.transaction()
   end
 
@@ -147,16 +144,20 @@ defmodule Engine.DB.Block do
   Get a forming block or inserts a new one if current forming block hit it's transaction limit
   """
   def get_or_insert_forming_block(repo, params) do
-    {:ok, forming_block} = get_forming_block_for_update(repo, params)
+    {:ok, block} = get_forming_block_for_update(repo, params)
 
-    case forming_block do
-      nil -> insert_block(repo)
-      block -> {:ok, block}
+    case block do
+      nil ->
+        insert_block(repo)
+
+      block ->
+        {:ok, block}
     end
   end
 
   defp get_forming_block_for_update(repo, _params) do
     block = repo.one(from(block in __MODULE__, where: block.state == ^:forming, lock: "FOR UPDATE"))
+
     {:ok, block}
   end
 
@@ -206,14 +207,15 @@ defmodule Engine.DB.Block do
     end
   end
 
-  def insert_block_if_transaction_limit_exceeded(repo, %{current_forming_block: block}) do
+  def get_block_and_tx_index_for_transaction(repo, params) do
+    %{current_forming_block: block} = params
     # this is safe as long as we lock on currently forming block
     last_tx_index = repo.one(from(t in Transaction, where: t.block_id == ^block.id, select: max(t.tx_index))) || -1
 
     # basic version, checking transaction limit is postponed until we get transction fees implemented
     case last_tx_index >= @max_transaction_in_block do
       true ->
-        {:ok, _} = prepare_for_submission(repo, %{"block" => block})
+        {:ok, _} = finalize_block(repo, block)
         {:ok, new_forming_block} = insert_block(repo)
         {:ok, %{block: new_forming_block, next_tx_index: 0}}
 
@@ -233,16 +235,20 @@ defmodule Engine.DB.Block do
 
     blknum = nonce * Configuration.child_block_interval()
 
-    params = %{state: :forming, nonce: nonce, blknum: blknum, txcount: 0}
+    params = %{state: :forming, nonce: nonce, blknum: blknum}
 
-    %__MODULE__{}
-    |> changeset(params)
-    |> repo.insert()
+    IO.inspect(
+      %__MODULE__{}
+      |> changeset(params)
+      |> repo.insert(on_conflict: :nothing, returning: true)
+    )
   end
 
   defp query_max_nonce(), do: from(block in __MODULE__, select: max(block.nonce))
 
-  defp prepare_for_submission(repo, %{"block" => block}) do
+  defp prepare_for_submission(repo, params) do
+    block = params.block
+
     hash =
       block.id
       |> fetch_tx_bytes_in_block()
@@ -252,6 +258,12 @@ defmodule Engine.DB.Block do
     repo.update(changeset)
   end
 
+  defp finalize_block(repo, block) do
+    block
+    |> change(%{state: :finalizing})
+    |> repo.update()
+  end
+
   defp fetch_tx_bytes_in_block(block_id) do
     query = from(transaction in Transaction, where: transaction.block_id == ^block_id, order_by: transaction.tx_index)
 

apps/engine/lib/engine/db/transaction.ex

@@ -133,16 +133,16 @@ defmodule Engine.DB.Transaction do
 
   @doc """
   Inserts new transaction and associates it with currently forming block.
-  If including a new transaction in a block would violate maximum number of transaction per block
-  then the new transaction is associated with a newly inserted block.
+  If including a new transaction in forming block violates maximum number of transaction per block
+  then the transaction is associated with a newly inserted forming block.
   """
   def insert(changeset) do
     Multi.new()
     |> Multi.run(:current_forming_block, &Block.get_or_insert_forming_block/2)
-    |> Multi.run(:block_with_next_tx_index, &Block.insert_block_if_transaction_limit_exceeded/2)
+    |> Multi.run(:block_with_next_tx_index, &Block.get_block_and_tx_index_for_transaction/2)
     |> Multi.insert(:new_transaction, fn %{block_with_next_tx_index: block_with_next_tx_index} ->
-      %{block: block, next_tx_index: next_tx_index} = block_with_next_tx_index
-      Ecto.Changeset.change(changeset, %{block: block, tx_index: next_tx_index})
+      %{block: block, next_tx_index: tx_index} = block_with_next_tx_index
+      Ecto.Changeset.change(changeset, %{block: block, tx_index: tx_index})
     end)
     |> Repo.transaction()
   end

apps/engine/priv/repo/migrations/20200326103841_create_blocks_table.exs

@@ -11,8 +11,6 @@ defmodule Engine.Repo.Migrations.CreateBlocksTable do
       add(:nonce, :integer, null: false)
       # plasma block number
       add(:blknum, :integer, null: false)
-      # number of transactions in block
-      add(:txcount, :integer, null: false, default: 0)
       # submitted transaction hash (gets updated with submitted_at_ethereum_height)
       add(:tx_hash, :binary)
       # at which height did we form the block
@@ -40,14 +38,6 @@ defmodule Engine.Repo.Migrations.CreateBlocksTable do
       )
     )
 
-    create(
-      constraint(
-        :blocks,
-        :txcount,
-        check: "txcount <= 65000"
-      )
-    )
-
     execute("SELECT ecto_manage_updated_at('blocks');")
   end
 end

apps/engine/test/engine/db/block_test.exs

@@ -17,7 +17,7 @@ defmodule Engine.DB.BlockTest do
   describe "form/0" do
     test "forms a block with all transaction associated with it" do
       _ = insert_transaction(:payment_v1_transaction)
-      {:ok, %{"block-for-submission" => block}} = Block.form()
+      {:ok, %{:block_for_submission => block}} = Block.form()
 
       transactions = Repo.all(from(t in Transaction, where: t.block_id == ^block.id))
 
@@ -29,12 +29,12 @@ defmodule Engine.DB.BlockTest do
       txn1 = insert_transaction(:payment_v1_transaction)
       hash = Merkle.root_hash([Transaction.encode_unsigned(txn1)])
 
-      assert {:ok, %{"block-for-submission" => block}} = Block.form()
+      assert {:ok, %{:block_for_submission => block}} = Block.form()
       assert block.hash == hash
     end
 
     test "correctly generates block hash for empty txs list" do
-      assert {:ok, %{"block-for-submission" => block}} = Block.form()
+      assert {:ok, %{:block_for_submission => block}} = Block.form()
 
       assert block.hash ==
                <<246, 9, 190, 253, 254, 144, 102, 254, 20, 231, 67, 179, 98, 62, 174, 135, 143, 188, 70, 128, 5, 96,
@@ -67,7 +67,7 @@ defmodule Engine.DB.BlockTest do
         _ = insert_transaction(:payment_v1_transaction, tx_bytes: tx_bytes)
       end)
 
-      assert {:ok, %{"block-for-submission" => block}} = Block.form()
+      assert {:ok, %{:block_for_submission => block}} = Block.form()
 
       assert block.hash ==
                <<189, 245, 69, 5, 94, 45, 148, 210, 5, 89, 98, 245, 201, 111, 222, 48, 61, 114, 145, 55, 122, 84, 196,
@@ -87,22 +87,25 @@ defmodule Engine.DB.BlockTest do
 
       bob_address = <<207, 194, 79, 222, 88, 128, 171, 217, 153, 41, 195, 239, 138, 178, 227, 16, 72, 173, 118, 35>>
 
-      Enum.each(1..15_000, fn index ->
-        tx_bytes =
-          ExPlasma.payment_v1()
-          |> ExPlasma.Builder.new()
-          |> ExPlasma.Builder.add_input(blknum: 1, txindex: index, oindex: index)
-          |> ExPlasma.Builder.add_output(
-            output_type: 1,
-            output_data: %{output_guard: bob_address, token: <<0::160>>, amount: 100}
-          )
-          |> ExPlasma.Builder.sign!([alice_priv_key])
-          |> ExPlasma.encode!()
+      time =
+        Enum.reduce(1..15_000, 0, fn index, acc ->
+          tx_bytes =
+            ExPlasma.payment_v1()
+            |> ExPlasma.Builder.new()
+            |> ExPlasma.Builder.add_input(blknum: 1, txindex: index, oindex: index)
+            |> ExPlasma.Builder.add_output(
+              output_type: 1,
+              output_data: %{output_guard: bob_address, token: <<0::160>>, amount: 100}
+            )
+            |> ExPlasma.Builder.sign!([alice_priv_key])
+            |> ExPlasma.encode!()
+
+          acc + insert_transaction(:payment_v1_transaction, tx_bytes: tx_bytes)
+        end)
 
-        _ = insert_transaction(:payment_v1_transaction, tx_bytes: tx_bytes)
-      end)
+      IO.inspect(time / 1_000_000)
 
-      assert {:ok, %{"block-for-submission" => block}} = Block.form()
+      assert {:ok, %{:block_for_submission => block}} = Block.form()
 
       assert block.hash ==
                <<12, 40, 202, 7, 16, 175, 119, 138, 7, 95, 8, 3, 148, 93, 162, 168, 136, 226, 196, 236, 83, 62, 220, 75,
@@ -112,20 +115,20 @@ defmodule Engine.DB.BlockTest do
     test "assigns nonce and blknum" do
       _ = insert(:payment_v1_transaction)
 
-      assert {:ok, %{"block-for-submission" => block}} = Block.form()
+      assert {:ok, %{:block_for_submission => block}} = Block.form()
       assert block.nonce == 1
       assert block.blknum == 1_000
     end
 
     test "autoincrements nonce and blknum" do
-      assert {:ok, %{"block-for-submission" => block1}} = Block.form()
+      assert {:ok, %{:block_for_submission => block1}} = Block.form()
       assert block1.nonce == 1
       assert block1.blknum == 1_000
 
       tx = build(:payment_v1_transaction)
       _ = Transaction.insert(tx)
 
-      assert {:ok, %{"block-for-submission" => block2}} = Block.form()
+      assert {:ok, %{:block_for_submission => block2}} = Block.form()
       assert block2.nonce == 2
       assert block2.blknum == 2_000
     end
@@ -135,7 +138,7 @@ defmodule Engine.DB.BlockTest do
     test "fails to insert block when blknum != 1000 * nonce" do
       assert_raise Ecto.ConstraintError, ~r/block_number_nonce/, fn ->
         %Block{}
-        |> Block.changeset(%{nonce: 1, blknum: 2000, state: :forming, txcount: 0})
+        |> Block.changeset(%{nonce: 1, blknum: 2000, state: :forming})
         |> Repo.insert()
       end
     end
@@ -144,7 +147,7 @@ defmodule Engine.DB.BlockTest do
   describe "get_by_hash/2" do
     test "returns the block without preloads" do
       _ = insert(:payment_v1_transaction)
-      {:ok, %{"block-for-submission" => block}} = Block.form()
+      {:ok, %{:block_for_submission => block}} = Block.form()
 
       assert {:ok, block_result} = Block.get_by_hash(block.hash, [])
       refute Ecto.assoc_loaded?(block_result.transactions)
@@ -153,7 +156,7 @@ defmodule Engine.DB.BlockTest do
 
     test "returns the block with preloads" do
       %{tx_hash: tx_hash} = insert_transaction(:payment_v1_transaction)
-      {:ok, %{"block-for-submission" => block}} = Block.form()
+      {:ok, %{:block_for_submission => block}} = Block.form()
 
       assert {:ok, block_result} = Block.get_by_hash(block.hash, :transactions)
       assert [%{tx_hash: ^tx_hash}] = block_result.transactions
@@ -455,11 +458,9 @@ defmodule Engine.DB.BlockTest do
   end
 
   defp insert_transaction(kind, attr \\ []) do
-    {:ok, %{:new_transaction => transaction}} =
-      kind
-      |> build(attr)
-      |> Transaction.insert()
+    changeset = build(kind, attr)
+    {time, {:ok, %{:new_transaction => transaction}}} = :timer.tc(fn -> Transaction.insert(changeset) end)
 
-    transaction
+    time
   end
 end

apps/engine/test/engine/db/transaction_test.exs

@@ -37,6 +37,28 @@ defmodule Engine.DB.TransactionTest do
 
       assert tx1.tx_index + 1 == tx2.tx_index
     end
+
+    test "does not create conflicts when inserting a new forming block" do
+      # if there is no forming block in the db or block reached it's transaction limit
+      # new block is inserted
+      no_conflicts =
+        1..10
+        |> Enum.map(fn blknum -> build(:payment_v1_transaction, %{blknum: blknum}) end)
+        |> Enum.map(fn changeset -> Task.async(fn -> Transaction.insert(changeset) end) end)
+        |> Enum.map(fn task -> Task.await(task) end)
+        |> Enum.all?(fn
+          {:ok, _} -> true
+          _ -> false
+        end)
+
+      assert no_conflicts
+
+      [tx1 | transactions] = Repo.all(from(t in Transaction))
+      refute tx1.block_id == nil
+
+      all_in_same_block = Enum.all?(transactions, fn t -> t.block_id == tx1.block_id end)
+      assert all_in_same_block
+    end
   end
 
   describe "decode/2" do

apps/engine/test/support/db/factory.ex

@@ -198,7 +198,6 @@ defmodule Engine.DB.Factory do
       nonce: nonce,
       blknum: blknum,
       state: Map.get(attr, :state, :forming),
-      txcount: Map.get(attr, :txcount, 0),
       tx_hash: :crypto.strong_rand_bytes(64),
       formed_at_ethereum_height: 1,
       submitted_at_ethereum_height: Map.get(attr, :submitted_at_ethereum_height, 1),