[ADP-454] Runtime metrics

[hasufell]

ADP-454

Teaser Prometheus

$ curl http://localhost:13798/metrics 
cardano_wallet_metrics_Stat_rtpriority_int 0
cardano_wallet_metrics_Stat_itrealvalue_int 0
rts_gc_par_max_bytes_copied 0
cardano_wallet_metrics_IO_syscr_int 54
cardano_wallet_metrics_Stat_minflt_int 1441
cardano_wallet_metrics_RTS_maxCompactBytes_B 0
cardano_wallet_metrics_Stat_cminflt_int 0
cardano_wallet_metrics_RTS_gcCopiedBytes_B 47608
rts_gc_num_gcs 221
cardano_wallet_metrics_Stat_enddata_int 29771744
cardano_wallet_metrics_Net_IpExt:OutOctets_B 1377542865
cardano_wallet_metrics_Stat_priority_int 20
cardano_wallet_metrics_IO_wchar_B 83
cardano_wallet_metrics_Stat_rsslim_int -1
cardano_wallet_metrics_RTS_cpuNs_ns 1920962
cardano_wallet_metrics_Stat_envend_int 140726935355228
cardano_wallet_metrics_Stat_pid_int 12809
cardano_wallet_metrics_Stat_ttynr_int 34823
cardano_wallet_metrics_Stat_startcode_int 4194304
rts_gc_mutator_wall_ms 9575
cardano_wallet_metrics_Stat_sigignore_int 0
rts_gc_gc_cpu_ms 105
cardano_wallet_metrics_Stat_cputicks_int 0
rts_gc_par_avg_bytes_copied 0
cardano_wallet_metrics_Stat_sigcatch_int 528390
cardano_wallet_metrics_Stat_majflt_int 0
cardano_wallet_metrics_Stat_cnswap_int 0
cardano_wallet_metrics_Stat_utime_int 0
cardano_wallet_metrics_Stat_blocked_int 0
cardano_wallet_metrics_Stat_cmajflt_int 0
rts_gc_init_cpu_ms 0
cardano_wallet_metrics_Stat_wchan_int 0
cardano_wallet_metrics_Stat_argend_int 140726935350409
cardano_wallet_metrics_IO_wbytes_B 0
cardano_wallet_metrics_Stat_startbrk_int 34394112
cardano_wallet_metrics_Stat_kstkeip_int 0
cardano_wallet_metrics_Stat_numthreads_int 6
cardano_wallet_metrics_Stat_session_int 26622
cardano_wallet_metrics_Stat_tpgid_int 12809
cardano_wallet_metrics_Mem_shared_int 5451
cardano_wallet_metrics_RTS_elapsedNs_ns 301678938
rts_gc_bytes_allocated 230364424
rts_gc_num_bytes_usage_samples 15
cardano_wallet_metrics_Mem_resident_int 6365
rts_gc_current_bytes_slop 227408
cardano_wallet_metrics_Stat_kstkesp_int 0
cardano_wallet_metrics_RTS_maxUsedMemBytes_B 2097152
cardano_wallet_metrics_Stat_argstart_int 140726935349989
cardano_wallet_metrics_Sys_Platform_int 1
rts_gc_max_bytes_slop 313624
cardano_wallet_metrics_IO_rchar_B 11062
rts_gc_par_tot_bytes_copied 0
cardano_wallet_metrics_Stat_flags_int 1077952512
cardano_wallet_metrics_Mem_data_int 11577
cardano_wallet_metrics_Stat_endcode_int 27238048
cardano_wallet_metrics_Mem_text_int 5626
cardano_wallet_metrics_RTS_gcElapsedNs_ns 82336
rts_gc_cumulative_bytes_used 68159136
cardano_wallet_metrics_Stat_rss_int 6365
cardano_wallet_metrics_Mem_size_int 268523351
ekg_server_timestamp_ms 1605554789360
cardano_wallet_metrics_Stat_cstime_int 0
cardano_wallet_metrics_RTS_maxLiveBytes_B 0
cardano_wallet_metrics_Stat_exitcode_int 0
cardano_wallet_metrics_Stat_guesttime_int 0
cardano_wallet_metrics_RTS_gcLiveBytes_B 133704
cardano_wallet_metrics_Stat_ppid_int 26622
rts_gc_gc_wall_ms 105
cardano_wallet_metrics_Stat_envstart_int 140726935350409
cardano_wallet_metrics_Stat_pgrp_int 12809
cardano_wallet_metrics_RTS_gcNum_int 1
rts_gc_mutator_cpu_ms 160
cardano_wallet_metrics_Stat_starttime_int 38375463
rts_gc_peak_megabytes_allocated 17
cardano_wallet_metrics_Stat_policy_int 0
cardano_wallet_metrics_Stat_startdata_int 27245632
cardano_wallet_metrics_IO_cxwbytes_B 0
cardano_wallet_metrics_Stat_vsize_int 1099871645696
cardano_wallet_metrics_RTS_maxSlopBytes_B 0
rts_gc_init_wall_ms 0
cardano_wallet_metrics_Stat_cutime_int 0
cardano_wallet_metrics_Net_IpExt:InOctets_B 12129589852
rts_gc_max_bytes_used 7362280
cardano_wallet_metrics_Stat_nswap_int 0
rts_gc_bytes_copied 27291728
cardano_wallet_metrics_Stat_blkio_int 0
cardano_wallet_metrics_IO_rbytes_B 0
cardano_wallet_metrics_Stat_startstack_int 140726935348176
cardano_wallet_metrics_RTS_maxLargeBytes_B 0
cardano_wallet_metrics_Stat_cguesttime_int 0
cardano_wallet_metrics_RTS_gcMajorNum_int 0
cardano_wallet_metrics_Sys_Pid_int 12809
rts_gc_wall_ms 9681
cardano_wallet_metrics_RTS_bytesAllocated_B 1129192
rts_gc_current_bytes_used 5765040
cardano_wallet_metrics_RTS_gcCpuNs_ns 82551
cardano_wallet_metrics_Stat_stime_int 0
cardano_wallet_metrics_Stat_signal_int 0
rts_gc_cpu_ms 266
cardano_wallet_metrics_Stat_exitsignal_int 17
cardano_wallet_metrics_Stat_processor_int 5
cardano_wallet_metrics_RTS_liveBytes_B 0
cardano_wallet_metrics_Stat_nice_int 0
cardano_wallet_metrics_IO_syscw_int 11

Teaser EKG

curl -H "Accept: application/json" http://localhost:13788/ | jq .

{
  "iohk-monitoring version": {
    "type": "l",
    "val": "0.1.10.1"
  },
  "ekg": {
    "server_timestamp_ms": {
      "type": "c",
      "val": 1605553522731
    }
  },
  "rts": {
    "gc": {
      "gc_cpu_ms": {
        "type": "c",
        "val": 124
      },
      "mutator_wall_ms": {
        "type": "c",
        "val": 12178
      },
      "mutator_cpu_ms": {
        "type": "c",
        "val": 598
      },
      "gc_wall_ms": {
        "type": "c",
        "val": 118
      },
      "wall_ms": {
        "type": "c",
        "val": 12296
      },
      "bytes_copied": {
        "type": "c",
        "val": 60092216
      },
      "init_wall_ms": {
        "type": "c",
        "val": 0
      },
      "init_cpu_ms": {
        "type": "c",
        "val": 0
      },
      "max_bytes_used": {
        "type": "g",
        "val": 26806536
      },
      "max_bytes_slop": {
        "type": "g",
        "val": 1001208
      },
      "num_bytes_usage_samples": {
        "type": "c",
        "val": 12
      },
      "peak_megabytes_allocated": {
        "type": "g",
        "val": 40
      },
      "cpu_ms": {
        "type": "c",
        "val": 723
      },
      "current_bytes_used": {
        "type": "g",
        "val": 23334656
      },
      "bytes_allocated": {
        "type": "c",
        "val": 529295488
      },
      "par_max_bytes_copied": {
        "type": "g",
        "val": 0
      },
      "current_bytes_slop": {
        "type": "g",
        "val": 758016
      },
      "cumulative_bytes_used": {
        "type": "c",
        "val": 122968880
      },
      "num_gcs": {
        "type": "c",
        "val": 503
      },
      "par_tot_bytes_copied": {
        "type": "g",
        "val": 0
      },
      "par_avg_bytes_copied": {
        "type": "g",
        "val": 0
      }
    }
  },
  "cardano-wallet": {
    "wallet-engine": {
      "type": "l",
      "val": "\"\\\"9914f38a: local tip: 46f10b60<-[b40aa5a2-11184306#2045091]\\\"\""
    },
    "pools-engine": {
      "type": "l",
      "val": "\"\\\"Applying blocks [11184306 ... 11184306]\\\"\""
    },
    "metrics": {
      "Net": {
        "IpExt:InOctets": {
          "B": {
            "type": "g",
            "val": 12110694532
          }
        },
        "IpExt:OutOctets": {
          "B": {
            "type": "g",
            "val": 1374876709
          }
        }
      },
      "Mem": {
        "size": {
          "int": {
            "type": "g",
            "val": 268523646
          }
        },
        "text": {
          "int": {
            "type": "g",
            "val": 5901
          }
        },
        "data": {
          "int": {
            "type": "g",
            "val": 12365
          }
        },
        "resident": {
          "int": {
            "type": "g",
            "val": 7752
          }
        },
        "shared": {
          "int": {
            "type": "g",
            "val": 6347
          }
        }
      },
      "IO": {
        "rchar": {
          "B": {
            "type": "g",
            "val": 11062
          }
        },
        "syscw": {
          "int": {
            "type": "g",
            "val": 21
          }
        },
        "rbytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "wchar": {
          "B": {
            "type": "g",
            "val": 93
          }
        },
        "cxwbytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "syscr": {
          "int": {
            "type": "g",
            "val": 54
          }
        },
        "wbytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        }
      },
      "Sys": {
        "Platform": {
          "int": {
            "type": "g",
            "val": 1
          }
        },
        "Pid": {
          "int": {
            "type": "g",
            "val": 7172
          }
        }
      },
      "Stat": {
        "minflt": {
          "int": {
            "type": "g",
            "val": 2009
          }
        },
        "rtpriority": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "flags": {
          "int": {
            "type": "g",
            "val": 1077952512
          }
        },
        "endcode": {
          "int": {
            "type": "g",
            "val": 28361072
          }
        },
        "envend": {
          "int": {
            "type": "g",
            "val": 140736416722780
          }
        },
        "rsslim": {
          "int": {
            "type": "g",
            "val": -1
          }
        },
        "rss": {
          "int": {
            "type": "g",
            "val": 7752
          }
        },
        "priority": {
          "int": {
            "type": "g",
            "val": 20
          }
        },
        "guesttime": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "starttime": {
          "int": {
            "type": "g",
            "val": 38248539
          }
        },
        "cnswap": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "sigcatch": {
          "int": {
            "type": "g",
            "val": 544774
          }
        },
        "startdata": {
          "int": {
            "type": "g",
            "val": 28367904
          }
        },
        "majflt": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "cputicks": {
          "int": {
            "type": "g",
            "val": 2
          }
        },
        "numthreads": {
          "int": {
            "type": "g",
            "val": 6
          }
        },
        "blkio": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "nswap": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "vsize": {
          "int": {
            "type": "g",
            "val": 1099872854016
          }
        },
        "argend": {
          "int": {
            "type": "g",
            "val": 140736416717961
          }
        },
        "startbrk": {
          "int": {
            "type": "g",
            "val": 53985280
          }
        },
        "stime": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "processor": {
          "int": {
            "type": "g",
            "val": 7
          }
        },
        "argstart": {
          "int": {
            "type": "g",
            "val": 140736416717541
          }
        },
        "itrealvalue": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "cminflt": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "enddata": {
          "int": {
            "type": "g",
            "val": 30976544
          }
        },
        "ppid": {
          "int": {
            "type": "g",
            "val": 26622
          }
        },
        "sigignore": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "startcode": {
          "int": {
            "type": "g",
            "val": 4194304
          }
        },
        "envstart": {
          "int": {
            "type": "g",
            "val": 140736416717961
          }
        },
        "cstime": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "pgrp": {
          "int": {
            "type": "g",
            "val": 7172
          }
        },
        "exitcode": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "pid": {
          "int": {
            "type": "g",
            "val": 7172
          }
        },
        "ttynr": {
          "int": {
            "type": "g",
            "val": 34823
          }
        },
        "policy": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "utime": {
          "int": {
            "type": "g",
            "val": 2
          }
        },
        "blocked": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "cmajflt": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "cutime": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "wchan": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "kstkeip": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "startstack": {
          "int": {
            "type": "g",
            "val": 140736416715888
          }
        },
        "session": {
          "int": {
            "type": "g",
            "val": 26622
          }
        },
        "exitsignal": {
          "int": {
            "type": "g",
            "val": 17
          }
        },
        "tpgid": {
          "int": {
            "type": "g",
            "val": 7172
          }
        },
        "cguesttime": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "signal": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "kstkesp": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "nice": {
          "int": {
            "type": "g",
            "val": 0
          }
        }
      },
      "RTS": {
        "maxUsedMemBytes": {
          "B": {
            "type": "g",
            "val": 2097152
          }
        },
        "gcCpuNs": {
          "ns": {
            "type": "g",
            "val": 128700
          }
        },
        "maxCompactBytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "maxLargeBytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "bytesAllocated": {
          "B": {
            "type": "g",
            "val": 1129192
          }
        },
        "cpuNs": {
          "ns": {
            "type": "g",
            "val": 3184072
          }
        },
        "gcMajorNum": {
          "int": {
            "type": "g",
            "val": 0
          }
        },
        "gcCopiedBytes": {
          "B": {
            "type": "g",
            "val": 47608
          }
        },
        "elapsedNs": {
          "ns": {
            "type": "g",
            "val": 302668800
          }
        },
        "liveBytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "gcNum": {
          "int": {
            "type": "g",
            "val": 1
          }
        },
        "maxSlopBytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "gcElapsedNs": {
          "ns": {
            "type": "g",
            "val": 128435
          }
        },
        "maxLiveBytes": {
          "B": {
            "type": "g",
            "val": 0
          }
        },
        "gcLiveBytes": {
          "B": {
            "type": "g",
            "val": 133704
          }
        }
      }
    },
    "application": {
      "type": "l",
      "val": "\"\\\"Node is Haskell Node on testnet (1097911063).\\\"\""
    },
    "main": {
      "type": "l",
      "val": "\"\\\"Wallet backend server listening on 127.0.0.1:8090\\\"\""
    },
    "ntp-client": {
      "type": "l",
      "val": "\"\\\"local clock is drifting by -3214\\206\\188s\\\"\""
    },
    "network": {
      "type": "l",
      "val": "\"\\\"Query GetUpdateInterfaceState took 0.001308712s\\\"\""
    },
    "pools-db": {
      "type": "l",
      "val": "\"\\\"Will open db at ../cardano-node/run/wallet-state-test/stake-pools.sqlite\\\"\""
    },
    "wallet-db": {
      "type": "l",
      "val": "\"\\\"rnd_state_address table does not contain required field 'account_ix'. Adding this field with a default value of 2147483648.\\\"\""
    }
  }
}

[hasufell]

A general remark:

The EKG includes log messages, I believe the last of every tracer, or so. Prometheus doesn't seem to have picked up those or any metrics of internal warnings/errors. I don't really know if there's a way to add that.

[hasufell]

@disassembler can you check the sample metrics and whether they satisfy the acceptance criteria?

[Anviking]

lgtm, modulo

1. Should probably allow the port to be configurable, as you said.
2. I saw

[cardano-wallet.#ekgview.realizeFrom:Error:4] [2020-11-17 11:01:00.46 UTC] EKGView backend disabled due to initialisation error: EKGServerStartupError "Network.Socket.bind: resource busy (Address already in use)"

running integration tests. Maybe some node-wallet port conflict.

Also, how do you get the Prometheus output? Some (at least partial) steps for how to reproduce would be nice in the PR description. For EKG it was simple as I just had to visit http://127.0.0.1:13788, but not sure about Prometheus.

[Anviking]

What does this do?

This does not affect the integration tests. And you don't have a similar call in the integration tests. Do both work as intended?

[hasufell]

Yes, the integration tests don't need EKG/prometheus logging. That was the point of this change.

[piotr-iohk]

I think a description of how to leverege this feature (enable/disable, configure ports etc.) would be really handy. Perhaps a wiki page?

Also AC says: "If no port is defined explicitely, there should be no metrics exposed." I think this is not the case and monitoring is always available on default ports even if custom ports are not provided.

[hasufell]

Also AC says: "If no port is defined explicitely, there should be no metrics exposed." I think this is not the case and monitoring is always available on default ports even if custom ports are not provided.

Good catch. Will adjust this.

[Anviking]

Yes, the integration tests don’t need EKG/prometheus logging. That was the point of this change.

You made it print e.g. EKG url: 127.0.0.1:2000 in the integration startup, but that url doesn't actually lead anywhere. Let's allow EKG in integration tests too, particularly now when they're off by default?

I don't understand your concern here. The DB tests don't initialise the ekg plugin, but they are free to do so, if they wanted (e.g. to test logging, if that would be a thing).

My concern is that unexpected things will happen when we build things on top of initTracer. without expecting it to use EKG. Perhaps "Host or port value for EKB metrics invalid. Will be disabled." will be printed where someone doesn't expect it. Maybe fine, but wouldn't have minded something else 🤔

[hasufell]

You made it print e.g. EKG url: 127.0.0.1:2000 in the integration startup, but that url doesn't actually lead anywhere. Let's allow EKG in integration tests too, particularly now when they're off by default?

That's not really the case anymore. The URLs are not set when the env variables are not set and the tracer message now shows: none

My concern is that unexpected things will happen when we build things on top of initTracer.

I don't really see what. But I can start EKG/prometheus for the tests as well.

[iohk-bors]

Build failed:

• buildkite/cardano-wallet

  PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1283)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}

finally

Failures:
--
  |  
  | src/Test/Integration/Scenario/API/Shelley/Migrations.hs:214:15:
  | 1) API Specifications, SHELLEY_MIGRATIONS, SHELLEY_MIGRATE_01_big_wallet -  migrate a big wallet requiring more than one tx
  | expected: Status {statusCode = 202, statusMessage = "Accepted"}
  | but got: Status {statusCode = 500, statusMessage = "Internal Server Error"}
  |  
  | from the following response: Left (DecodeFailure "Something went wrong")
  |  
  | While verifying (Status {statusCode = 500, statusMessage = "Internal Server Error"},Left (DecodeFailure "Something went wrong"))
  |  
  | To rerun use: --match "/API Specifications/SHELLEY_MIGRATIONS/SHELLEY_MIGRATE_01_big_wallet -  migrate a big wallet requiring more than one tx/"
  |  
  | Randomized with seed 1952397155

#2372

[hasufell]

bors r+

[iohk-bors]

Build failed:

• buildkite/cardano-wallet

First

 PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1282)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}

finally

src/Test/Integration/Framework/DSL.hs:1733:7:
--
  | 1) API Specifications, SHELLEY_MIGRATIONS, SHELLEY_MIGRATE_01_big_wallet -  migrate a big wallet requiring more than one tx
  | expected a successful response but got an error: DecodeFailure "Something went wrong"
  | While verifying (Status {statusCode = 500, statusMessage = "Internal Server Error"},Left (DecodeFailure "Something went wrong"))
  |  
  | To rerun use: --match "/API Specifications/SHELLEY_MIGRATIONS/SHELLEY_MIGRATE_01_big_wallet -  migrate a big wallet requiring more than one tx/"
  |  
  | src/Test/Integration/Scenario/API/Byron/Migrations.hs:290:27:
  | 2) API Specifications, BYRON_MIGRATIONS, BYRON_MIGRATE_01 -  migrate a big wallet requiring more than one tx
  | While verifying (Status {statusCode = 200, statusMessage = "OK"},Right (ApiWallet {id = ApiT {getApiT = WalletId {getWalletId = d8b145bfd3f291b779f282e981c41ed07f573346}}, addressPoolGap = ApiT {getApiT = AddressPoolGap {getAddressPoolGap = 20}}, balance = ApiT {getApiT = WalletBalance {available = Quantity {getQuantity = 639995363268}, total = Quantity {getQuantity = 639995363268}, reward = Quantity {getQuantity = 0}}}, delegation = ApiWalletDelegation {active = ApiWalletDelegationNext {status = NotDelegating, target = Nothing, changesAt = Nothing}, next = []}, name = ApiT {getApiT = WalletName {getWalletName = "Empty Wallet"}}, passphrase = Just (ApiWalletPassphraseInfo {lastUpdatedAt = 2020-12-02 21:32:48.372147015 UTC}), state = ApiT {getApiT = Syncing (Quantity {getQuantity = Percentage {getPercentage = 3601 % 5000}})}, tip = ApiBlockReference {absoluteSlotNumber = ApiT {getApiT = SlotNo 1282}, slotId = ApiSlotId {epochNumber = ApiT {getApiT = EpochNo {unEpochNo = 7}}, slotNumber = ApiT {getApiT = SlotInEpoch {unSlotInEpoch = 2}}}, time = 2020-12-02 21:33:32.4 UTC, block = ApiBlockInfo {height = Quantity {getQuantity = 550}}}}))
  | Waited longer than 90s to resolve action: "wallet balance = expectedBalance".
  | expected: Quantity {getQuantity = 999992866900}
  | but got: Quantity {getQuantity = 639995363268}
  |  
  | To rerun use: --match "/API Specifications/BYRON_MIGRATIONS/BYRON_MIGRATE_01 -  migrate a big wallet requiring more than one tx/"

#2372

[hasufell]

Bors r+

[iohk-bors]

Build failed:

• buildkite/cardano-wallet

PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1336)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}
--
  | PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1318)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}
  | PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1336)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}
  | PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-6nOs81F0YiqAvC2KVKdBOx", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-3B1qWkkZYE85MsiUh01uEA", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 1318)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 260s, boundSlot = SlotNo 1280, boundEpoch = EpochNo 7}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}

finally

1) API Specifications, BYRON_MIGRATIONS, BYRON_MIGRATE_01 -  migrate a big wallet requiring more than one tx
--
  | While verifying (Status {statusCode = 200, statusMessage = "OK"},Right (ApiWallet {id = ApiT {getApiT = WalletId {getWalletId = 317fe33f3f9945aca5bd6e3f1c4090df817cee6a}}, addressPoolGap = ApiT {getApiT = AddressPoolGap {getAddressPoolGap = 20}}, balance = ApiT {getApiT = WalletBalance {available = Quantity {getQuantity = 599995739261}, total = Quantity {getQuantity = 599995739261}, reward = Quantity {getQuantity = 0}}}, delegation = ApiWalletDelegation {active = ApiWalletDelegationNext {status = NotDelegating, target = Nothing, changesAt = Nothing}, next = []}, name = ApiT {getApiT = WalletName {getWalletName = "Empty Wallet"}}, passphrase = Just (ApiWalletPassphraseInfo {lastUpdatedAt = 2020-12-03 00:25:50.856414966 UTC}), state = ApiT {getApiT = Syncing (Quantity {getQuantity = Percentage {getPercentage = 1809 % 2500}})}, tip = ApiBlockReference {absoluteSlotNumber = ApiT {getApiT = SlotNo 1336}, slotId = ApiSlotId {epochNumber = ApiT {getApiT = EpochNo {unEpochNo = 7}}, slotNumber = ApiT {getApiT = SlotInEpoch {unSlotInEpoch = 56}}}, time = 2020-12-03 00:26:42.2 UTC, block = ApiBlockInfo {height = Quantity {getQuantity = 571}}}}))
  | Waited longer than 90s to resolve action: "wallet balance = expectedBalance".
  | expected: Quantity {getQuantity = 999992866900}
  | but got: Quantity {getQuantity = 599995739261}
  |  
  | To rerun use: --match "/API Specifications/BYRON_MIGRATIONS/BYRON_MIGRATE_01 -  migrate a big wallet requiring more than one tx/"
  |  
  | Randomized with seed 538408917
  |  
  | Finished in 632.0300 seconds
  | 709 examples, 1 failure, 9 pending

#2372

[hasufell]

bors r+

[iohk-bors]

Build failed:

• ci/hydra-build:required

PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-H6qaNabY1R36ZYg4ITqJo2", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-7zDKmakj5C46Wuy75liU0K", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-7zDKmakj5C46Wuy75liU0K", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 882)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 180s, boundSlot = SlotNo 880, boundEpoch = EpochNo 5}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}
PastHorizon {pastHorizonCallStack = [("runQuery",SrcLoc {srcLocPackage = "ouroboros-consensus-0.1.0.0-H6qaNabY1R36ZYg4ITqJo2", srcLocModule = "Ouroboros.Consensus.HardFork.History.Qry", srcLocFile = "src/Ouroboros/Consensus/HardFork/History/Qry.hs", srcLocStartLine = 426, srcLocStartCol = 44, srcLocEndLine = 426, srcLocEndCol = 64}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-7zDKmakj5C46Wuy75liU0K", srcLocModule = "Cardano.Wallet.Primitive.Slotting", srcLocFile = "src/Cardano/Wallet/Primitive/Slotting.hs", srcLocStartLine = 368, srcLocStartCol = 15, srcLocEndLine = 368, srcLocEndCol = 34}),("interpretQuery",SrcLoc {srcLocPackage = "cardano-wallet-core-2020.11.26-7zDKmakj5C46Wuy75liU0K", srcLocModule = "Cardano.Wallet.DB.Sqlite", srcLocFile = "src/Cardano/Wallet/DB/Sqlite.hs", srcLocStartLine = 856, srcLocStartCol = 25, srcLocEndLine = 856, srcLocEndCol = 82})], pastHorizonExpression = Some (ELet (ERelSlotToEpoch (EAbsToRelSlot (ELit (SlotNo 885)))) (\x0 -> EPair (ERelToAbsEpoch (EVar x0)) (ESnd (EVar x0)))), pastHorizonSummary = [EraSummary {eraStart = Bound {boundTime = RelativeTime 0s, boundSlot = SlotNo 0, boundEpoch = EpochNo 0}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}), eraParams = EraParams {eraEpochSize = EpochSize 80, eraSlotLength = SlotLength 0.25s, eraSafeZone = StandardSafeZone 16 NoLowerBound}},EraSummary {eraStart = Bound {boundTime = RelativeTime 20s, boundSlot = SlotNo 80, boundEpoch = EpochNo 1}, eraEnd = EraEnd (Bound {boundTime = RelativeTime 180s, boundSlot = SlotNo 880, boundEpoch = EpochNo 5}), eraParams = EraParams {eraEpochSize = EpochSize 200, eraSlotLength = SlotLength 0.2s, eraSafeZone = StandardSafeZone 60 NoLowerBound}}]}
    SHELLEY_MIGRATE_01_big_wallet -  migrate a big wallet requiring more than one tx FAILED [1]

#2372

[piotr-iohk]

@hasufell I'm trying on testnet and have 2 questions:

1. EKG GC and memory statistics

Starting like this:

CARDANO_WALLET_PROMETHEUS_HOST=6666 \
CARDANO_WALLET_EKG_PORT=7777 \
cardano-wallet serve --port 8090 \
  --node-socket ../relay1/node.socket \
  --testnet testnet-byron-genesis.json  \
  --database ./wallet-db 

I can see EKG metrics -> http://localhost:7777.
I can see Prometheus:

 $ curl http://localhost:6666/metrics 
cardano_wallet_metrics_Stat_rtpriority_int 0
...

But for EKG, there does not seem to be GC and memory statistics showing up/updating:

(Not sure if this is OK, or maybe sort of a config issue? Asking because I am able to see those metrics for the cardano-node when EKG is set up for it)

2. Binding to something other than localhost

There are those env variables:

CARDANO_WALLET_EKG_HOST
CARDANO_WALLET_PROMETHEUS_HOST

So I wanted to bind metrics to 0.0.0.0 or 127.0.0.1 but with that my wallet doesn't want to connect to the node 🤷‍♂️

CARDANO_WALLET_EKG_PORT=6666 \
CARDANO_WALLET_EKG_HOST=0.0.0.0 \
CARDANO_WALLET_PROMETHEUS_PORT=7777 \
CARDANO_WALLET_PROMETHEUS_HOST=0.0.0.0 \
cardano-wallet serve --port 8090 \
  --node-socket /path_to/cardano-node.socket \
  --testnet testnet-byron-genesis.json \
  --database ./wallet-db
...
[cardano-wallet.network:Notice:14] [2020-12-03 12:39:12.28 UTC] Couldn't connect to node (x2). Retrying in a bit...
[cardano-wallet.network:Notice:16] [2020-12-03 12:39:12.28 UTC] Couldn't connect to node (x2). Retrying in a bit...
[cardano-wallet.network:Notice:25] [2020-12-03 12:39:12.31 UTC] Couldn't connect to node (x2). Retrying in a bit...
[cardano-wallet.network:Warning:16] [2020-12-03 12:39:12.53 UTC] Couldn't connect to node (x3). Retrying in a bit..

Basically it seems that if anything is put in the *_HOST env variable the result is 👆 .

[hasufell]

@piotr-iohk weird... I tried it and I cannot reproduce the issue with hostname

[KtorZ]

Note that EKG requires the -T runtime flag to work with monitoring https://hackage.haskell.org/package/ekg-0.4.0.15/docs/System-Remote-Monitoring.html

[hasufell]

bors r+

[iohk-bors]

Build failed:

• ci/hydra-build:required

 test/unit/Cardano/Wallet/Shelley/TransactionSpec.hs:393:9:
  1) Cardano.Wallet.Shelley.Transaction, estimateMaxNumberOfInputs for IcarusKey, bigger size  ==> more inputs
       Timeout (after 864 tests):
         Quantity {getQuantity = 297}
         224

  To rerun use: --match "/Cardano.Wallet.Shelley.Transaction/estimateMaxNumberOfInputs for IcarusKey/bigger size  ==> more inputs/"

#2152

[Anviking]

bors r+

[iohk-bors]

Build failed:

• ci/hydra-build:required

[piotr-iohk]

@piotr-iohk weird... I tried it and I cannot reproduce the issue with hostname

Must have been something on my end. I cannot reproduce it either now.

[hasufell]

bors r+

[iohk-bors]

Build failed:

• ci/hydra-build:required

[hasufell]

bors r+

[iohk-bors]

Build succeeded:

• ci/hydra-build:required
• buildkite/cardano-wallet