Merge pull request #23 from Groestlcoin/0.9.0

0.9.0

.gitignore

@@ -11,6 +11,7 @@
 *.po
 *.pyc
 .cppcheck-suppress
+.mypy_cache
 TAGS
 tags
 ccan/tools/configurator/configurator
@@ -19,6 +20,7 @@ gen_*
 cli/lightning-cli
 tools/check-bolt
 tools/hsmtool
+tools/lightning-hsmtool
 coverage
 ccan/config.h
 __pycache__
@@ -30,6 +32,8 @@ config.vars
 external/libbacktrace-build/
 external/libbacktrace.a
 external/libbacktrace.la
+external/libbase58/
+external/libbase58.a
 test/test_protocol
 test/test_sphinx
 tests/.pytest.restart
@@ -45,3 +49,4 @@ contrib/pyln-*/build/
 contrib/pyln-*/dist/
 contrib/pyln-*/pyln_*.egg-info/
 plugins/keysend
+release/

.gitmodules

@@ -9,5 +9,6 @@
 	url = https://github.com/ianlancetaylor/libbacktrace.git
 [submodule "external/libwally-core"]
 	path = external/libwally-core
-	url = https://github.com/ElementsProject/libwally-core.git
+	url = https://github.com/niftynei/libwally-core.git
 	ignore = dirty
+	branch = nifty/blessed-branch

.travis.yml

@@ -21,23 +21,27 @@ env:
 #  - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
 #  - VALGRIND=1 ARCH=64 DEVELOPER=0 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=3 SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=64 DEVELOPER=1 COMPILER=clang SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=7 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=8 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=9 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=10 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=11 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
-  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=12 TEST_GROUP_COUNT=12 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=1 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=2 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=3 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=4 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=5 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=6 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=7 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=8 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=9 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=10 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=11 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=12 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=13 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
+  - VALGRIND=1 ARCH=64 DEVELOPER=1 COMPILER=gcc TEST_GROUP=14 TEST_GROUP_COUNT=14 SOURCE_CHECK_ONLY=false
   - VALGRIND=0 ARCH=arm32v7 DEVELOPER=1 TARGET_HOST=arm-linux-gnueabihf
   - VALGRIND=0 ARCH=arm64v8 DEVELOPER=1 TARGET_HOST=aarch64-linux-gnu
 cache:
   directories:
-    - dependencies
+    - external/x86_64-linux-gnu
+    - external/aarch64-linux-gnu
+    - external/arm-linux-gnueabihf
 
 script:
   .travis/build.sh

.travis/build.sh

@@ -10,10 +10,10 @@ export SOURCE_CHECK_ONLY=${SOURCE_CHECK_ONLY:-"false"}
 export COMPAT=${COMPAT:-1}
 export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
 export PYTEST_PAR=2
+export PYTEST_SENTRY_ALWAYS_REPORT=1
 
-# If we're not in developer mode, tests spend a lot of time waiting for gossip!
-# But if we're under valgrind, we can run out of memory!
-if [ "$DEVELOPER" = 0 ] && [ "$VALGRIND" = 0 ]; then
+# Allow up to 4 concurrent tests when not under valgrind, which might run out of memory.
+if [ "$VALGRIND" = 0 ]; then
     PYTEST_PAR=4
 fi
 
@@ -40,6 +40,10 @@ pip3 install --user -U --quiet --progress-bar off \
      -r contrib/pyln-proto/requirements.txt \
      -r contrib/pyln-testing/requirements.txt
 
+pip3 install --user -U --quiet --progress-bar off \
+     pytest-sentry \
+     pytest-rerunfailures
+
 echo "Configuration which is going to be built:"
 echo -en 'travis_fold:start:script.1\\r'
 ./configure CC="$CC"
@@ -48,42 +52,45 @@ echo -en 'travis_fold:end:script.1\\r'
 
 cat > pytest.ini << EOF
 [pytest]
-addopts=-p no:logging --color=no --force-flaky
+addopts=-p no:logging --color=no --reruns=5
 EOF
 
-if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "arm-linux-gnueabihf" ]
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
 then
     export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
     export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
     export AR="$TARGET_HOST"-ar
     export AS="$TARGET_HOST"-as
     export CC="$TARGET_HOST"-gcc
     export CXX="$TARGET_HOST"-g++
     export LD="$TARGET_HOST"-ld
     export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
 
     wget -q https://zlib.net/zlib-1.2.11.tar.gz \
-    && tar xvf zlib-1.2.11.tar.gz \
+    && tar xf zlib-1.2.11.tar.gz \
     && cd zlib-1.2.11 \
     && ./configure --prefix="$QEMU_LD_PREFIX" \
     && make \
-    && sudo make install 
+    && sudo make install
     cd .. && rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
 
     wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip \
-    && unzip sqlite-src-3260000.zip \
+    && unzip -q sqlite-src-3260000.zip \
     && cd sqlite-src-3260000 \
+    && automake --add-missing --force-missing --copy || true \
     && ./configure --disable-tcl --enable-static --disable-readline --disable-threadsafe --disable-load-extension --host="$TARGET_HOST" --prefix="$QEMU_LD_PREFIX" \
     && make \
-    && sudo make install 
+    && sudo make install
     cd .. && rm sqlite-src-3260000.zip && rm -rf sqlite-src-3260000
 
     wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz \
-    && tar xvf gmp-6.1.2.tar.xz \
+    && tar xf gmp-6.1.2.tar.xz \
     && cd gmp-6.1.2 \
     && ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" \
     && make \
-    && sudo make install 
+    && sudo make install
     cd .. && rm gmp-6.1.2.tar.xz && rm -rf gmp-6.1.2
 
     ./configure --enable-static
@@ -92,9 +99,10 @@ then
     make -j3 > /dev/null
     echo -en 'travis_fold:end:script.2\\r'
 
-    echo -en 'travis_fold:start:script.3\\r'
-    make -j$PYTEST_PAR check-units
-    echo -en 'travis_fold:end:script.3\\r'
+    # Tests would need to be wrapped with qemu-<arch>-static
+    #echo -en 'travis_fold:start:script.3\\r'
+    #make -j$PYTEST_PAR check-units
+    #echo -en 'travis_fold:end:script.3\\r'
 elif [ "$SOURCE_CHECK_ONLY" == "false" ]; then
     echo -en 'travis_fold:start:script.2\\r'
     make -j3 > /dev/null

CHANGELOG.md

@@ -5,6 +5,77 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.9.0rc4] - 2020-07-28
+
+### Added
+
+ - plugin: `pay` was rewritten to use the new payment flow. See `legacypay` for old version ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+ - plugin: `pay` will split payments that are failing due to their size into smaller parts, if recipient supports the `basic_mpp` option ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+ - plugin: `pay` will split large payments into parts of approximately 10k sat if the recipient supports the `basic_mpp` option ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+ - plugin: The pay plugin has a new `--disable-mpp` flag that allows opting out of the above two multi-part payment addition.  ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+ - JSON-RPC: new low-level coin selection `fundpsbt` routine. ([3825](https://github.com/ElementsProject/lightning/pull/3825))
+ - JSON-RPC: The `pay` command now uses the new payment flow, the new `legacypay` command can be used to issue payment with the legacy code if required. ([3826](https://github.com/ElementsProject/lightning/pull/3826))
+ - JSON-RPC: The `keysend` command allows sending to a node without requiring an invoice first. ([3792](https://github.com/ElementsProject/lightning/pull/3792))
+ - JSON-RPC: `listfunds` now has a 'scriptpubkey' field. ([3821](https://github.com/ElementsProject/lightning/pull/3821))
+ - docker: Docker build now includes `LIGHTNINGD_NETWORK` ENV variable which defaults to "bitcoin". An user can override this (e.g. by `-e` option in `docker run`) to run docker container in regtest or testnet or any valid argument to `--network`. ([3813](https://github.com/ElementsProject/lightning/pull/3813))
+ - cli: We now install `lightning-hsmtool` for your `hsm_secret` needs. ([3802](https://github.com/ElementsProject/lightning/pull/3802))
+ - JSON-RPC: new call `signpsbt` which will add the wallet's signatures to a provided psbt ([3775](https://github.com/ElementsProject/lightning/pull/3775))
+ - JSON-RPC: new call `sendpsbt` which will finalize and send a signed PSBT ([3775](https://github.com/ElementsProject/lightning/pull/3775))
+ - JSON-RPC: Adds two new rpc methods, `reserveinputs` and `unreserveinputs`, which allow for reserving or unreserving wallet UTXOs ([3775](https://github.com/ElementsProject/lightning/pull/3775))
+ - Python: `pyln.spec.bolt{1,2,4,7}` packages providing python versions of the spec text and defined messages. ([3777](https://github.com/ElementsProject/lightning/pull/3777))
+ - pyln: new module `pyln.proto.message.bolts` ([3733](https://github.com/ElementsProject/lightning/pull/3733))
+ - cli: New `--flat` mode for easy grepping of `lightning-cli` output. ([3722](https://github.com/ElementsProject/lightning/pull/3722))
+ - plugins: new notification type, `coin_movement`, which tracks all fund movements for a node ([3614](https://github.com/ElementsProject/lightning/pull/3614))
+ - plugin: Added a new `commitment_revocation` hook that provides the plugin with penalty transactions for all revoked transactions, e.g., to push them to a watchtower. ([3659](https://github.com/ElementsProject/lightning/pull/3659))
+ - JSON-API: `listchannels` now shows channel `features`. ([3685](https://github.com/ElementsProject/lightning/pull/3685))
+ - plugin: New `invoice_creation` plugin event ([3658](https://github.com/ElementsProject/lightning/pull/3658))
+ - docs: Install documentation now has information about building for Alpine linux ([3660](https://github.com/ElementsProject/lightning/pull/3660))
+ - plugin: Plugins can opt out of having an RPC connection automatically initialized on startup. ([3857](https://github.com/ElementsProject/lightning/pull/3857))
+ - JSON-RPC: `sendonion` has a new optional `bolt11` argument for when it's used to pay an invoice. ([3878](https://github.com/ElementsProject/lightning/pull/3878))
+ - JSON-RPC: `sendonion` has a new optional `msatoshi` that is used to annotate the payment with the amount received by the destination. ([3878](https://github.com/ElementsProject/lightning/pull/3881))
+
+### Changed
+
+ - JSON-RPC: `fundchannel_cancel` no longer requires its undocumented `channel_id` argument after `fundchannel_complete`. ([3787](https://github.com/ElementsProject/lightning/pull/3787))
+ - JSON-RPC: `fundchannel_cancel` will now succeed even when executed while a `fundchannel_complete` is ongoing; in that case, it will be considered as cancelling the funding *after* the `fundchannel_complete` succeeds. ([3778](https://github.com/ElementsProject/lightning/pull/3778))
+ - JSON-RPC: `listfunds` 'outputs' now includes reserved outputs, designated as 'reserved' = true ([3764](https://github.com/ElementsProject/lightning/pull/3764))
+ - JSON-RPC: `txprepare` now prepares transactions whose `nLockTime` is set to the tip blockheight, instead of using 0. `fundchannel` will use `nLockTime` set to the tip blockheight as well. ([3797](https://github.com/ElementsProject/lightning/pull/3797))
+ - build: default compile output is prettier and much less verbose ([3686](https://github.com/ElementsProject/lightning/pull/3686))
+ - config: the `plugin-disable` option now works even if specified before the plugin is found. ([3679](https://github.com/ElementsProject/lightning/pull/3679))
+ - plugins: The `autoclean` plugin is no longer dynamic (you cannot manage it with the `plugin` RPC command anymore). ([3788](https://github.com/ElementsProject/lightning/pull/3788))
+ - plugin: The `paystatus` output changed as a result of the payment flow rework ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+
+### Deprecated
+
+Note: You should always set `allow-deprecated-apis=false` to test for changes.
+
+ - JSON-RPC: the `legacypay` method from the pay plugin will be removed after `pay` proves stable ([3809](https://github.com/ElementsProject/lightning/pull/3809))
+
+### Removed
+
+ - protocol: support for optional fields of the reestablish message are now compulsory. ([3782](https://github.com/ElementsProject/lightning/pull/3782))
+
+### Fixed
+
+ - JSON-RPC: Reject some bad JSON at parsing. ([3761](https://github.com/ElementsProject/lightning/pull/3761))
+ - JSON-RPC: The `feerate` parameters now correctly handle the standardness minimum when passed as `perkb`. ([3772](https://github.com/ElementsProject/lightning/pull/3772))
+ - JSON-RPC: `listtransactions` now displays all txids as little endian ([3741](https://github.com/ElementsProject/lightning/pull/3741))
+ - JSON-RPC: `pay` now respects maxfeepercent, even for tiny amounts. ([3693](https://github.com/ElementsProject/lightning/pull/3693))
+ - JSON-RPC: `withdraw` and `txprepare` `feerate` can be a JSON number. ([3821](https://github.com/ElementsProject/lightning/pull/3821))
+ - bitcoin: `lightningd` now always exits if the Bitcoin backend failed unexpectedly. ([3675](https://github.com/ElementsProject/lightning/pull/3675))
+ - cli: Bash completion on `lightning-cli` now works again ([3719](https://github.com/ElementsProject/lightning/pull/3719))
+ - config: we now take the `--commit-fee` parameter into account. ([3732](https://github.com/ElementsProject/lightning/pull/3732))
+ - db: Fixed a failing assertion if we reconnect to a peer that we had a channel with before, and then attempt to insert the peer into the DB twice. ([3801](https://github.com/ElementsProject/lightning/pull/3801))
+ - hsmtool: Make the password argument optional for `guesstoremote` and `dumpcommitments` sub-commands, as shown in our documentation and help text. ([3822](https://github.com/ElementsProject/lightning/pull/3822))
+ - macOS: Build for macOS Catalina / Apple clang v11.0.3 fixed ([3756](https://github.com/ElementsProject/lightning/pull/3756))
+ - protocol: Fixed a deviation from BOLT#2: if both nodes advertised `option_upfront_shutdown_script` feature: MUST include ... a zero-length `shutdown_scriptpubkey`. ([3816](https://github.com/ElementsProject/lightning/pull/3816))
+ - wumbo: negotiate successfully with Eclair nodes. ([3712](https://github.com/ElementsProject/lightning/pull/3712))
+ - plugin: `bcli` no longer logs a harmless warning about being unable to connect to the JSON-RPC interface. ([3857](https://github.com/ElementsProject/lightning/pull/3857))
+
+### Security
+
+
+
 ## [0.8.2] - 2020-04-30: "A Scalable Ethereum Blockchain"
 
 This release was named by @arowser.
@@ -400,6 +471,8 @@ changes.
 
 ### Security
 
+- Fixes CVE-2019-12998 ([Full Disclosure](https://lists.linuxfoundation.org/pipermail/lightning-dev/2019-September/002174.html)).
+
 ## [0.7.0] - 2019-02-28: "Actually an Altcoin"
 
 This release was named by Mark Beckwith @wythe.
@@ -712,6 +785,7 @@ There predate the BOLT specifications, and are only of vague historic interest:
 6. [0.5.1] - 2016-10-21
 7. [0.5.2] - 2016-11-21: "Bitcoin Savings & Trust Daily Interest II"
 
+[0.9.0rc4]: https://github.com/ElementsProject/lightning/releases/tag/v0.9.0rc4
 [0.8.2]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.2
 [0.8.1]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.1
 [0.8.0]: https://github.com/ElementsProject/lightning/releases/tag/v0.8.0

Dockerfile

@@ -68,12 +68,13 @@ RUN ./configure --prefix=/tmp/lightning_install --enable-static && make -j3 DEVE
 
 FROM debian:stretch-slim as final
 COPY --from=downloader /opt/tini /usr/bin/tini
-RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools \
+RUN apt-get update && apt-get install -y --no-install-recommends socat inotify-tools python3 python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
 ENV LIGHTNINGD_DATA=/root/.lightning
 ENV LIGHTNINGD_RPC_PORT=9835
 ENV LIGHTNINGD_PORT=9735
+ENV LIGHTNINGD_NETWORK=bitcoin
 
 RUN mkdir $LIGHTNINGD_DATA && \
     touch $LIGHTNINGD_DATA/config