Rebase forked master

.circleci/config.yml

@@ -0,0 +1,144 @@
+version: 2
+jobs:
+  # publish jobs require $DOCKERHUB_REPO, $DOCKERHUB_USER, $DOCKERHUB_PASS defined
+  amd64:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-amd64 -f Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-amd64
+  arm32v7:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            # Make sure the builder is copy the arm emulator
+            sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
+            sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm32v7.Dockerfile"
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 -f contrib/linuxarm32v7.Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm32v7
+  arm64v8:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            # Make sure the builder is copy the arm emulator
+            sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
+            sed -i -e 's/#EnableQEMU //g' "contrib/linuxarm64v8.Dockerfile"
+            #
+            sudo docker build --pull -t $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 -f contrib/linuxarm64v8.Dockerfile .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
+  publish_docker_dev:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=false" -t "$DOCKERHUB_REPO:$LATEST_TAG-dev" .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-dev
+  publish_docker_bench:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run:
+          command: |
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            #
+            sudo docker build --build-arg "DEVELOPER=1" --build-arg "TRACE_TOOLS=true" -t "$DOCKERHUB_REPO:$LATEST_TAG-bench" .
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            sudo docker push $DOCKERHUB_REPO:$LATEST_TAG-bench
+  multiarch:
+    machine:
+      enabled: true
+      image: circleci/classic:201808-01
+    steps:
+      - run:
+          command: |
+            # Turn on Experimental features
+            sudo mkdir $HOME/.docker
+            sudo sh -c 'echo "{ \"experimental\": \"enabled\" }" >> $HOME/.docker/config.json'
+            #
+            sudo docker login --username=$DOCKERHUB_USER --password=$DOCKERHUB_PASS
+            #
+            LATEST_TAG=${CIRCLE_TAG:8} #trim "basedon-" from tag
+            sudo docker manifest create --amend $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 $DOCKERHUB_REPO:$LATEST_TAG-arm64v8
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-amd64 --os linux --arch amd64
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm32v7 --os linux --arch arm --variant v7
+            sudo docker manifest annotate $DOCKERHUB_REPO:$LATEST_TAG $DOCKERHUB_REPO:$LATEST_TAG-arm64v8 --os linux --arch arm64 --variant v8
+            sudo docker manifest push $DOCKERHUB_REPO:$LATEST_TAG -p
+
+workflows:
+  version: 2
+  publish:
+    jobs:
+      - amd64:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - arm32v7:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - arm64v8:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - publish_docker_dev:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - publish_docker_bench:
+          filters:
+            # ignore any commit on any branch by default
+            branches:
+              ignore: /.*/
+            # only act on version tags
+            tags:
+              only: /basedon-.+/
+      - multiarch:
+          requires:
+            - amd64
+            - arm32v7
+            - arm64v8
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /basedon-.+/
+

.dockerignore

@@ -0,0 +1,3 @@
+Dockerfile
+linuxarm32v7.Dockerfile
+.circleci/config.yml

.editorconfig

@@ -16,6 +16,3 @@ indent_size = 8
 indent_style = space
 indent_size = 4
 
-[.travis.yml]
-indent_style = space
-indent_size = 2

.gitattributes

@@ -7,8 +7,14 @@ configure text eol=lf
 # The following files are generated and should not be shown in the
 # diffs by default on Github.
 doc/lightning*.7 linguist-generated=true
-wallet/db_*_sqlgen.c linguist-generated=true
-wallet/statements_gettextgen.po linguist-generated=true
+db_*_sqlgen.c linguist-generated=true
+statements_gettextgen.po linguist-generated=true
 *_wiregen.? linguist-generated=true
 *_printgen.? linguist-generated=true
 
+# The following are marked as binary and generated since they can be
+# easily overwritten and don't need deep review.
+cln-grpc/proto/node.proto -text -diff linguist-generated=true
+cln-grpc/src/convert.rs -text -diff linguist-generated=true
+cln-rpc/src/model.rs -text -diff linguist-generated=true
+contrib/pyln-testing/pyln/testing/node_pb2.py -text -diff linguist-generated=true

.github/CODEOWNERS

@@ -4,17 +4,20 @@
 # also can optionally require approval from a code owner before the
 # author can merge a pull request in the repository.
 
-wallet/                 @cdecker
-*.py                    @cdecker
+cln-grpc/               @cdecker
+cln-rpc/                @cdecker
+plugins/src/            @cdecker
+plugins/grpc-plugin/    @cdecker
+contrib/reprobuild/     @cdecker
+contrib/msggen/         @cdecker
+contrib/pyln-client/    @cdecker
+contrib/pyln-testing/   @cdecker
+# Needed to ensure hsmd wire compatibility between releases
+hsmd/hsmd_wire.csv      @cdecker
 
-wallet/invoices.*       @ZmnSCPxj
+wallet/invoices.*          @ZmnSCPxj
 plugins/multifundchannel.c @ZmnSCPxj
-doc/BACKUP.md           @ZmnSCPxj @cdecker
-
-common/param.*          @wythe
-common/json.*           @wythe
-common/json_tok.*       @wythe
-common/wallet_tx.*      @wythe
+doc/BACKUP.md              @ZmnSCPxj
 
 # See https://help.github.com/articles/about-codeowners/ for more
 # information

.github/scripts/build.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
-
+set -e
 echo "Running in $(pwd)"
 export ARCH=${ARCH:-64}
-export BOLTDIR=lightning-rfc
+export BOLTDIR=bolts
 export CC=${COMPILER:-gcc}
 export COMPAT=${COMPAT:-1}
 export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
@@ -21,38 +21,22 @@ export VALGRIND=${VALGRIND:-0}
 export FUZZING=${FUZZING:-0}
 export LIGHTNINGD_POSTGRES_NO_VACUUM=1
 
-pip3 install --user -U \
-     -r requirements.lock
-
-timeout 60 pip3 install --user \
-     --use-feature=in-tree-build \
-     ./contrib/pyln-client \
-     ./contrib/pyln-proto \
-     ./contrib/pyln-testing
+# Fail if any commands fail.
+set -e
 
-# Install utilities that aren't dependencies, but make
-# running tests easier/feasible on CI (and pytest which
-# keeps breaking the rerunfailures plugin).
-pip3 install --user \
-     blinker \
-     flake8 \
-     flaky \
-     mako \
-     pytest-sentry \
-     pytest-test-groups==1.0.3 \
-     pytest-custom-exit-code==0.3.0 \
-     pytest-timeout \
-     pytest-json-report
+pip3 install --user poetry
+poetry config virtualenvs.create false --local
+poetry install
 
-git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
+git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
 git submodule update --init --recursive
 
 ./configure CC="$CC"
 cat config.vars
 
 cat << EOF > pytest.ini
 [pytest]
-addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
 markers =
     slow_test: marks tests as slow (deselect with '-m "not slow_test"')
 EOF
@@ -70,14 +54,14 @@ then
     export STRIP="$TARGET_HOST"-strip
     export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
 
-    wget -q https://zlib.net/zlib-1.2.11.tar.gz
-    tar xf zlib-1.2.11.tar.gz
-    cd zlib-1.2.11 || exit 1
+    wget -q https://zlib.net/zlib-1.2.12.tar.gz
+    tar xf zlib-1.2.12.tar.gz
+    cd zlib-1.2.12 || exit 1
     ./configure --prefix="$QEMU_LD_PREFIX"
     make
     sudo make install
     cd .. || exit 1
-    rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
+    rm zlib-1.2.12.tar.gz && rm -rf zlib-1.2.12
 
     wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
     unzip -q sqlite-src-3260000.zip
@@ -108,7 +92,7 @@ then
 
     ./configure CC="$TARGET_HOST-gcc" --enable-static
 
-    make -j32 CC="$TARGET_HOST-gcc" > /dev/null
+    make -s -j32 CC="$TARGET_HOST-gcc"
 else
     eatmydata make -j32
     # shellcheck disable=SC2086

.github/scripts/setup.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
-
+set -e
 export DEBIAN_FRONTEND=noninteractive
 export BITCOIN_VERSION=0.20.1
 export ELEMENTS_VERSION=0.18.1.8
+export RUST_VERSION=stable
+
 sudo useradd -ms /bin/bash tester
 sudo apt-get update -qq
 
@@ -66,3 +68,8 @@ sudo chmod 0440 /etc/sudoers.d/tester
        elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
        elements-$ELEMENTS_VERSION
 )
+
+if [ "$RUST" == "1" ]; then
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
+      -y --default-toolchain ${RUST_VERSION}
+fi

.github/workflows/bsd.yml

@@ -1,10 +1,14 @@
 name: FreeBSD Test
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
 
 jobs:
   testfreebsd:
-    runs-on: macos-latest
+    runs-on: macos-10.15
     name: Build and test on FreeBSD
     env:
       DEVELOPER: 1
@@ -35,6 +39,7 @@ jobs:
             bash \
             gettext \
             sqlite3 \
+            lowdown \
             curl
 
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1
@@ -59,21 +64,22 @@ jobs:
             blinker \
             flake8 \
             mako \
-            mrkd \
             pytest-sentry \
             pytest-test-groups==1.0.3 \
             pytest-custom-exit-code==0.3.0 \
             pytest-json-report
 
-          git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
+          git clone https://github.com/lightning/bolts.git ../bolts
+          # fatal: unsafe repository ('/Users/runner/work/lightning/lightning' is owned by someone else)
+          git config --global --add safe.directory `pwd`
           git submodule update --init --recursive
 
           ./configure CC="$CC"
           cat config.vars
 
           cat << EOF > pytest.ini
           [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
           markers =
               slow_test: marks tests as slow (deselect with '-m "not slow_test"')
           EOF