From d61f12de7d011e010d8e0f7664c591cd85ae0f92 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 11 Jan 2022 18:05:53 +1030 Subject: [PATCH] DEBUG --- .github/workflows/bsd.yml | 86 --------------------- .github/workflows/ci.yaml | 143 ----------------------------------- .github/workflows/macos.yaml | 102 ------------------------- .github/workflows/pypi.yml | 72 ------------------ Makefile | 2 +- connectd/connectd.c | 13 +++- connectd/multiplex.c | 30 +++++++- 7 files changed, 41 insertions(+), 407 deletions(-) delete mode 100644 .github/workflows/bsd.yml delete mode 100644 .github/workflows/macos.yaml delete mode 100644 .github/workflows/pypi.yml diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml deleted file mode 100644 index bea605fef489..000000000000 --- a/.github/workflows/bsd.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: FreeBSD Test - -on: [push, pull_request] - -jobs: - testfreebsd: - runs-on: macos-10.15 - name: Build and test on FreeBSD - env: - DEVELOPER: 1 - VALGRIND: 0 - EXPERIMENTAL_FEATURES: 0 - COMPAT: 1 - steps: - - uses: actions/checkout@v2 - - name: Test in FreeBSD - id: test - uses: vmactions/freebsd-vm@v0.1.5 - with: - envs: 'DEVELOPER VALGRIND EXPERIMENTAL_FEATURES COMPAT' - usesh: true - prepare: | - pkg install -y \ - wget \ - py38-pip \ - py38-sqlite3 \ - gmake \ - git \ - python \ - postgresql12-server \ - autoconf \ - automake \ - libtool \ - gmp \ - bash \ - gettext \ - sqlite3 \ - curl - - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2021-08-3z1 - - cd /tmp/ || exit 1 - wget https://storage.googleapis.com/c-lightning-tests/bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2 - tar -xjf bitcoin-0.20.1-x86_64-linux-gnu.tar.bz2 - sudo mv bitcoin-0.20.1/bin/* /usr/local/bin - rm -rf \ - bitcoin-0.20.1-x86_64-linux-gnu.tar.gz \ - bitcoin-0.20.1 - - run: | - PATH=/root/.local/bin:$PATH - pip install --user -U wheel pip - pip install --user -U -r requirements.txt - - # Install utilities that aren't dependencies, but make - # running tests easier/feasible on CI (and pytest which - # keeps breaking the rerunfailures plugin). - pip install --user -U \ - blinker \ - flake8 \ - mako \ - mrkd \ - pytest-sentry \ - pytest-test-groups==1.0.3 \ - pytest-custom-exit-code==0.3.0 \ - pytest-json-report - - git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc - git submodule update --init --recursive - - ./configure CC="$CC" - cat config.vars - - cat << EOF > pytest.ini - [pytest] - addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2 - markers = - slow_test: marks tests as slow (deselect with '-m "not slow_test"') - EOF - - # Just run a "quick" test without memory checking - gmake - - # Clean up to maximize rsync's chances of succeeding - gmake clean - diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f71e5c72ce85..22272a32ae62 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,151 +2,9 @@ name: Continuous Integration on: [push, pull_request] jobs: - smoke-test: - name: Smoke Test ${{ matrix.cfg }} - runs-on: ubuntu-20.04 - env: - DEVELOPER: 1 - VALGRIND: 0 - EXPERIMENTAL_FEATURES: 0 - COMPAT: 1 - strategy: - fail-fast: true - matrix: - include: - - CFG: "make" - TEST_CMD: "make" - - CFG: "make-O3-check" - TEST_CMD: "make check-source check-units installcheck check-gen-updated" - COPTFLAGS: "-O3" - - CFG: "make-32-bit-nodev-check" - ARCH: 32 - TEST_CMD: "make check-source check-units installcheck" - DEVELOPER: 0 - - CFG: "make-EXPERIMENTAL-check" - TEST_CMD: "make check-source check-units installcheck check-gen-updated" - EXPERIMENTAL_FEATURES: 1 - steps: - - name: Checkout - uses: actions/checkout@v2.0.0 - - - name: Fetch tags for auto versioning - run: git fetch --prune --unshallow --tags -f - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Build - env: - VALGRIND: ${{ matrix.VALGRIND }} - DEVELOPER: ${{ matrix.DEVELOPER }} - EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }} - COMPILER: ${{ matrix.COMPILER }} - ARCH: ${{ matrix.ARCH }} - COMPAT: ${{ matrix.COMPAT }} - PYTEST_PAR: ${{ matrix.PYTEST_PAR }} - PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }} - COPTFLAGS: ${{ matrix.COPTFLAGS }} - NETWORK: ${{ matrix.NETWORK }} - TEST_CMD: ${{ matrix.TEST_CMD }} - TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }} - TEST_GROUP: ${{ matrix.TEST_GROUP }} - run: | - bash -x .github/scripts/build.sh - - - name: Upload Unit Test Results - if: always() - uses: actions/upload-artifact@v2 - with: - name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }} - path: report.* - if-no-files-found: ignore - - check-dock: - name: Check c-lightning doc - runs-on: ubuntu-20.04 - env: - DEVELOPER: 1 - VALGRIND: 0 - EXPERIMENTAL_FEATURES: 0 - COMPAT: 1 - steps: - - name: Checkout - uses: actions/checkout@v2.0.0 - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - - name: Install dependencies - run: bash -x .github/scripts/setup.sh - - - name: Check Doc - run: | - pip install mako - ./configure - make check-doc - - proto-test: - name: Protocol Test Config - runs-on: ubuntu-20.04 - needs: [smoke-test] - env: - DEVELOPER: 1 - EXPERIMENTAL_FEATURES: 1 - COMPAT: 0 - PYTEST_PAR: 2 - TEST_CMD: "make check-protos" - TEST_GROUP: 1 - TEST_GROUP_COUNT: 1 - strategy: - fail-fast: true - matrix: - include: - - {compiler: clang, db: sqlite3} - - {compiler: gcc, db: postgres} - steps: - - name: Checkout - uses: actions/checkout@v2.0.0 - - - name: Setup Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Build - env: - ARCH: ${{ matrix.arch }} - COMPILER: ${{ matrix.compiler }} - DB: ${{ matrix.db }} - NETWORK: ${{ matrix.network }} - TARGET_HOST: ${{ matrix.TARGET_HOST }} - VALGRIND: ${{ matrix.valgrind }} - run: | - bash -x .github/scripts/build.sh - - - name: Upload Unit Test Results - if: always() - uses: actions/upload-artifact@v2 - with: - name: Junit Report ${{ github.run_number }}.{{ matrix.cfg }} - path: report.* - normal-test: name: Normal Test Config ${{ matrix.cfg }} runs-on: ubuntu-20.04 - needs: [smoke-test] env: DEVELOPER: 1 VALGRIND: 0 @@ -238,7 +96,6 @@ jobs: valgrind-test: name: Valgrind Test Config ${{ matrix.cfg }} runs-on: ubuntu-20.04 - needs: [smoke-test] env: DEVELOPER: 1 EXPERIMENTAL_FEATURES: 0 diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml deleted file mode 100644 index 027a537f84d5..000000000000 --- a/.github/workflows/macos.yaml +++ /dev/null @@ -1,102 +0,0 @@ ---- -name: Mac OS pytest -on: [pull_request] -jobs: - smoke-test: - name: Smoke Test macOS - runs-on: macos-latest - env: - DEVELOPER: 1 - VALGRIND: 0 - EXPERIMENTAL_FEATURES: 1 - COMPAT: 0 - strategy: - fail-fast: true - steps: - - name: Checkout - uses: actions/checkout@v2.0.0 - - - name: Fetch tags for auto versioning - run: git fetch --prune --unshallow --tags - - - name: Set up Python 3.6 - uses: actions/setup-python@v2 - with: - python-version: 3.6 - - - name: Install dependencies - run: | - export BITCOIN_VERSION=0.20.1 - brew install wget python autoconf automake libtool python3 gmp gnu-sed gettext libsodium - - ( - cd /tmp/ - wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-osx64.tar.gz -O bitcoin.tar.gz - tar -xvzf bitcoin.tar.gz - sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin - ) - - pip install --upgrade mako pip - ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt - export PATH="/usr/local/opt:$PATH" - - - name: Build - env: - VALGRIND: ${{ matrix.VALGRIND }} - DEVELOPER: ${{ matrix.DEVELOPER }} - EXPERIMENTAL_FEATURES: ${{ matrix.EXPERIMENTAL_FEATURES }} - COMPILER: ${{ matrix.COMPILER }} - COMPAT: ${{ matrix.COMPAT }} - PYTEST_PAR: ${{ matrix.PYTEST_PAR }} - PYTEST_OPTS: ${{ matrix.PYTEST_OPTS }} - NO_PYTHON: ${{ matrix.NO_PYTHON }} - COPTFLAGS: ${{ matrix.COPTFLAGS }} - NETWORK: ${{ matrix.NETWORK }} - TEST_CMD: ${{ matrix.TEST_CMD }} - TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }} - TEST_GROUP: ${{ matrix.TEST_GROUP }} - run: | - export PATH="/usr/local/opt:/Users/runner/.local/bin:$PATH" - export LDFLAGS="-L/usr/local/opt/sqlite/lib" - export CPPFLAGS="-I/usr/local/opt/sqlite/include" - - pip3 install --user -U \ - -r requirements.lock - - pip3 install --user --no-index \ - --use-feature=in-tree-build \ - ./contrib/pyln-spec/bolt7 \ - ./contrib/pyln-client \ - ./contrib/pyln-proto \ - ./contrib/pyln-testing - - # Install utilities that aren't dependencies, but make - # running tests easier/feasible on CI (and pytest which - # keeps breaking the rerunfailures plugin). - pip3 install --user -U \ - blinker \ - flake8 \ - flaky \ - mako \ - pytest-sentry \ - pytest-test-groups==1.0.3 \ - pytest-custom-exit-code==0.3.0 \ - pytest-json-report - - cat << EOF > pytest.ini - [pytest] - addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2 - markers = - slow_test: marks tests as slow (deselect with '-m "not slow_test"') - EOF - - ./configure - make - - - name: Upload Unit Test Results - if: always() - uses: actions/upload-artifact@v2 - with: - name: Junit Report ${{ github.run_number }}.${{ matrix.cfg }} - path: report.* - if-no-files-found: ignore diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml deleted file mode 100644 index 6e08825477da..000000000000 --- a/.github/workflows/pypi.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI -on: - # Only deploy if we're the result of a PR being merged - push: - branches: - - master -jobs: - deploy: - name: Build and publish ${{ matrix.package }} 🐍 - runs-on: ubuntu-20.04 - strategy: - fail-fast: true - matrix: - include: - - PACKAGE: pyln-client - WORKDIR: contrib/pyln-client - - PACKAGE: pyln-testing - WORKDIR: contrib/pyln-testing - - PACKAGE: pyln-proto - WORKDIR: contrib/pyln-proto - - PACKAGE: pyn-bolt1 - WORKDIR: contrib/pyln-spec/bolt1/ - - PACKAGE: pyn-bolt2 - WORKDIR: contrib/pyln-spec/bolt2/ - - PACKAGE: pyn-bolt4 - WORKDIR: contrib/pyln-spec/bolt4/ - - PACKAGE: pyn-bolt7 - WORKDIR: contrib/pyln-spec/bolt7/ - steps: - - uses: actions/checkout@master - with: - # Need to fetch entire history in order to locate the version tag - fetch-depth: 0 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - - name: Install pypa/build - run: >- - python -m pip install build --user - - - name: Build a binary wheel and a source tarball - env: - WORKDIR: ${{ matrix.WORKDIR }} - run: >- - cd ${{ env.WORKDIR}} && - python -m build --sdist --wheel --outdir dist/ . - - - name: Publish distribution 📦 to Test PyPI - if: github.repository == 'ElementsProject/lightning' - uses: pypa/gh-action-pypi-publish@master - env: - WORKDIR: ${{ matrix.WORKDIR }} - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - packages_dir: "${{ env.WORKDIR}}/dist" - skip_existing: true - - - name: Publish distribution 📦 to PyPI - if: startsWith(github.ref, 'refs/tags') && github.repository == 'ElementsProject/lightning' - uses: pypa/gh-action-pypi-publish@master - env: - WORKDIR: ${{ matrix.WORKDIR }} - with: - password: ${{ secrets.PYPI_API_TOKEN }} - packages_dir: "${{ env.WORKDIR}}/dist" - # We should never have a conflict here, the version tags are unique - skip_existing: false diff --git a/Makefile b/Makefile index 5a768d1ed00c..7143cd95ad9d 100644 --- a/Makefile +++ b/Makefile @@ -390,7 +390,7 @@ endif # processes and result in loads in loads of output. So we only tell py.test to # abort early if we aren't running in parallel. ifneq ($(PYTEST_PAR),) -PYTEST_OPTS += -n=$(PYTEST_PAR) +PYTEST_OPTS += -n=$(PYTEST_PAR) -x else PYTEST_OPTS += -x endif diff --git a/connectd/connectd.c b/connectd/connectd.c index e8897c7d3ad6..76c464da4f8a 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -396,9 +396,13 @@ struct io_plan *peer_connected(struct io_conn *conn, int subd_fd, gossip_fd; peer = peer_htable_get(&daemon->peers, id); - if (peer) + if (peer) { + status_peer_debug(id, "peer_reconnected:" + " told_to_close = %u, to_peer = %p, to_subd = %p", + peer->told_to_close, peer->to_peer, peer->to_subd); return peer_reconnected(conn, daemon, id, addr, cs, their_features, incoming); + } /* We promised we'd take it by marking it TAKEN above; prepare to free it. */ if (taken(their_features)) @@ -1735,8 +1739,13 @@ static void try_connect_peer(struct daemon *daemon, struct connecting *connect; /* Already done? May happen with timer. */ - if (peer_htable_get(&daemon->peers, id)) + if (peer_htable_get(&daemon->peers, id)) { + struct peer *peer = peer_htable_get(&daemon->peers, id); + status_peer_debug(id, "try_connect_peer: already have it:" + " told_to_close = %u, to_peer = %p, to_subd = %p", + peer->told_to_close, peer->to_peer, peer->to_subd); return; + } /* If we're trying to connect it right now, that's OK. */ if ((connect = find_connecting(daemon, id))) { diff --git a/connectd/multiplex.c b/connectd/multiplex.c index cdcb17a0c7bc..df571a953252 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -281,6 +281,8 @@ static struct io_plan *encrypt_and_send(struct peer *peer, #endif set_urgent_flag(peer, is_urgent(type)); + status_peer_debug(&peer->id, "write_to_peer %s", + peer_wire_name(fromwire_peektype(msg))); /* We free this and the encrypted version in next write_to_peer */ peer->sent_to_peer = cryptomsg_encrypt_msg(peer, &peer->cs, msg); return io_write(peer->to_peer, @@ -452,6 +454,10 @@ static struct io_plan *read_from_subd(struct io_conn *subd_conn, static struct io_plan *read_from_subd_done(struct io_conn *subd_conn, struct peer *peer) { + status_peer_debug(&peer->id, "read_from_subd %s", + peer_wire_name(fromwire_peektype(peer->subd_in))); + + /* Tell them to encrypt & write. */ queue_peer_msg(peer, take(peer->subd_in)); peer->subd_in = NULL; @@ -491,6 +497,8 @@ static struct io_plan *write_to_subd(struct io_conn *subd_conn, write_to_subd, peer); } + status_peer_debug(&peer->id, "write_to_subd %s", + peer_wire_name(fromwire_peektype(msg))); return io_write_wire(subd_conn, take(msg), write_to_subd, peer); } @@ -510,26 +518,33 @@ static struct io_plan *read_body_from_peer_done(struct io_conn *peer_conn, } tal_free(peer->peer_in); - /* dev_disconnect can disable read */ + status_peer_debug(&peer->id, "read_from_peer %s", + peer_wire_name(fromwire_peektype(decrypted))); + + /* dev_disconnect can disable read */ if (!IFDEV(peer->dev_read_enabled, true)) { + status_peer_debug(&peer->id, "... !dev_read_enabled"); tal_free(decrypted); return read_hdr_from_peer(peer_conn, peer); } /* Don't process packets while we're closing */ if (peer->told_to_close) { + status_peer_debug(&peer->id, "... told_to_close"); tal_free(decrypted); return read_hdr_from_peer(peer_conn, peer); } /* If we swallow this, just try again. */ if (handle_message_locally(peer, decrypted)) { + status_peer_debug(&peer->id, "... local handle"); tal_free(decrypted); return read_hdr_from_peer(peer_conn, peer); } /* If there's no subd, discard and keep reading. */ if (!peer->to_subd) { + status_peer_debug(&peer->id, "... !to_subd"); tal_free(decrypted); return read_hdr_from_peer(peer_conn, peer); } @@ -593,6 +608,10 @@ static void destroy_subd_conn(struct io_conn *subd_conn, struct peer *peer) * it hangs up! */ io_wake(&peer->peer_in); + status_peer_debug(&peer->id, "destroy_subd_conn:" + " told_to_close = %u, to_peer = %p, to_subd = %p", + peer->told_to_close, peer->to_peer, peer->to_subd); + /* If no peer, finally time to close */ if (!peer->to_peer && peer->told_to_close) peer_conn_closed(peer); @@ -605,10 +624,15 @@ void close_peer_conn(struct peer *peer) /* Already dead? */ if (!peer->to_subd && !peer->to_peer) { + status_peer_debug(&peer->id, "close_peer_conn: already done"); peer_conn_closed(peer); return; } + status_peer_debug(&peer->id, "close_peer_conn:" + " told_to_close = %u, to_peer = %p, to_subd = %p", + peer->told_to_close, peer->to_peer, peer->to_subd); + /* In case it's not currently writing, wake write_to_peer */ msg_wake(peer->peer_outq); } @@ -633,6 +657,10 @@ static void destroy_peer_conn(struct io_conn *peer_conn, struct peer *peer) assert(peer->to_peer == peer_conn); peer->to_peer = NULL; + status_peer_debug(&peer->id, "destroy_peer_conn:" + " told_to_close = %u, to_peer = %p, to_subd = %p", + peer->told_to_close, peer->to_peer, peer->to_subd); + /* Flush internal connections if not already. */ if (peer->to_subd) { msg_wake(peer->subd_outq);