Port upstream PRs

.gitignore

@@ -35,8 +35,10 @@ _testmain.go
 /dcrlncli-itest
 
 # Integration test log files
-output*.log
-/.backendlogs
+lntest/itest/output*.log
+lntest/itest/pprof*.log
+lntest/itest/.backendlogs
+lntest/itest/.minerlogs
 
 cmd/cmd
 *.key

.golangci.yml

@@ -0,0 +1,10 @@
+issues:
+        exclude-rules:
+                # Ignore govet.lostcancel in integration test files. The
+                # integration tests make substantial use of context.WithTimeout()
+                # without corresponding cancel() calls, which trigger this. So
+                # disable these warnings for now until this is fixed upstream
+                # (otherwise merging upstream itest changes will become 
+                # significantly harder)
+                - path: lnd.*_test\.go
+                  text: lostcancel

.travis.yml

@@ -30,13 +30,14 @@ script:
   - 'if [ "$RACE" = true ]; then make travis-race ; fi'
 
   # Run integration tests.
-  - 'if [ "$ITEST" = true ]; then make itest; fi'
+  - 'if [ "$ITEST" = true ]; then make travis-itest; fi'
 
   # Run unit tests and generate coverage report.
   - 'if [ "$COVER" = true ]; then make travis-cover; fi'
 
 after_script:
+  - LOG_FILES=./lntest/itest/*.log
   - echo "Uploading to termbin.com..."
-  - find *.log | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
+  - find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
   - echo "Uploading to file.io..."
-  - tar -zcvO *.log | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
+  - tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'

Makefile

@@ -10,11 +10,13 @@ ESCPKG := github.com\/decred\/dcrlnd
 DCRD_PKG := github.com/decred/dcrd
 GOVERALLS_PKG := github.com/mattn/goveralls
 LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
+GOACC_PKG := github.com/ory/go-acc
 
 GO_BIN := ${GOPATH}/bin
 DCRD_BIN := $(GO_BIN)/dcrd
 GOVERALLS_BIN := $(GO_BIN)/goveralls
 LINT_BIN := $(GO_BIN)/golangci-lint
+GOACC_BIN := $(GO_BIN)/go-acc
 
 DCRD_DIR :=${GOPATH}/src/$(DCRD_PKG)
 
@@ -27,6 +29,8 @@ DCRD_COMMIT := $(shell cat go.mod | \
 		awk -F " " '{ print $$2 }' | \
 		awk -F "/" '{ print $$1 }')
 
+GOACC_COMMIT := ddc355013f90fea78d83d3a6c71f1d37ac07ecd5
+
 GOBUILD := GO111MODULE=on go build -v
 GOINSTALL := GO111MODULE=on go install -v
 GOTEST := GO111MODULE=on go test -v
@@ -44,23 +48,6 @@ include make/testing_flags.mk
 
 DEV_TAGS := $(if ${tags},$(DEV_TAGS) ${tags},$(DEV_TAGS))
 
-COVER = for dir in $(GOLISTCOVER); do \
-		$(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" \
-			-covermode=count \
-			-coverprofile=$$dir/profile.tmp $$dir; \
-		\
-		if [ $$? != 0 ] ; \
-		then \
-			exit 1; \
-		fi ; \
-		\
-		if [ -f $$dir/profile.tmp ]; then \
-			cat $$dir/profile.tmp | \
-				tail -n +2 >> profile.cov; \
-			$(RM) $$dir/profile.tmp; \
-		fi \
-	done
-
 LINT = $(LINT_BIN) \
 	run \
 	--build-tags rpctest \
@@ -94,6 +81,11 @@ $(LINT_BIN):
 	@$(call print, "Fetching golangci-lint")
 	GO111MODULE=on go get -u $(LINT_PKG)
 
+$(GOACC_BIN):
+	@$(call print, "Fetching go-acc")
+	go get -u -v $(GOACC_PKG)@$(GOACC_COMMIT)
+	$(GOINSTALL) $(GOACC_PKG)
+
 dcrd:
 	@$(call print, "Installing dcrd $(DCRD_COMMIT).")
 	GO111MODULE=on go get -v github.com/decred/dcrd/@$(DCRD_COMMIT)
@@ -104,17 +96,17 @@ dcrd:
 
 build:
 	@$(call print, "Building debug dcrlnd and dcrlncli.")
-	$(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlnd-debug $(LDFLAGS) $(PKG)
+	$(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlnd-debug $(LDFLAGS) $(PKG)/cmd/dcrlnd
 	$(GOBUILD) -tags="$(DEV_TAGS)" -o dcrlncli-debug $(LDFLAGS) $(PKG)/cmd/dcrlncli
 
 build-itest:
 	@$(call print, "Building itest dcrlnd and dcrlncli.")
-	$(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlnd-itest $(LDFLAGS) $(PKG)
+	$(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlnd-itest $(LDFLAGS) $(PKG)/cmd/dcrlnd
 	$(GOBUILD) -tags="$(ITEST_TAGS)" -o dcrlncli-itest $(LDFLAGS) $(PKG)/cmd/dcrlncli
 
 install:
 	@$(call print, "Installing dcrlnd and dcrlncli.")
-	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)
+	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/dcrlnd
 	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/dcrlncli
 
 scratch: build
@@ -127,7 +119,7 @@ scratch: build
 check: unit itest
 
 itest-only:
-	@$(call print, "Running integration tests.")
+	@$(call print, "Running integration tests with ${backend} backend.")
 	$(ITEST)
 
 itest: dcrd build-itest itest-only
@@ -138,35 +130,36 @@ unit-only:
 
 unit: dcrd unit-only
 
-unit-cover:
+unit-cover: $(GOACC_BIN)
 	@$(call print, "Running unit coverage tests.")
-	echo "mode: count" > profile.cov
-	$(COVER)
+	$(GOACC_BIN) $(COVER_PKG) -- -test.tags="$(DEV_TAGS) $(LOG_TAGS)"
+
 
 unit-race:
 	@$(call print, "Running unit race tests.")
 	env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE)
 
 goveralls: $(GOVERALLS_BIN)
 	@$(call print, "Sending coverage report.")
-	$(GOVERALLS_BIN) -coverprofile=profile.cov -service=travis-ci
+	$(GOVERALLS_BIN) -coverprofile=coverage.txt -service=travis-ci
 
 travis-race: dcrd unit-race
 
 travis-cover: dcrd unit-cover goveralls
 
+travis-itest: itest
+
 # =============
 # FLAKE HUNTING
 # =============
 
-flakehunter: build
-	@$(call print, "Flake hunting integration tests.")
+flakehunter: build-itest
+	@$(call print, "Flake hunting ${backend} integration tests.")
 	while [ $$? -eq 0 ]; do $(ITEST); done
 
 flake-unit:
 	@$(call print, "Flake hunting unit tests.")
-	GOTRACEBACK=all $(UNIT) -count=1
-	while [ $$? -eq 0 ]; do /bin/sh -c "GOTRACEBACK=all $(UNIT) -count=1"; done
+	while [ $$? -eq 0 ]; do GOTRACEBACK=all $(UNIT) -count=1; done
 
 # =========
 # UTILITIES
@@ -212,6 +205,7 @@ clean:
 	goveralls \
 	travis-race \
 	travis-cover \
+	travis-itest \
 	flakehunter \
 	flake-unit \
 	fmt \

aezeed/README.md

@@ -1,15 +1,15 @@
 # aezeed
 
-[In this PR](https://github.com/lightningnetwork/lnd/pull/773) we add a new package implementing the aezeed cipher
-seed scheme (based on [aez](http://web.cs.ucdavis.edu/~rogaway/aez/) ).
+[In this PR](https://github.com/lightningnetwork/lnd/pull/773) we added a new package implementing the aezeed cipher
+seed scheme (based on [aez](http://web.cs.ucdavis.edu/~rogaway/aez/)).
 
-This is a new scheme developed that aims to overcome the
-two major short comings of BIP39: a lack of a version, and a lack of a
-wallet birthday. A lack a version means that wallets may not
+This new scheme aims to address
+two major features lacking in [BIP39](https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki): versioning, and a
+wallet birthday. The lack a version means that wallets may not
 necessarily know how to re-derive addresses during the recovery
 process. A lack of a birthday means that wallets don’t know how far
 back to look in the chain to ensure that they derive all the proper
-user addresses. Additionally, BIP39 use a very weak KDF. We use
+user addresses. Additionally, BIP39 use a very weak [KDF](https://en.wikipedia.org/wiki/Key_derivation_function). We use
 scrypt with modern parameters (n=32768, r=8, p=1). A set of benchmarks has
 been added, on my laptop I get about 100ms per attempt):
 
@@ -25,8 +25,8 @@ PASS
 ok      github.com/lightningnetwork/lnd/aezeed  4.168s
 ```
 
-Aside from addressing the shortcomings of BIP 39 a cipher  seed
-can: be upgraded, and have it's password changed,
+Aside from addressing the shortcomings of BIP 39, an aezeed cipher seed
+can both be upgraded, and have its password changed.
 
 Sample seed:
 
@@ -49,23 +49,23 @@ the keys of the wallet.
 The 2 byte timestamp is expressed in Decred Days Genesis, meaning that
 the number of days since the timestamp in Decred’s genesis block. This
 allow us to save space, and also avoid using a wasteful level of
-granularity. With the currently, this can express time up until 2188.
+granularity. This can currently express time up until 2188.
 
-Finally, the entropy is raw entropy that should be used to derive
+Finally, the entropy is raw entropy that should be used to derive the
 wallet’s HD root.
 
 ## aezeed enciphering/deciperhing
 
 Next, we’ll take the plaintext seed described above and encipher it to
 procure a final cipher text. We’ll then take this cipher text (the
-CipherSeed) and encode that using a 24-word mnemonic. The enciphering
-process takes a user defined passphrase. If no passphrase is provided,
+_CipherSeed_) and encode that using a 24-word mnemonic. The enciphering
+process takes a user-defined passphrase. If no passphrase is provided,
 then the string “aezeed” will be used.
 
 To encipher a plaintext seed (19 bytes) to arrive at an enciphered
 cipher seed (33 bytes), we apply the following operations:
 
-* First we take the external version an append it to our buffer. The
+* First we take the external version and append it to our buffer. The
 external version describes how we encipher. For the first version
 (version 0), we’ll use scrypt(n=32768, r=8, p=1) and aezeed.
 * Next, we’ll use scrypt (with the version 9 params) to generate a
@@ -79,15 +79,15 @@ has what’s essentially a configurable MAC size. In our scheme we’ll use
 a value of 8, which acts as a 64-bit checksum. We’ll encrypt with our
 generated seed, and use an AD of (version || salt).
 * Finally, we’ll encode this 33-byte cipher text using the default
-world list of BIP 39 to produce 24 english words.
+word list of BIP 39 to produce 24 English words.
 
 ## Properties of the aezeed cipher seed
 
 The aezeed cipher seed scheme has a few cool properties, notably:
 
 * The mnemonic itself is a cipher text, meaning leaving it in
-plaintext is advisable if the user also set a passphrase. This is in
-contrast to BIP 39 where the mnemonic alone (without a passrphase) may
+plaintext is advisable if the user also sets a passphrase. This is in
+contrast to BIP 39 where the mnemonic alone (without a passphrase) may
 be sufficient to steal funds.
 * A cipherseed can be modified to change the passphrase. This
 means that if the users wants a stronger passphrase, they can decipher

aezeed/cipherseed.go

@@ -510,10 +510,13 @@ func (m *Mnemonic) Decipher(pass []byte) ([DecipheredCipherSeedSize]byte, error)
 	// Before we attempt to map the mnemonic back to the original
 	// ciphertext, we'll ensure that all the word are actually a part of
 	// the current default word list.
-	for _, word := range m {
+	for i, word := range m {
 		if !strings.Contains(englishWordList, word) {
 			emptySeed := [DecipheredCipherSeedSize]byte{}
-			return emptySeed, ErrUnknownMnenomicWord{word}
+			return emptySeed, ErrUnknownMnenomicWord{
+				Word:  word,
+				Index: uint8(i),
+			}
 		}
 	}
 

aezeed/cipherseed_test.go

@@ -543,8 +543,12 @@ func TestDecipherUnknownMnenomicWord(t *testing.T) {
 		t.Fatalf("expected ErrUnknownMnenomicWord instead got %T", err)
 	}
 
-	if wordErr.word != "kek" {
-		t.Fatalf("word mismatch: expected %v, got %v", "kek", wordErr.word)
+	if wordErr.Word != "kek" {
+		t.Fatalf("word mismatch: expected %v, got %v", "kek", wordErr.Word)
+	}
+	if int32(wordErr.Index) != randIndex {
+		t.Fatalf("wrong index detected: expected %v, got %v",
+			randIndex, wordErr.Index)
 	}
 }
 

aezeed/errors.go

@@ -21,10 +21,16 @@ var (
 // ErrUnknownMnenomicWord is returned when attempting to decipher and
 // enciphered mnemonic, but a word encountered isn't a member of our word list.
 type ErrUnknownMnenomicWord struct {
-	word string
+	// Word is the unknown word in the mnemonic phrase.
+	Word string
+
+	// Index is the index (starting from zero) within the slice of strings
+	// that makes up the mnemonic that points to the incorrect word.
+	Index uint8
 }
 
 // Error returns a human readable string describing the error.
 func (e ErrUnknownMnenomicWord) Error() string {
-	return fmt.Sprintf("word %v isn't a part of default word list", e.word)
+	return fmt.Sprintf("word %v isn't a part of default word list "+
+		"(index=%v)", e.Word, e.Index)
 }