sqlite db backend

.github/workflows/main.yml

@@ -208,6 +208,7 @@ jobs:
           - btcd unit-cover
           - unit tags="kvdb_etcd"
           - unit tags="kvdb_postgres"
+          - unit tags="kvdb_sqlite"
           - btcd unit-race
     steps:
       - name: git checkout
@@ -265,6 +266,8 @@ jobs:
             args: backend=bitcoind dbbackend=etcd
           - name: bitcoind-postgres
             args: backend=bitcoind dbbackend=postgres
+          - name: bitcoind-sqlite
+            args: backend=bitcoind dbbackend=sqlite
           - name: neutrino
             args: backend=neutrino
     steps:

.github/workflows/release.yaml

@@ -94,7 +94,7 @@ jobs:
 
             Our release binaries are fully reproducible. Third parties are able to verify that the release binaries were produced properly without having to trust the release manager(s). See our [reproducible builds guide](https://github.com/lightningnetwork/lnd/tree/master/build/release) for how this can be achieved.
             The release binaries are compiled with `go${{ env.GO_VERSION }}`, which is required by verifiers to arrive at the same ones.
-            They include the following build tags: `autopilotrpc`, `signrpc`, `walletrpc`, `chainrpc`, `invoicesrpc`, `neutrinorpc`, `routerrpc`, `watchtowerrpc`, `monitoring`, `peersrpc`, `kvdb_postrgres`, and `kvdb_etcd`. Note that these are already included in the release script, so they do not need to be provided.
+            They include the following build tags: `autopilotrpc`, `signrpc`, `walletrpc`, `chainrpc`, `invoicesrpc`, `neutrinorpc`, `routerrpc`, `watchtowerrpc`, `monitoring`, `peersrpc`, `kvdb_postgres`, `kvdb_sqlite`, and `kvdb_etcd`. Note that these are already included in the release script, so they do not need to be provided.
 
             The `make release` command can be used to ensure one rebuilds with all the same flags used for the release. If one wishes to build for only a single platform, then `make release sys=<OS-ARCH> tag=<tag>` can be used. 
 

.golangci.yml

@@ -24,6 +24,7 @@ run:
     - watchtowerrpc
     - kvdb_etcd
     - kvdb_postgres
+    - kvdb_sqlite
 
 linters-settings:
   govet:

Makefile

@@ -139,8 +139,8 @@ install:
 
 release-install:
 	@$(call print, "Installing release lnd and lncli.")
-	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd
-	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lncli
+	env CGO_ENABLED=1 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd
+	env CGO_ENABLED=1 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lncli
 
 # Make sure the generated mobile RPC stubs don't influence our vendor package
 # by removing them first in the clean-mobile target.

channeldb/db_test.go

@@ -26,7 +26,7 @@ func TestOpenWithCreate(t *testing.T) {
 	t.Parallel()
 
 	// Checking for db file existence is not possible with postgres.
-	if kvdb.PostgresBackend {
+	if kvdb.TestBackend == kvdb.PostgresBackendName {
 		t.Skip()
 	}
 

docs/INSTALL.md

@@ -253,7 +253,8 @@ will have the following tags:
 - [watchtowerrpc](/lnrpc/watchtowerrpc/watchtower.proto)
 - [monitoring](/monitoring) (for Prometheus integration)
 - [peersrpc](/lnrpc/peersrpc/peers.proto)
-- [kvdb_postrgres](/docs/postgres.md)
+- [kvdb_postgres](/docs/postgres.md)
+- [kvdb_sqlite](/docs/sqlite.md)
 - [kvdb_etcd](/docs/etcd.md)
 
 The `dev` tag is used for development builds, and is not included in the

docs/sqlite.md

@@ -0,0 +1,23 @@
+# SQLite support in LND
+
+With the introduction of the `kvdb` interface, LND can support multiple database
+backends. One of the supported backends is [sqlite](https://www.sqlite.org/index.html). This document
+describes how use LND with a sqlite backend.
+
+## Configuring LND for SQLite
+
+LND is configured for SQLite through the following configuration options:
+
+* `db.backend=sqlite` to select the SQLite backend.
+* `db.sqlite.filename=...` to set the file where sqlite data will be stored. 
+  This file will be created if it does not exist.
+* `db.sqlite.timeout=...` to set the connection timeout. If not set, no
+  timeout applies.
+
+Example as follows:
+```
+[db]
+db.backend=sqlite
+db.sqlite.dsn=/var/data/lnd.db
+db.sqlite.timeout=0
+```

go.mod

@@ -102,6 +102,7 @@ require (
 	github.com/lib/pq v1.10.3 // indirect
 	github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mattn/go-sqlite3 v1.14.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mholt/archiver/v3 v3.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -167,6 +168,9 @@ replace github.com/ulikunitz/xz => github.com/ulikunitz/xz v0.5.8
 // https://deps.dev/advisory/OSV/GO-2021-0053?from=%2Fgo%2Fgithub.com%252Fgogo%252Fprotobuf%2Fv1.3.1
 replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 
+// TODO: temp for local development
+replace github.com/lightningnetwork/lnd/kvdb => ./kvdb
+
 // If you change this please also update .github/pull_request_template.md and
 // docs/INSTALL.md.
 go 1.17

go.sum

@@ -479,8 +479,6 @@ github.com/lightningnetwork/lnd/clock v1.1.0/go.mod h1:KnQudQ6w0IAMZi1SgvecLZQZ4
 github.com/lightningnetwork/lnd/healthcheck v1.0.0/go.mod h1:u92p1JGFJNMSkMvztKEwmt1P3TRnLeJBXZ3M85xkU1E=
 github.com/lightningnetwork/lnd/healthcheck v1.2.2 h1:im+qcpgSuteqRCGeorT9yqVXuLrS6A7/acYzGgarMS4=
 github.com/lightningnetwork/lnd/healthcheck v1.2.2/go.mod h1:IWY0GChlarRbXFkFDdE4WY5POYJabe/7/H1iCZt4ZKs=
-github.com/lightningnetwork/lnd/kvdb v1.3.1 h1:gEz3zudNNRrCLEvqRaktYoKwsUblyHX+MKjR0aI3QnM=
-github.com/lightningnetwork/lnd/kvdb v1.3.1/go.mod h1:x+IpsuDynubjokUofavLXroeGfS/WrqUXXTK6vN/gp4=
 github.com/lightningnetwork/lnd/queue v1.0.1/go.mod h1:vaQwexir73flPW43Mrm7JOgJHmcEFBWWSl9HlyASoms=
 github.com/lightningnetwork/lnd/queue v1.1.0 h1:YpCJjlIvVxN/R7ww2aNiY8ex7U2fucZDLJ67tI3HFx8=
 github.com/lightningnetwork/lnd/queue v1.1.0/go.mod h1:YTkTVZCxz8tAYreH27EO3s8572ODumWrNdYW2E/YKxg=
@@ -513,6 +511,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
+github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=

kvdb/backend.go

@@ -240,15 +240,16 @@ func updateLastCompactionDate(dbFile string) error {
 
 // GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd
 // backed database (for testing), and returns a kvdb.Backend and a cleanup
-// func. Whether to create/open bbolt or embedded etcd database is based
-// on the TestBackend constant which is conditionally compiled with build tag.
+// func. Whether to create/open bbolt, embedded etcd, postgres, or sqlite
+// database is based on the TestBackend constant which is conditionally compiled
+// with build tag.
 // The passed path is used to hold all db files, while the name is only used
 // for bbolt.
 func GetTestBackend(path, name string) (Backend, func(), error) {
 	empty := func() {}
 
-	switch {
-	case PostgresBackend:
+	switch TestBackend {
+	case PostgresBackendName:
 		key := filepath.Join(path, name)
 		keyHash := sha256.Sum256([]byte(key))
 
@@ -260,7 +261,7 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 			_ = f.DB().Close()
 		}, nil
 
-	case TestBackend == BoltBackendName:
+	case BoltBackendName:
 		db, err := GetBoltBackend(&BoltBackendConfig{
 			DBPath:         path,
 			DBFileName:     name,
@@ -272,7 +273,7 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 		}
 		return db, empty, nil
 
-	case TestBackend == EtcdBackendName:
+	case EtcdBackendName:
 		etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")
 		if err != nil {
 			return nil, empty, err
@@ -282,6 +283,13 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 		)
 		return backend, cancel, err
 
+	case SqliteBackendName:
+		f, err := NewSqliteFixture()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return f.DB(), f.Cleanup, nil
 	}
 
 	return nil, nil, fmt.Errorf("unknown backend")

kvdb/config.go

@@ -18,6 +18,11 @@ const (
 	// by a live instance of postgres.
 	PostgresBackendName = "postgres"
 
+	// SqliteBackendName is the name of the backend that should be passed
+	// into kvdb.Create to initialize a new instance of kvdb.Backend backed
+	// by a live instance of sqlite.
+	SqliteBackendName = "sqlite"
+
 	// DefaultBoltAutoCompactMinAge is the default minimum time that must
 	// have passed since a bolt database file was last compacted for the
 	// compaction to be considered again.

kvdb/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/lib/pq v1.10.3 // indirect
 	github.com/lightningnetwork/lnd/healthcheck v1.0.0
+	github.com/mattn/go-sqlite3 v1.14.13 // indirect
 	github.com/nwaples/rardecode v1.1.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.8 // indirect
 	github.com/stretchr/testify v1.7.0

kvdb/go.sum

@@ -291,6 +291,8 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I=
+github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=

kvdb/kvdb_boltdb.go

@@ -0,0 +1,7 @@
+//go:build !kvdb_etcd && !kvdb_postgres && !kvdb_sqlite
+// +build !kvdb_etcd,!kvdb_postgres,!kvdb_sqlite
+
+package kvdb
+
+// When no other db backends are specified via dev flags, boltdb will be the default test db backend.
+const TestBackend = BoltBackendName

kvdb/kvdb_no_etcd.go

@@ -9,10 +9,6 @@ import (
 	"github.com/lightningnetwork/lnd/kvdb/etcd"
 )
 
-// TestBackend is conditionally set to bdb when the kvdb_etcd build tag is
-// not defined, allowing testing our database code with bolt backend.
-const TestBackend = BoltBackendName
-
 var errEtcdNotAvailable = fmt.Errorf("etcd backend not available")
 
 // StartEtcdTestBackend  is a stub returning nil, and errEtcdNotAvailable error.

kvdb/kvdb_no_postgres.go

@@ -9,8 +9,6 @@ import (
 	"github.com/lightningnetwork/lnd/kvdb/postgres"
 )
 
-const PostgresBackend = false
-
 func NewPostgresFixture(dbName string) (postgres.Fixture, error) {
 	return nil, errors.New("postgres backend not available")
 }

kvdb/kvdb_no_sqlite.go

@@ -0,0 +1,14 @@
+//go:build !kvdb_sqlite
+// +build !kvdb_sqlite
+
+package kvdb
+
+import (
+	"errors"
+
+	"github.com/lightningnetwork/lnd/kvdb/sqlite"
+)
+
+func NewSqliteFixture() (sqlite.Fixture, error) {
+	return nil, errors.New("sqlite backend not available")
+}

kvdb/kvdb_postgres.go

@@ -5,7 +5,7 @@ package kvdb
 
 import "github.com/lightningnetwork/lnd/kvdb/postgres"
 
-const PostgresBackend = true
+const TestBackend = PostgresBackendName
 
 func NewPostgresFixture(dbName string) (postgres.Fixture, error) {
 	return postgres.NewFixture(dbName)

kvdb/kvdb_sqlite.go

@@ -0,0 +1,14 @@
+//go:build kvdb_sqlite
+// +build kvdb_sqlite
+
+package kvdb
+
+import "github.com/lightningnetwork/lnd/kvdb/sqlite"
+
+const TestBackend = SqliteBackendName
+
+func NewSqliteFixture() (sqlite.Fixture, error) {
+	f, err := sqlite.NewSqliteTestFixture("test")
+
+	return f, err
+}

kvdb/log.go

@@ -3,6 +3,7 @@ package kvdb
 import (
 	"github.com/btcsuite/btclog"
 	"github.com/lightningnetwork/lnd/kvdb/postgres"
+	"github.com/lightningnetwork/lnd/kvdb/sqlite"
 )
 
 // log is a logger that is initialized as disabled.  This means the package will
@@ -14,4 +15,5 @@ func UseLogger(logger btclog.Logger) {
 	log = logger
 
 	postgres.UseLogger(log)
+	sqlite.UseLogger(log)
 }

kvdb/sqlite/config.go

@@ -0,0 +1,10 @@
+package sqlite
+
+import "time"
+
+// Config holds sqlite configuration data.
+type Config struct {
+	TablePrefix string        `long:"table_prefix" description:"Prefix that will be added to each database table."`
+	Filename    string        `long:"filename" description:"Full path to sqlite database file."`
+	Timeout     time.Duration `long:"timeout" description:"Database connection timeout. Set to zero to disable."`
+}