Skip to content

Commit

Permalink
FAB-1585 Refactor History to enable LevelDB
Browse files Browse the repository at this point in the history 
The purpose of this refactor is to create a common interface and to
isolate the db implementation, so that we can enable history on
LevelDB. This change only refactors the existing code and tests.
It does not add new functionality or any new tests. There will be
future changes that add history support on LevelDB and clean up
tests to work against LevelDB.

Note: History is not enabled by default. History must be enabled in
core.yaml historyDatabase for it's unit tests to run.

Change-Id: Ia5293cd345be4499e5e3df0c2949af16f71a608d
Signed-off-by: denyeart <[email protected]>
  • Loading branch information
@denyeart
denyeart committed Jan 20, 2017
1 parent 77cd1dc commit 74eeb66
Show file tree
Hide file tree
Showing 9 changed files with 187 additions and 106 deletions.
54 changes: 10 additions & 44 deletions core/ledger/history/couchdb_histmgr.go → ...histmgr/couchdbhistmgr/couchdb_histmgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,14 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package history
package couchdbhistmgr

import (
"bytes"
"encoding/json"
"fmt"
"strconv"

"github.com/hyperledger/fabric/core/ledger"
helper "github.com/hyperledger/fabric/core/ledger/kvledger/history/histmgr"
"github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/rwset"
"github.com/hyperledger/fabric/core/ledger/util/couchdb"
"github.com/hyperledger/fabric/protos/common"
Expand All @@ -39,11 +38,9 @@ type couchSavepointData struct {
UpdateSeq string `json:"UpdateSeq"`
}

var logger = logging.MustGetLogger("history")
var logger = logging.MustGetLogger("couchdbhistmgr")

var compositeKeySep = []byte{0x00}

// CouchDBHistMgr a simple implementation of interface `histmgmt.HistMgr'.
// CouchDBHistMgr a simple implementation of interface `histmgr.HistMgr'.
// TODO This implementation does not currently use a lock but may need one to ensure query's are consistent
type CouchDBHistMgr struct {
couchDB *couchdb.CouchDatabase // COUCHDB new properties for CouchDB
Expand All @@ -64,12 +61,12 @@ func NewCouchDBHistMgr(couchDBConnectURL string, dbName string, id string, pw st
return &CouchDBHistMgr{couchDB: couchDB}
}

// NewHistoryQueryExecutor implements method in interface `histmgmt.HistMgr'.
// NewHistoryQueryExecutor implements method in interface `histr.HistMgr'.
func (histmgr *CouchDBHistMgr) NewHistoryQueryExecutor() (ledger.HistoryQueryExecutor, error) {
return &CouchDBHistQueryExecutor{histmgr}, nil
}

// Commit implements method in interface `histmgmt.HistMgr`
// Commit implements method in interface `histmgr.HistMgr`
// This writes to a separate history database.
// TODO dpending on how invalid transactions are handled may need to filter what history commits.
func (histmgr *CouchDBHistMgr) Commit(block *common.Block) error {
Expand Down Expand Up @@ -113,7 +110,7 @@ func (histmgr *CouchDBHistMgr) Commit(block *common.Block) error {
for _, kvWrite := range nsRWSet.Writes {
writeKey := kvWrite.Key
writeValue := kvWrite.Value
compositeKey := constructCompositeKey(ns, writeKey, blockNo, tranNo)
compositeKey := helper.ConstructCompositeKey(ns, writeKey, blockNo, tranNo)
var bytesDoc []byte

logger.Debugf("===HISTORYDB=== ns (namespace or cc id) = %v, writeKey: %v, compositeKey: %v, writeValue = %v",
Expand Down Expand Up @@ -195,8 +192,8 @@ func (histmgr *CouchDBHistMgr) getTransactionsForNsKey(namespace string, key str
var compositeStartKey []byte
var compositeEndKey []byte
if key != "" {
compositeStartKey = constructPartialCompositeKey(namespace, key, false)
compositeEndKey = constructPartialCompositeKey(namespace, key, true)
compositeStartKey = helper.ConstructPartialCompositeKey(namespace, key, false)
compositeEndKey = helper.ConstructPartialCompositeKey(namespace, key, true)
}

//TODO the limit should not be hardcoded. Need the config.
Expand Down Expand Up @@ -227,37 +224,6 @@ func (txmgr *CouchDBHistMgr) GetBlockNumFromSavepoint() (uint64, error) {
return savepointDoc.BlockNum, nil
}

func constructCompositeKey(ns string, key string, blocknum uint64, trannum uint64) string {
//History Key is: "namespace key blocknum trannum"", with namespace being the chaincode id

// TODO - We will likely want sortable varint encoding, rather then a simple number, in order to support sorted key scans
var buffer bytes.Buffer
buffer.WriteString(ns)
buffer.WriteByte(0)
buffer.WriteString(key)
buffer.WriteByte(0)
buffer.WriteString(strconv.Itoa(int(blocknum)))
buffer.WriteByte(0)
buffer.WriteString(strconv.Itoa(int(trannum)))

return buffer.String()
}

func constructPartialCompositeKey(ns string, key string, endkey bool) []byte {
compositeKey := []byte(ns)
compositeKey = append(compositeKey, compositeKeySep...)
compositeKey = append(compositeKey, []byte(key)...)
if endkey {
compositeKey = append(compositeKey, []byte("1")...)
}
return compositeKey
}

func splitCompositeKey(compositePartialKey []byte, compositeKey []byte) (string, string) {
split := bytes.SplitN(compositeKey, compositePartialKey, 2)
return string(split[0]), string(split[1])
}

type histScanner struct {
cursor int
compositePartialKey []byte
Expand All @@ -283,7 +249,7 @@ func (scanner *histScanner) next() (*historicValue, error) {

selectedValue := scanner.results[scanner.cursor]

_, blockNumTranNum := splitCompositeKey(scanner.compositePartialKey, []byte(selectedValue.ID))
_, blockNumTranNum := helper.SplitCompositeKey(scanner.compositePartialKey, []byte(selectedValue.ID))

return &historicValue{blockNumTranNum, selectedValue.Value}, nil

Expand Down
70 changes: 39 additions & 31 deletions core/ledger/history/couchdb_histmgr_test.go → ...gr/couchdbhistmgr/couchdb_histmgr_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package history
package couchdbhistmgr

import (
"fmt"
"testing"

helper "github.com/hyperledger/fabric/core/ledger/kvledger/history/histmgr"
"github.com/hyperledger/fabric/core/ledger/ledgerconfig"
"github.com/hyperledger/fabric/core/ledger/testutil"
)
Expand All @@ -34,48 +35,48 @@ otherwise HistoryDB may not be installed and all the tests would fail
func TestHistoryDatabaseAutoCreate(t *testing.T) {

//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../peer")
testutil.SetupCoreYAMLConfig("./../../../../../../peer")
logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())

if ledgerconfig.IsHistoryDBEnabled() == true {

env := newTestEnvHistoryCouchDB(t, "history-test")
env.cleanup() //cleanup at the beginning to ensure the database doesn't exist already
defer env.cleanup() //and cleanup at the end
env := newTestEnvCouchDB(t, "history-test")
env.cleanupCouchDB() //cleanup at the beginning to ensure the database doesn't exist already
defer env.cleanupCouchDB() //and cleanup at the end

logger.Debugf("===HISTORYDB=== env.couchDBAddress: %v , env.couchDatabaseName: %v env.couchUsername: %v env.couchPassword: %v\n",
env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
logger.Debugf("===HISTORYDB=== env.CouchDBAddress: %v , env.CouchDatabaseName: %v env.CouchUsername: %v env.CouchPassword: %v\n",
env.CouchDBAddress, env.CouchDatabaseName, env.CouchUsername, env.CouchPassword)

histMgr := NewCouchDBHistMgr(
env.couchDBAddress, //couchDB Address
env.couchDatabaseName, //couchDB db name
env.couchUsername, //enter couchDB id
env.couchPassword) //enter couchDB pw
env.CouchDBAddress, //couchDB Address
env.CouchDatabaseName, //couchDB db name
env.CouchUsername, //enter couchDB id
env.CouchPassword) //enter couchDB pw

//NewCouchDBhistMgr should have automatically created the database, let's make sure it has been created
//Retrieve the info for the new database and make sure the name matches
dbResp, _, errdb := histMgr.couchDB.GetDatabaseInfo()
testutil.AssertNoError(t, errdb, fmt.Sprintf("Error when trying to retrieve database information"))
testutil.AssertEquals(t, dbResp.DbName, env.couchDatabaseName)
testutil.AssertEquals(t, dbResp.DbName, env.CouchDatabaseName)

//Call NewCouchDBhistMgr again, this time the database will already exist from last time
histMgr2 := NewCouchDBHistMgr(
env.couchDBAddress, //couchDB Address
env.couchDatabaseName, //couchDB db name
env.couchUsername, //enter couchDB id
env.couchPassword) //enter couchDB pw
env.CouchDBAddress, //couchDB Address
env.CouchDatabaseName, //couchDB db name
env.CouchUsername, //enter couchDB id
env.CouchPassword) //enter couchDB pw

//Retrieve the info for the database again, and make sure the name still matches
dbResp2, _, errdb2 := histMgr2.couchDB.GetDatabaseInfo()
testutil.AssertNoError(t, errdb2, fmt.Sprintf("Error when trying to retrieve database information"))
testutil.AssertEquals(t, dbResp2.DbName, env.couchDatabaseName)
testutil.AssertEquals(t, dbResp2.DbName, env.CouchDatabaseName)

}
}

func TestConstructCompositeKey(t *testing.T) {
compositeKey := constructCompositeKey("ns1", "key1", 1, 1)
compositeKey := helper.ConstructCompositeKey("ns1", "key1", 1, 1)

var compositeKeySep = []byte{0x00}
var strKeySep = string(compositeKeySep)
Expand All @@ -86,24 +87,29 @@ func TestConstructCompositeKey(t *testing.T) {
//TestSavepoint tests the recordSavepoint and GetBlockNumfromSavepoint methods for recording and reading a savepoint document
func TestSavepoint(t *testing.T) {

//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../../../../peer")
logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())

if ledgerconfig.IsHistoryDBEnabled() == true {

env := newTestEnvHistoryCouchDB(t, "history-test")
env.cleanup() //cleanup at the beginning to ensure the database doesn't exist already
defer env.cleanup() //and cleanup at the end
env := newTestEnvCouchDB(t, "history-test")
env.cleanupCouchDB() //cleanup at the beginning to ensure the database doesn't exist already
defer env.cleanupCouchDB() //and cleanup at the end

logger.Debugf("===HISTORYDB=== env.couchDBAddress: %v , env.couchDatabaseName: %v env.couchUsername: %v env.couchPassword: %v\n",
env.couchDBAddress, env.couchDatabaseName, env.couchUsername, env.couchPassword)
env.CouchDBAddress, env.CouchDatabaseName, env.CouchUsername, env.CouchPassword)

histMgr := NewCouchDBHistMgr(
env.couchDBAddress, //couchDB Address
env.couchDatabaseName, //couchDB db name
env.couchUsername, //enter couchDB id
env.couchPassword) //enter couchDB pw
env.CouchDBAddress, //couchDB Address
env.CouchDatabaseName, //couchDB db name
env.CouchUsername, //enter couchDB id
env.CouchPassword) //enter couchDB pw

// read the savepoint
blockNum, err := histMgr.GetBlockNumFromSavepoint()
testutil.AssertEquals(t, blockNum, 0)
testutil.AssertEquals(t, blockNum, uint64(0))

// record savepoint
blockNo := uint64(5)
Expand All @@ -117,16 +123,18 @@ func TestSavepoint(t *testing.T) {
}
}

/*
//History Database commit and read is being tested with kv_ledger_test.go.
//This test will push some of the testing down into history itself
func TestHistoryDatabaseCommit(t *testing.T) {
//call a helper method to load the core.yaml
testutil.SetupCoreYAMLConfig("./../../../peer")
logger.Debugf("===HISTORYDB=== TestHistoryDatabaseCommit IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
testutil.SetupCoreYAMLConfig("./../../../../../../peer")
logger.Debugf("===HISTORYDB=== TestHistoryDatabaseAutoCreate IsCouchDBEnabled()value: %v , IsHistoryDBEnabled()value: %v\n",
ledgerconfig.IsCouchDBEnabled(), ledgerconfig.IsHistoryDBEnabled())
if ledgerconfig.IsHistoryDBEnabled() == true {
//TODO Build the necessary infrastructure so that history can be tested iwthout ledger
//TODO Build the necessary infrastructure so that history can be tested without ledger
}
}
*/
3 changes: 2 additions & 1 deletion ...history/couchdb_history_query_executer.go → ...histmgr/couchdb_history_query_executer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package history
package couchdbhistmgr

import "github.com/hyperledger/fabric/core/ledger"

Expand Down Expand Up @@ -47,6 +47,7 @@ func (itr *qHistoryItr) Next() (ledger.QueryResult, error) {
return nil, nil
}
//TODO Returning blockNumTrannum as TxID for now but eventually will return txID instead
//TODO This includes the seperator before the blocknum and tran num: aka(see test): strKeySep+"1"+strKeySep+"1"
return &ledger.KeyModification{TxID: historicValue.blockNumTranNum, Value: historicValue.value}, nil
}

Expand Down
47 changes: 24 additions & 23 deletions core/ledger/history/pkg_test.go → ...istory/histmgr/couchdbhistmgr/pkg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package history
package couchdbhistmgr

import (
"testing"
Expand All @@ -23,33 +23,34 @@ import (
"github.com/hyperledger/fabric/core/ledger/util/couchdb"
)

//Complex setup to test the use of couch in ledger
type testEnvHistoryCouchDB struct {
couchDBAddress string
couchDatabaseName string
couchUsername string
couchPassword string
//TestEnvCouchDB Complex setup to test the use of couchDB in ledger
type testEnvCouchDB struct {
CouchDBAddress string
CouchDatabaseName string
CouchUsername string
CouchPassword string
}

func newTestEnvHistoryCouchDB(t testing.TB, dbName string) *testEnvHistoryCouchDB {

couchDBDef := ledgerconfig.GetCouchDBDefinition()

return &testEnvHistoryCouchDB{
couchDBAddress: couchDBDef.URL,
couchDatabaseName: dbName,
couchUsername: couchDBDef.Username,
couchPassword: couchDBDef.Password,
}
}

func (env *testEnvHistoryCouchDB) cleanup() {

//CleanupCouchDB to clean up the test of couchDB in ledger
func (env *testEnvCouchDB) cleanupCouchDB() {
//create a new connection
couchInstance, err := couchdb.CreateCouchInstance(env.couchDBAddress, env.couchUsername, env.couchPassword)
couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, env.couchDatabaseName)
couchInstance, err := couchdb.CreateCouchInstance(env.CouchDBAddress, env.CouchUsername, env.CouchPassword)
couchDB, err := couchdb.CreateCouchDatabase(*couchInstance, env.CouchDatabaseName)
if err == nil {
//drop the test database if it already existed
couchDB.DropDatabase()
}
}

//NewTestEnvCouchDB to clean up the test of couchDB in ledger
func newTestEnvCouchDB(t testing.TB, dbName string) *testEnvCouchDB {

couchDBDef := ledgerconfig.GetCouchDBDefinition()

return &testEnvCouchDB{
CouchDBAddress: couchDBDef.URL,
CouchDatabaseName: dbName,
CouchUsername: couchDBDef.Username,
CouchPassword: couchDBDef.Password,
}
}
2 changes: 1 addition & 1 deletion core/ledger/history/histmgmt.go → ...edger/kvledger/history/histmgr/histmgr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package history
package histmgr

import "github.com/hyperledger/fabric/protos/common"
import "github.com/hyperledger/fabric/core/ledger"
Expand Down
Loading

0 comments on commit 74eeb66

Please sign in to comment.