General refactoring

cmd/run_cmd.go

@@ -4,10 +4,9 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 
-	"github.com/pkg/errors"
-
 	"github.com/cortze/ipfs-cid-hoarder/pkg/config"
 	"github.com/cortze/ipfs-cid-hoarder/pkg/hoarder"
+	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"

pkg/config/config.go

@@ -15,8 +15,7 @@ const (
 	RandomSource   = "random-content-gen"
 	TextFileSource = "text-file"
 	BitswapSource  = "bitswap"
-
-	RandomContent = "random"
+	RandomContent  = "random"
 )
 
 // Harcoded variables for the tool's profiling
@@ -64,12 +63,15 @@ type Config struct {
 
 // Init takes the command line argumenst from the urfave/cli context and composes the configuration
 func NewConfig(ctx *cli.Context) (*Config, error) {
-	c := &Config{}
-	c.apply(ctx)
 
 	// TODO: work on reading config file from custom path/file (reproducibility)
 	// 		 export the current conf into a file?
+	//Read file function here
+
+	c := &Config{}
+	c.apply(ctx)
 	return c, nil
+
 }
 
 // apply parses the arguments readed from cli.Context

pkg/hoarder/cid-tracker.go

@@ -16,11 +16,6 @@ import (
 	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
 )
 
-type CidSource interface {
-	GetNewCid() ([]byte, cid.Cid, error)
-	Type() string
-}
-
 // CidTracker composes the basic ojbject that generates and publishes the set of CIDs defined in the configuration
 type CidTracker struct {
 	ctx context.Context

pkg/hoarder/cid_source.go

@@ -0,0 +1,142 @@
+package hoarder
+
+import (
+	"bufio"
+	"io"
+	"math/rand"
+	"os"
+	"strings"
+
+	"github.com/ipfs/go-cid"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+
+	mh "github.com/multiformats/go-multihash"
+)
+
+var DefCIDContLen = 1024 // 1KB
+
+type CidSource interface {
+	GetNewCid() ([]byte, cid.Cid, error)
+	Type() string
+}
+
+//Read CIDs and their content from a file. The struct will contain the file pointer that it's opened along with a pointer to a scanner
+//struct. When you want to access a new CID from the file the GetNewCid() function must be called. The scanner keeps the state and will
+//read the file until the end.
+type FileCIDSource struct {
+	filename string
+	file     *os.File
+	scanner  *bufio.Scanner
+}
+
+type BitswapCIDSource struct {
+}
+
+type RandomCidGen struct {
+	contentSize int
+}
+
+func NewRandomCidGen(contentSize int) *RandomCidGen {
+	return &RandomCidGen{
+		contentSize: contentSize,
+	}
+}
+
+//Creates a new:
+//
+// 	type FileCIDSource struct {
+//		filename string
+//		file     *os.File
+//		scanner  *bufio.Scanner
+// 	}
+//If the file cannot be opened it returns the corresponding error.
+func newFileCIDSource(filename string) (*FileCIDSource, error) {
+	file, err := os.Open(filename)
+	defer file.Close()
+	if err != nil {
+		return nil, errors.Wrap(err, "opening CID file")
+	}
+
+	scanner := bufio.NewScanner(file)
+	scanner.Split(bufio.ScanLines)
+
+	return &FileCIDSource{
+		filename: filename,
+		file:     file,    //this a pointer to a file
+		scanner:  scanner, //this a pointer to the scanner
+	}, nil
+}
+
+func newBitswapCIDSource() *BitswapCIDSource {
+	return &BitswapCIDSource{}
+}
+
+func (g *RandomCidGen) GetNewCid() ([]byte, cid.Cid, error) {
+	return genRandomContent(g.contentSize)
+}
+
+func (g *RandomCidGen) Type() string {
+	return "random-content-gen"
+}
+
+//Scans the file and reads each line of the file. When it reaches the end of file it returns an EOF error. If another error occurs
+//it returns the error. The end of file error means that the file was read successfully.
+func (file_cid_source *FileCIDSource) GetNewCid() ([]byte, cid.Cid, error) {
+
+	error_flag := file_cid_source.scanner.Scan()
+	if !error_flag {
+		//scanner.Err() returns a nil when error_flag = false and the error is an EOF error. Else it return the error normally.
+		err := file_cid_source.scanner.Err()
+		if err != io.EOF {
+			return nil, cid.Undef, err
+		}
+		return nil, cid.Undef, io.EOF
+	}
+	temp := strings.Fields(file_cid_source.scanner.Text())
+
+	cid_temp, err := cid.Parse(temp[0])
+	if err != nil {
+		return nil, cid.Undef, errors.Wrap(err, " while parsing cid")
+	}
+	return []byte(temp[1]), cid_temp, nil
+}
+
+func (file_cid_source *FileCIDSource) Type() string {
+	return "text-file"
+}
+
+func (bitswap_cid_source *BitswapCIDSource) GetNewCid() ([]byte, cid.Cid, error) {
+	//TODO function that reads bitswap content
+	return read_content_from_file()
+}
+
+func (bitswap_cid_source *BitswapCIDSource) Type() string {
+	return "bitswap"
+}
+
+// TODO: is it worth keeping the content?
+// getRandomContent returns generates an array of random bytes with the given size and the composed CID of the content
+func genRandomContent(byteLen int) ([]byte, cid.Cid, error) {
+	// generate random bytes
+	content := make([]byte, byteLen)
+	rand.Read(content)
+
+	//TODO do we have to have different CID types?
+	// configure the type of CID that we want
+	pref := cid.Prefix{
+		Version:  1,
+		Codec:    cid.Raw,
+		MhType:   mh.SHA2_256,
+		MhLength: -1,
+	}
+
+	// get the CID of the content we just generated
+	contID, err := pref.Sum(content)
+	if err != nil {
+		return content, cid.Cid{}, errors.Wrap(err, "composing CID")
+	}
+
+	log.Infof("generated new CID %s", contID.Hash().B58String())
+	return content, contID, nil
+}

pkg/hoarder/hoarder.go

@@ -82,7 +82,10 @@ func NewCidHoarder(ctx context.Context, conf *config.Config) (*CidHoarder, error
 	cidPinger := NewCidPinger(ctx, &studyWG, h, db, reqInterval, rounds, conf.Workers)
 
 	// ----- Generate the CidTracker -----
-	cidSource := NewRandomCidGen(conf.CidContentSize)
+	cidSource, err := find_cid_source(conf)
+	if err != nil {
+		return nil, errors.Wrap(err, " error generating the CID Tracker")
+	}
 	studyWG.Add(1)
 	cidTracker, err := NewCidTracker(ctx, &studyWG, h, db, cidSource, cidPinger, conf.K, conf.CidNumber, conf.Workers, reqInterval, studyDuration)
 	if err != nil {
@@ -120,3 +123,27 @@ func (c *CidHoarder) Run() error {
 
 	return nil
 }
+
+//Generates the cid source given a specific config struct. E.g. a randomly generated cid source uses the Random_Cid_Gen struct, this function must return that.
+//The options for the cid source are defined in this enum like const value.
+//
+//  const (
+//	RandomSource   = "random-content-gen"
+//	TextFileSource = "text-file"
+//	BitswapSource  = "bitswap"
+//	RandomContent  = "random"
+//  )
+//
+func find_cid_source(conf *config.Config) (CidSource, error) {
+	switch conf.CidSource {
+	case config.TextFileSource:
+		return newFileCIDSource(), nil
+	case config.RandomContent:
+		return NewRandomCidGen(conf.CidContentSize), nil
+	case config.BitswapSource:
+		return newBitswapCIDSource(), nil
+
+	default:
+		return nil, errors.New("Could not figure out cid source")
+	}
+}