Merge pull request #3 from JackalLabs/marston/acacia-compat

Compat: v4 Acacia

Makefile

@@ -6,6 +6,12 @@ export GO111MODULE = on
 
 all: lint test-unit
 
+install:
+	@go install
+
+.PHONY: install
+
+
 ###############################################################################
 ###                          Tools & Dependencies                           ###
 ###############################################################################

api/data.go

@@ -1,18 +1,16 @@
 package api
 
 import (
-	"encoding/json"
 	"net/http"
 
 	"github.com/JackalLabs/sequoia/api/types"
 	"github.com/JackalLabs/sequoia/file_system"
-	"github.com/dgraph-io/badger/v4"
 	"github.com/rs/zerolog/log"
 )
 
-func DumpDBHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
+func DumpDBHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
-		dump, err := file_system.Dump(db)
+		dump, err := f.Dump()
 		if err != nil {
 			v := types.ErrorResponse{
 				Error: err.Error(),

api/file_handler.go

@@ -1,115 +1,145 @@
 package api
 
 import (
-	"encoding/json"
+	"context"
+	"encoding/hex"
 	"fmt"
 	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/JackalLabs/sequoia/proofs"
+
+	"github.com/desmos-labs/cosmos-go-wallet/wallet"
 
 	"github.com/JackalLabs/sequoia/api/types"
 	"github.com/JackalLabs/sequoia/file_system"
-	"github.com/JackalLabs/sequoia/queue"
-	"github.com/dgraph-io/badger/v4"
 	"github.com/gorilla/mux"
 	storageTypes "github.com/jackalLabs/canine-chain/v3/x/storage/types"
 	"github.com/rs/zerolog/log"
 )
 
-const MaxFileSize = 32 << 30
+// const MaxFileSize = 32 << 30
+const MaxFileSize = 0
+
+func handleErr(err error, w http.ResponseWriter, code int) {
+	v := types.ErrorResponse{
+		Error: err.Error(),
+	}
+	w.WriteHeader(code)
+	err = json.NewEncoder(w).Encode(v)
+	if err != nil {
+		log.Error().Err(err)
+	}
+}
 
-func PostFileHandler(db *badger.DB, q *queue.Queue, address string) func(http.ResponseWriter, *http.Request) {
+func PostFileHandler(fio *file_system.FileSystem, prover *proofs.Prover, wl *wallet.Wallet, chunkSize int64) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
 		err := req.ParseMultipartForm(MaxFileSize) // MAX file size lives here
 		if err != nil {
-			v := types.ErrorResponse{
-				Error: err.Error(),
-			}
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(v)
-			if err != nil {
-				log.Error().Err(err)
-			}
+			handleErr(fmt.Errorf("cannot parse form %w", err), w, http.StatusBadRequest)
 			return
 		}
 		sender := req.Form.Get("sender")
+		merkleString := req.Form.Get("merkle")
+		merkle, err := hex.DecodeString(merkleString)
+		if err != nil {
+			handleErr(fmt.Errorf("cannot parse merkle: %w", err), w, http.StatusBadRequest)
+			return
+		}
 
-		file, _, err := req.FormFile("file") // Retrieve the file from form data
+		startBlockString := req.Form.Get("start")
+		startBlock, err := strconv.ParseInt(startBlockString, 10, 64)
 		if err != nil {
-			v := types.ErrorResponse{
-				Error: err.Error(),
-			}
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(v)
-			if err != nil {
-				log.Error().Err(err)
-			}
+			handleErr(fmt.Errorf("cannot parse start block: %w", err), w, http.StatusBadRequest)
 			return
 		}
 
-		merkle, fid, cid, size, err := file_system.WriteFile(db, file, sender, address, "")
+		file, fh, err := req.FormFile("file") // Retrieve the file from form data
 		if err != nil {
-			v := types.ErrorResponse{
-				Error: err.Error(),
-			}
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(v)
-			if err != nil {
-				log.Error().Err(err)
-			}
+			handleErr(fmt.Errorf("cannot get file from form: %w", err), w, http.StatusBadRequest)
+			return
 		}
 
-		msg := storageTypes.NewMsgPostContract(
-			address,
-			sender,
-			fmt.Sprintf("%d", size),
-			fid,
-			merkle,
-		)
+		readSize := fh.Size
+		if readSize == 0 {
+			handleErr(fmt.Errorf("file cannot be empty"), w, http.StatusBadRequest)
+			return
+		}
 
-		if err := msg.ValidateBasic(); err != nil {
-			v := types.ErrorResponse{
-				Error: err.Error(),
-			}
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(v)
-			if err != nil {
-				log.Error().Err(err)
-			}
+		cl := storageTypes.NewQueryClient(wl.Client.GRPCConn)
+		queryParams := storageTypes.QueryFile{
+			Merkle: merkle,
+			Owner:  sender,
+			Start:  startBlock,
+		}
+		res, err := cl.File(context.Background(), &queryParams)
+		if err != nil {
+			handleErr(fmt.Errorf("failed to find file on chain: %w", err), w, http.StatusInternalServerError)
+			return
 		}
 
-		m, wg := q.Add(msg)
+		f := res.File
 
-		wg.Wait() // wait for queue to process
+		if readSize != f.FileSize {
+			handleErr(fmt.Errorf("cannot accept form file that doesn't match the chain data %d != %d", readSize, f.FileSize), w, http.StatusInternalServerError)
+			return
+		}
 
-		if m.Error() != nil {
-			v := types.ErrorResponse{
-				Error: m.Error().Error(),
-			}
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(v)
-			if err != nil {
-				log.Error().Err(err)
+		if hex.EncodeToString(f.Merkle) != merkleString {
+			handleErr(fmt.Errorf("cannot accept file that doesn't match the chain data %x != %x", f.Merkle, merkle), w, http.StatusInternalServerError)
+			return
+		}
+
+		if len(f.Proofs) == int(f.MaxProofs) {
+			if !f.ContainsProver(wl.AccAddress()) {
+				handleErr(fmt.Errorf("cannot accept file that I cannot claim"), w, http.StatusInternalServerError)
+				return
 			}
 		}
 
+		size, err := fio.WriteFile(file, merkle, sender, startBlock, wl.AccAddress(), chunkSize)
+		if err != nil {
+			handleErr(fmt.Errorf("failed to write file to disk: %w", err), w, http.StatusInternalServerError)
+			return
+		}
+
+		if int64(size) != f.FileSize {
+			handleErr(fmt.Errorf("cannot accept file that doesn't match the chain data %d != %d", int64(size), f.FileSize), w, http.StatusInternalServerError)
+			return
+		}
+
 		resp := types.UploadResponse{
-			CID: cid,
-			FID: fid,
+			Merkle: merkle,
+			Owner:  sender,
+			Start:  startBlock,
 		}
 
 		err = json.NewEncoder(w).Encode(resp)
 		if err != nil {
-			log.Error().Err(err)
+			log.Error().Err(fmt.Errorf("can't encode json : %w", err))
 		}
+
+		_ = prover.PostProof(merkle, sender, startBlock, startBlock, time.Now())
 	}
 }
 
-func DownloadFileHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
+func DownloadFileHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
 		vars := mux.Vars(req)
 
-		fid := vars["fid"]
+		merkleString := vars["merkle"]
+		merkle, err := hex.DecodeString(merkleString)
+		if err != nil {
+			v := types.ErrorResponse{
+				Error: err.Error(),
+			}
+			w.WriteHeader(http.StatusInternalServerError)
+			_ = json.NewEncoder(w).Encode(v)
+
+		}
 
-		file, err := file_system.GetFileDataByFID(db, fid)
+		file, err := f.GetFileDataByMerkle(merkle)
 		if err != nil {
 			v := types.ErrorResponse{
 				Error: err.Error(),

api/files.go

@@ -1,18 +1,17 @@
 package api
 
 import (
-	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/JackalLabs/sequoia/api/types"
 	"github.com/JackalLabs/sequoia/file_system"
-	"github.com/dgraph-io/badger/v4"
 	"github.com/rs/zerolog/log"
 )
 
-func ListFilesHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
+func ListFilesHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
-		files, err := file_system.ListFiles(db)
+		merkles, _, _, err := f.ListFiles()
 		if err != nil {
 			v := types.ErrorResponse{
 				Error: err.Error(),
@@ -25,9 +24,14 @@ func ListFilesHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
 			return
 		}
 
+		mm := make([]string, len(merkles))
+		for i, merkle := range merkles {
+			mm[i] = fmt.Sprintf("%x", merkle)
+		}
+
 		f := types.ListResponse{
-			Files: files,
-			Count: len(files),
+			Files: mm,
+			Count: len(mm),
 		}
 
 		err = json.NewEncoder(w).Encode(f)
@@ -37,9 +41,9 @@ func ListFilesHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
 	}
 }
 
-func LegacyListFilesHandler(db *badger.DB) func(http.ResponseWriter, *http.Request) {
+func LegacyListFilesHandler(f *file_system.FileSystem) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, req *http.Request) {
-		files, err := file_system.ListFiles(db)
+		merkles, owners, _, err := f.ListFiles()
 		if err != nil {
 			v := types.ErrorResponse{
 				Error: err.Error(),
@@ -52,12 +56,12 @@ func LegacyListFilesHandler(db *badger.DB) func(http.ResponseWriter, *http.Reque
 			return
 		}
 
-		cids := make([]types.LegacyAPIListValue, len(files))
+		cids := make([]types.LegacyAPIListValue, len(merkles))
 
-		for i, file := range files {
+		for i, m := range merkles {
 			cids[i] = types.LegacyAPIListValue{
-				CID: file,
-				FID: "",
+				CID: fmt.Sprintf("%x", m),
+				FID: owners[i],
 			}
 		}
 

api/index.go

@@ -1,7 +1,6 @@
 package api
 
 import (
-	"encoding/json"
 	"net/http"
 
 	"github.com/JackalLabs/sequoia/api/types"

api/server.go

@@ -1,18 +1,28 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
-	"github.com/JackalLabs/sequoia/queue"
+	"github.com/JackalLabs/sequoia/file_system"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/JackalLabs/sequoia/proofs"
+	"github.com/rs/zerolog/log"
+
 	"github.com/desmos-labs/cosmos-go-wallet/wallet"
-	"github.com/dgraph-io/badger/v4"
 	"github.com/gorilla/mux"
 )
+import jsoniter "github.com/json-iterator/go"
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
 
 type API struct {
 	port int64
+	srv  *http.Server
 }
 
 func NewAPI(port int64) *API {
@@ -21,29 +31,38 @@ func NewAPI(port int64) *API {
 	}
 }
 
-func (a *API) Serve(db *badger.DB, q *queue.Queue, wallet *wallet.Wallet) {
+func (a *API) Close() error {
+	return a.srv.Close()
+}
+
+func (a *API) Serve(f *file_system.FileSystem, p *proofs.Prover, wallet *wallet.Wallet, chunkSize int64) {
 	r := mux.NewRouter()
 	r.HandleFunc("/", IndexHandler(wallet.AccAddress()))
-	r.HandleFunc("/upload", PostFileHandler(db, q, wallet.AccAddress()))
-	r.HandleFunc("/download/{fid}", DownloadFileHandler(db))
+	r.HandleFunc("/upload", PostFileHandler(f, p, wallet, chunkSize))
+	r.HandleFunc("/download/{fid}", DownloadFileHandler(f))
 
-	r.HandleFunc("/list", ListFilesHandler(db))
-	r.HandleFunc("/api/data/fids", LegacyListFilesHandler(db))
+	r.HandleFunc("/list", ListFilesHandler(f))
+	r.HandleFunc("/api/data/fids", LegacyListFilesHandler(f))
 
-	r.HandleFunc("/dump", DumpDBHandler(db))
+	r.HandleFunc("/dump", DumpDBHandler(f))
 
 	r.HandleFunc("/version", VersionHandler(wallet))
 
-	srv := &http.Server{
+	r.Handle("/metrics", promhttp.Handler())
+
+	a.srv = &http.Server{
 		Handler: r,
 		Addr:    fmt.Sprintf("0.0.0.0:%d", a.port),
 		// Good practice: enforce timeouts for servers you create!
 		WriteTimeout: 15 * time.Second,
 		ReadTimeout:  15 * time.Second,
 	}
 
-	err := srv.ListenAndServe()
+	log.Logger.Info().Msg(fmt.Sprintf("Sequoia API now listening on %s", a.srv.Addr))
+	err := a.srv.ListenAndServe()
 	if err != nil {
-		panic(err)
+		if !errors.Is(err, http.ErrServerClosed) {
+			panic(err)
+		}
 	}
 }