build: Update for Mac M1 support (#980)

.circleci/config.yml

@@ -17,7 +17,7 @@ workflows:
           name: test_with_go_<< matrix.go_version >>
           matrix: &go-version-matrix
             parameters:
-              go_version: ["1.14.7", "1.15.15"]
+              go_version: ["1.17.9"]
 
   circleci_build_and_test_nightly:
     triggers:
@@ -32,7 +32,7 @@ workflows:
           context: slack-secrets
           matrix: &go-version-matrix
             parameters:
-              go_version: ["1.14.7", "1.15.15"]
+              go_version: ["1.17.9"]
       - indexer_vs_algod_nightly:
           name: nightly_test_indexer_vs_algod
           context: slack-secrets
@@ -120,7 +120,7 @@ commands:
 
       - run: 
           name: Install golint
-          command: go get -u golang.org/x/lint/golint
+          command: go install golang.org/x/lint/golint
 
   run_tests:
     steps:

Makefile

@@ -3,6 +3,12 @@ VERSION		:= $(shell $(SRCPATH)/mule/scripts/compute_build_number.sh)
 OS_TYPE		?= $(shell $(SRCPATH)/mule/scripts/ostype.sh)
 ARCH			?= $(shell $(SRCPATH)/mule/scripts/archtype.sh)
 PKG_DIR		= $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/$(VERSION)
+ifeq ($(OS_TYPE), darwin)
+ifeq ($(ARCH), arm64)
+export CPATH=/opt/homebrew/include
+export LIBRARY_PATH=/opt/homebrew/lib
+endif
+endif
 
 # TODO: ensure any additions here are mirrored in misc/release.py
 GOLDFLAGS += -X github.com/algorand/indexer/version.Hash=$(shell git log -n 1 --pretty="%H")
@@ -28,7 +34,7 @@ idb/postgres/internal/schema/setup_postgres_sql.go:	idb/postgres/internal/schema
 	cd idb/postgres/internal/schema && go generate
 
 idb/mocks/IndexerDb.go:	idb/idb.go
-	go get github.com/vektra/mockery/.../
+	go install github.com/vektra/mockery/.../
 	cd idb && mockery -name=IndexerDb
 
 # check that all packages (except tests) compile

api/Makefile

@@ -23,7 +23,6 @@ indexer.oas3.yml:	.3tmp.json
 	curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./indexer.oas2.json -o .3tmp.json
 
 oapi-codegen:	.PHONY
-	# This weirdness is to workaround incompatible dependencies in our go.mod
-	cd /tmp && GO111MODULE=on go get "github.com/algorand/oapi-codegen/cmd/[email protected]"
+	go install "github.com/algorand/oapi-codegen/[email protected]"
 
 .PHONY:

api/handlers.go

@@ -1059,7 +1059,7 @@ func (si *ServerImplementation) fetchBlock(ctx context.Context, round uint64) (g
 			Seed:              blockHeader.Seed[:],
 			Timestamp:         uint64(blockHeader.TimeStamp),
 			Transactions:      nil,
-			TransactionsRoot:  blockHeader.TxnRoot[:],
+			TransactionsRoot:  blockHeader.TxnCommitments.NativeSha512_256Commitment[:],
 			TxnCounter:        uint64Ptr(blockHeader.TxnCounter),
 			UpgradeState:      &upgradeState,
 			UpgradeVote:       &upgradeVote,

api/handlers_e2e_test.go

@@ -1139,7 +1139,7 @@ func TestAccountClearsNonUTF8(t *testing.T) {
 
 	assetName := "valid"
 	//url := "https://my.embedded.\000.null.asset"
-	urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leQ==")
+	urlBytes, _ := base64.StdEncoding.DecodeString("8J+qmSBNb25leSwgd2FudAo=")
 	url := string(urlBytes)
 	unitName := "asset\rwith\nnon-printable\tcharacters"
 	createAsset := test.MakeAssetConfigTxn(0, 100, 0, false, unitName, assetName, url, test.AccountA)

cmd/block-generator/docker/Dockerfile-generator

@@ -1,4 +1,4 @@
-FROM golang:1.13.12-alpine
+FROM golang:1.17.9-alpine
 
 # Build and run the block generator
 

cmd/block-generator/docker/Dockerfile-indexer

@@ -1,4 +1,4 @@
-FROM golang:1.13.12-alpine
+FROM golang:1.17.9-alpine
 
 # Build and run indexer
 RUN mkdir /work

cmd/block-generator/generator/generate.go

@@ -334,13 +334,13 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error {
 	numTxnForBlock := g.txnForRound(round)
 
 	header := bookkeeping.BlockHeader{
-		Round:       basics.Round(g.round),
-		Branch:      bookkeeping.BlockHash{},
-		Seed:        committee.Seed{},
-		TxnRoot:     crypto.Digest{},
-		TimeStamp:   g.timestamp,
-		GenesisID:   g.genesisID,
-		GenesisHash: g.genesisHash,
+		Round:          basics.Round(g.round),
+		Branch:         bookkeeping.BlockHash{},
+		Seed:           committee.Seed{},
+		TxnCommitments: bookkeeping.TxnCommitments{NativeSha512_256Commitment: crypto.Digest{}},
+		TimeStamp:      g.timestamp,
+		GenesisID:      g.genesisID,
+		GenesisHash:    g.genesisHash,
 		RewardsState: bookkeeping.RewardsState{
 			FeeSink:                   g.feeSink,
 			RewardsPool:               g.rewardsPool,

cmd/import-validator/core/service.go

@@ -188,23 +188,27 @@ func getModifiedState(l *ledger.Ledger, block *bookkeeping.Block) (map[basics.Ad
 	return modifiedAccounts, modifiedResources, nil
 }
 
-func normalizeAccountResource(r ledgercore.AccountResource) ledgercore.AccountResource {
+// Converts an `ledgercore.AppResource` to a `ledgercore.AccountResource`.
+// Returns a copy with the `GlobalState` and `KeyValue` fields normalized if needed.
+func convertAppResource(r ledgercore.AppResource) (ar ledgercore.AccountResource) {
+	ar.AppLocalState = r.AppLocalState
+	ar.AppParams = r.AppParams
 	if (r.AppParams != nil) && (len(r.AppParams.GlobalState) == 0) {
 		// Make a copy of `AppParams` to avoid modifying ledger's storage.
 		appParams := new(basics.AppParams)
 		*appParams = *r.AppParams
 		appParams.GlobalState = nil
-		r.AppParams = appParams
+		ar.AppParams = appParams
 	}
 	if (r.AppLocalState != nil) && (len(r.AppLocalState.KeyValue) == 0) {
 		// Make a copy of `AppLocalState` to avoid modifying ledger's storage.
 		appLocalState := new(basics.AppLocalState)
 		*appLocalState = *r.AppLocalState
 		appLocalState.KeyValue = nil
-		r.AppLocalState = appLocalState
+		ar.AppLocalState = appLocalState
 	}
 
-	return r
+	return ar
 }
 
 func checkModifiedState(db *postgres.IndexerDb, l *ledger.Ledger, block *bookkeeping.Block, addresses map[basics.Address]struct{}, resources map[basics.Address]map[ledger.Creatable]struct{}) error {
@@ -255,14 +259,31 @@ func checkModifiedState(db *postgres.IndexerDb, l *ledger.Ledger, block *bookkee
 			resourcesForAddress := make(map[ledger.Creatable]ledgercore.AccountResource)
 			resourcesAlgod[address] = resourcesForAddress
 			for creatable := range creatables {
-				var resource ledgercore.AccountResource
-				resource, err1 =
-					l.LookupResource(block.Round(), address, creatable.Index, creatable.Type)
-				if err1 != nil {
-					err1 = fmt.Errorf("checkModifiedState() lookup resource err1: %w", err1)
+				switch creatable.Type {
+				case basics.AssetCreatable:
+					var assetResource ledgercore.AssetResource
+					assetResource, err1 = l.LookupAsset(block.Round(), address, basics.AssetIndex(creatable.Index))
+					if err1 != nil {
+						err1 = fmt.Errorf("checkModifiedState() lookup asset resource err1: %w", err1)
+						return
+					}
+					ar := ledgercore.AccountResource{
+						AssetParams:  assetResource.AssetParams,
+						AssetHolding: assetResource.AssetHolding,
+					}
+					resourcesForAddress[creatable] = ar
+				case basics.AppCreatable:
+					var appResource ledgercore.AppResource
+					appResource, err1 = l.LookupApplication(block.Round(), address, basics.AppIndex(creatable.Index))
+					if err1 != nil {
+						err1 = fmt.Errorf("checkModifiedState() lookup application resource err1: %w", err1)
+						return
+					}
+					resourcesForAddress[creatable] = convertAppResource(appResource)
+				default:
+					err1 = fmt.Errorf("checkModifiedState() unexpected creatable type %v", creatable.Type)
 					return
 				}
-				resourcesForAddress[creatable] = normalizeAccountResource(resource)
 			}
 		}
 	}()

go.mod

@@ -1,14 +1,14 @@
 module github.com/algorand/indexer
 
-go 1.13
+go 1.17
 
 replace github.com/algorand/go-algorand => ./third_party/go-algorand
 
 require (
 	github.com/algorand/go-algorand v0.0.0-20220211161928-53b157beb10f
 	github.com/algorand/go-algorand-sdk v1.9.1
-	github.com/algorand/go-codec/codec v1.1.7
-	github.com/algorand/oapi-codegen v1.3.5-algorand5
+	github.com/algorand/go-codec/codec v1.1.8
+	github.com/algorand/oapi-codegen v1.3.7
 	github.com/davecgh/go-spew v1.1.1
 	github.com/getkin/kin-openapi v0.22.0
 	github.com/jackc/pgconn v1.10.0
@@ -19,13 +19,91 @@ require (
 	github.com/orlangure/gnomock v0.12.0
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.10.0
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+)
+
+require (
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/algorand/falcon v0.0.0-20220130164023-c9e1d466f123 // indirect
+	github.com/algorand/go-deadlock v0.2.2 // indirect
+	github.com/algorand/go-sumhash v0.1.0 // indirect
+	github.com/algorand/msgp v1.1.51 // indirect
+	github.com/algorand/websocket v1.4.5 // indirect
+	github.com/aws/aws-sdk-go v1.30.19 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cpuguy83/go-md2man v1.0.10 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 // indirect
+	github.com/dchest/siphash v1.2.1 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v1.13.1 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
+	github.com/gorilla/mux v1.7.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.8.1 // indirect
+	github.com/jackc/puddle v1.1.3 // indirect
+	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/labstack/gommon v0.3.0 // indirect
+	github.com/lib/pq v1.10.2 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-sqlite3 v1.10.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/miekg/dns v1.1.27 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/olivere/elastic v6.2.14+incompatible // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/pelletier/go-toml v1.4.0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.25.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/russross/blackfriday v1.5.2 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasttemplate v1.2.1 // indirect
 	github.com/vektra/mockery v1.1.2 // indirect
+	go.uber.org/atomic v1.6.0 // indirect
+	go.uber.org/multierr v1.5.0 // indirect
+	go.uber.org/zap v1.15.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	golang.org/x/tools v0.1.10 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.51.0 // indirect
+	gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )