feat: add fs-repo-12-to-13

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 name: Tests
 
 env:
-  GO: 1.16
+  GO: 1.18
 
 on:
   push:
@@ -23,5 +23,5 @@ jobs:
       with:
         go-version: ${{ env.GO }}
 
-    - name: Make
+    - name: Test
       run: make -k test

Makefile

@@ -1,15 +1,17 @@
 GO111MODULE = on
 
 MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
+IGNORED_DIRS := $(shell cat ignored-migrations)
+ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS))
 
-.PHONY: all build clean cmd sharness test test_go
+.PHONY: all build clean cmd sharness test test_go test_12_to_13
 
 all: build
 
 show: $(MIG_DIRS)
 	@echo "$(MIG_DIRS)"
 
-build: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/build.fs-repo/') cmd
+build: $(subst fs-repo,build.fs-repo,$(ACTIVE_DIRS)) cmd
 	@echo OK
 
 build.%: MIGRATION=$*
@@ -24,9 +26,9 @@ fs-repo-migrations/fs-repo-migrations:
 sharness:
 	make -C sharness
 
-test: test_go sharness
+test: test_go sharness test_12_to_13
 
-clean: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/clean.fs-repo/')
+clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS))
 	@make -C sharness clean
 	@cd fs-repo-migrations && go clean
 	@echo OK
@@ -35,9 +37,12 @@ clean.%: MIGRATION=$*
 clean.%:
 	make -C $(MIGRATION) clean
 
-test_go: $(shell ls -d fs-repo-*-to-* | sed -e 's/fs-repo/test_go.fs-repo/')
+test_go: $(subst fs-repo,test_go.fs-repo,$(ACTIVE_DIRS))
 	@echo OK
 
 test_go.%: MIGRATION=$*
 test_go.%:
 	@cd $(MIGRATION)/migration && go test -mod=vendor
+
+test_12_to_13:
+	@cd fs-repo-12-to-13/not-sharness && ./test.sh

README.md

@@ -47,7 +47,8 @@ Here is the table showing which repo version corresponds to which Kubo version:
 |                 9 | 0.5.0 - 0.6.0    |
 |                10 | 0.6.0 - 0.7.0    |
 |                11 | 0.8.0 - 0.11.0   |
-|                12 | 0.12.0 - current |
+|                12 | 0.12.0 - 0.17.0  |
+|                13 | 0.18.0 - current |
 
 ### How to Run Migrations
 
@@ -80,6 +81,11 @@ git tag fs-repo-99-to-100/v1.0.1
 git push origin fs-repo-99-to-100/v1.0.1
 ```
 
+#### Ignoring legacy migrations
+
+We "archive" legacy migrations by adding them to `ignored-migrations` file.
+This keeps CI fast, and removes issues caused by old, with unsupported go code.
+
 ### Dependencies
 
 Dependencies must be vendored independently for each migration. Each migration is a separate go module with its own `vendor` directory (created with `go mod vendor` for that migration).  All migrations are built using `go build -mod=vendor` to ensure dependencies come from the module's `vendor` directory.

fs-repo-12-to-13/Makefile

@@ -0,0 +1,7 @@
+.PHONY: build clean
+
+build:
+	go build -mod=vendor
+
+clean:
+	go clean

fs-repo-12-to-13/atomicfile/atomicfile.go

@@ -0,0 +1,59 @@
+// Package atomicfile provides the ability to write a file with an eventual
+// rename on Close (using os.Rename). This allows for a file to always be in a
+// consistent state and never represent an in-progress write.
+//
+// NOTE: `os.Rename` may not be atomic on your operating system.
+package atomicfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// File behaves like os.File, but does an atomic rename operation at Close.
+type File struct {
+	*os.File
+	path string
+}
+
+// New creates a new temporary file that will replace the file at the given
+// path when Closed.
+func New(path string, mode os.FileMode) (*File, error) {
+	f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return nil, err
+	}
+	if err := os.Chmod(f.Name(), mode); err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &File{File: f, path: path}, nil
+}
+
+// Close the file replacing the configured file.
+func (f *File) Close() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.File.Name())
+		return err
+	}
+	if err := os.Rename(f.Name(), f.path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Abort closes the file and removes it instead of replacing the configured
+// file. This is useful if after starting to write to the file you decide you
+// don't want it anymore.
+func (f *File) Abort() error {
+	if err := f.File.Close(); err != nil {
+		os.Remove(f.Name())
+		return err
+	}
+	if err := os.Remove(f.Name()); err != nil {
+		return err
+	}
+	return nil
+}

fs-repo-12-to-13/go.mod

@@ -0,0 +1,5 @@
+module github.com/ipfs/fs-repo-migrations/fs-repo-12-to-13
+
+go 1.18
+
+require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-12-to-13/go.sum

@@ -0,0 +1,2 @@
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
+github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-12-to-13/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	mg12 "github.com/ipfs/fs-repo-migrations/fs-repo-12-to-13/migration"
+	migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
+)
+
+func main() {
+	m := mg12.Migration{}
+	migrate.Main(m)
+}

fs-repo-12-to-13/migration/config.go

@@ -0,0 +1,121 @@
+package mg12
+
+import (
+	log "github.com/ipfs/fs-repo-migrations/tools/stump"
+)
+
+// convertQuicAddrs converts quic multiaddrs to v1 and enables webtransport listener
+// https://github.com/ipfs/kubo/issues/9410
+// https://github.com/ipfs/kubo/issues/9292
+func convertQuicAddrs(confMap map[string]any) {
+	// run this first to avoid having both quic and quic-v1 webtransport addresses
+	runOnAllAddressFields(confMap, multiaddrPatternReplace(false, "/quic/webtransport", "/quic-v1/webtransport"))
+
+	runOnAllAddressFields(confMap, multiaddrPatternReplace(true, "/quic", "/quic-v1", "/p2p-circuit"))
+	runOnAllAddressFields(confMap, multiaddrPatternReplace(true, "/quic-v1", "/quic-v1/webtransport", "/p2p-circuit", "/webtransport"))
+}
+
+// convertRouting converts Routing.Type to implicit default
+// https://github.com/ipfs/kubo/pull/9475
+func convertRouting(confMap map[string]any) {
+	routing, _ := confMap["Routing"].(map[string]any)
+	if routing == nil {
+		log.Log("No Routing field in config, skipping")
+		return
+	}
+
+	routers, ok := routing["Routers"].(map[string]any)
+	if len(routers) > 0 {
+		log.Log("Custom Routing.Routers in config, skipping")
+		return
+	}
+	methods, ok := routing["Methods"].(map[string]any)
+	if len(methods) > 0 {
+		log.Log("Custom Routing.Methods in config, skipping")
+		return
+	}
+
+	rType, ok := routing["Type"].(string)
+	if !ok {
+		log.Log("No Routing.Type field in config, skipping")
+		return
+	}
+	if rType == "dht" || rType == "" {
+		delete(routing, "Type")
+	} else {
+		log.Log("Routing.Type settings is different than the old default, skipping")
+	}
+}
+
+// convertReprovider converts Reprovider to implicit defaults
+// https://github.com/ipfs/kubo/pull/9326
+func convertReprovider(confMap map[string]any) {
+	reprovider, _ := confMap["Reprovider"].(map[string]any)
+	if reprovider == nil {
+		log.Log("No Reprovider field in config, skipping")
+		return
+	}
+
+	interval, ok := reprovider["Interval"].(string)
+	if !ok {
+		log.Log("No Reprovider.Interval field in config, skipping")
+		return
+	}
+
+	strategy, ok := reprovider["Strategy"].(string)
+	if !ok {
+		log.Log("No Reprovider.Strategy field in config, skipping")
+		return
+	}
+
+	if interval == "12h" && strategy == "all" {
+		delete(reprovider, "Strategy")
+		delete(reprovider, "Interval")
+	} else {
+		log.Log("Reprovider settings are different than the old default, skipping")
+	}
+}
+
+// convertConnMgr converts Swarm.ConnMgr to implicit defaults
+// https://github.com/ipfs/kubo/pull/9467
+func convertConnMgr(confMap map[string]any) {
+	swarm, _ := confMap["Swarm"].(map[string]any)
+	if swarm == nil {
+		log.Log("No Swarm field in config, skipping")
+		return
+	}
+	connmgr, _ := swarm["ConnMgr"].(map[string]any)
+	if connmgr == nil {
+		log.Log("No Swarm.ConnMgr field in config, skipping")
+		return
+	}
+	cmType, ok := connmgr["Type"].(string)
+	if !ok {
+		log.Log("No Swarm.ConnMgr.Type field in config, skipping")
+		return
+	}
+	cmLowWater, ok := connmgr["LowWater"].(float64)
+	if !ok {
+		log.Log("No Swarm.ConnMgr.LowWater field in config, skipping")
+		return
+	}
+	cmHighWater, ok := connmgr["HighWater"].(float64)
+	if !ok {
+		log.Log("No Swarm.ConnMgr.HighWater field in config, skipping")
+		return
+	}
+	cmGrace, ok := connmgr["GracePeriod"].(string)
+	if !ok {
+		log.Log("No Swarm.ConnMgr.GracePeriod field in config, skipping")
+		return
+	}
+
+	if cmType == "basic" && int(cmLowWater) == 600 && int(cmHighWater) == 900 && cmGrace == "20s" {
+		delete(connmgr, "Type")
+		delete(connmgr, "GracePeriod")
+		delete(connmgr, "LowWater")
+		delete(connmgr, "HighWater")
+	} else {
+		log.Log("Swarm.ConnMgr settings are different than the old defaults, skipping")
+	}
+}