Merge pull request '[dbt] Allow enforcing dbt flags persistency at th…

…e top level MODULE file' (#20) from ie/persist-flags-per-project into master

Reviewed-on: https://gitea.daedalean.ai/daedalean-github/dbt/pulls/20

CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+### v1.4.0
+
+- Switched to semantic versioning of the binaries.
+- The MODULE file format is no longer directly linked to the minor version of DBT,
+  versions 1 through 3 are supported.
+- The "persist-flags" configuration can be enforced for a project in top level MODULE file.
+  The field is retrofitted into all known MODULE file formats.
+  The value specified in the field is applied to all builds of the project,
+  overriding the user configuration.

README.md

@@ -25,6 +25,7 @@ sudo update-alternatives --config gofmt
 ## Installation
 
 You can install the latest version of DBT by running:
+
 ```
 # for go < 1.18
 go get github.com/daedaleanai/dbt
@@ -33,6 +34,11 @@ go get github.com/daedaleanai/dbt
 go install github.com/daedaleanai/dbt@latest
 ```
 
+Notice that DBT follows sematic versioning, and major version change introduce breaking changes.
+You might in practice prefer to pin to a specific major version.
+
+CHANGELOG.md lists the changes between versions.
+
 ### Setting up a local mirror
 
 DBT can read a configuration file located at:

cmd/build.go

@@ -19,7 +19,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/daedaleanai/dbt/config"
 	"github.com/daedaleanai/dbt/log"
 	"github.com/daedaleanai/dbt/module"
 	"github.com/daedaleanai/dbt/util"
@@ -217,7 +216,8 @@ func runBuild(args []string, mode mode, modeArgs []string) {
 
 	util.EnsureManagedDir(util.BuildDirName)
 
-	workspaceFlags := module.ReadModuleFile(workspaceRoot).Flags
+	moduleFile := module.ReadModuleFile(workspaceRoot)
+	workspaceFlags := moduleFile.Flags
 	positivePatterns, negativePatterns, cmdlineFlags := parseArgs(args)
 	_, _, legacyFlags := parseArgs(args)
 
@@ -236,14 +236,14 @@ func runBuild(args []string, mode mode, modeArgs []string) {
 	}
 	log.Debug("Output directory: %s.\n", outputDir)
 	genInput := generatorInput{
-		DbtVersion:           util.DbtVersion,
+		DbtVersion:           util.VersionTriplet(),
 		OutputDir:            outputDir,
 		CmdlineFlags:         cmdlineFlags,
 		WorkspaceFlags:       workspaceFlags,
 		TestArgs:             []string{},
 		RunArgs:              []string{},
 		BuildAnalyzerTargets: false,
-		PersistFlags:         config.GetConfig().PersistFlags,
+		PersistFlags:         moduleFile.PersistFlags,
 
 		// Legacy fields
 		Version:        2,
@@ -461,7 +461,7 @@ func printNinjaOutput(dir, fileName, label string, args []string) {
 
 func completeBuildArgs(toComplete string, mode mode) []string {
 	genOutput := runGenerator(generatorInput{
-		DbtVersion:      util.DbtVersion,
+		DbtVersion:      util.VersionTriplet(),
 		CompletionsOnly: true,
 
 		// Legacy field expected by dbt-rules < v1.10.0.

cmd/root.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/daedaleanai/dbt/log"
@@ -17,7 +16,7 @@ var (
 		Long: `The Daedalean Build Tool (dbt) helps setting up workspaces consisting
 of multiple modules (git repositories), managing dependencies between modules, and
 building build targets defined in those modules.`,
-		Version: fmt.Sprintf("v%d.%d.%d", util.DbtVersion[0], util.DbtVersion[1], util.DbtVersion[2]),
+		Version: util.Version(),
 	}
 )
 

manifest/manifest.go

@@ -52,11 +52,12 @@ func (c Commit) String() string {
 }
 
 func Generate(modules map[string]module.Module, allowUncommittedChanges bool) (Manifest, error) {
+	dbtVersion := util.VersionTriplet()
 	manifest := Manifest{
 		DbtVersion: DbtVersion{
-			Major:    util.DbtVersion[0],
-			Minor:    util.DbtVersion[1],
-			Revision: util.DbtVersion[2],
+			Major:    dbtVersion[0],
+			Minor:    dbtVersion[1],
+			Revision: dbtVersion[2],
 		},
 	}
 

module/file.go

@@ -3,6 +3,7 @@ package module
 import (
 	"path"
 
+	"github.com/daedaleanai/dbt/config"
 	"github.com/daedaleanai/dbt/log"
 	"github.com/daedaleanai/dbt/util"
 )
@@ -25,6 +26,7 @@ type ModuleFile struct {
 	Layout       string
 	Dependencies map[string]Dependency
 	Flags        map[string]string
+	PersistFlags bool `yaml:"persist-flags"`
 }
 
 // MODULE file version 2
@@ -46,6 +48,7 @@ type v2ModuleFile struct {
 	Dependencies       map[string]v2Dependency
 	PinnedDependencies map[string]v2PinnedDependency
 	Flags              map[string]string
+	PersistFlags       bool `yaml:"persist-flags"`
 }
 
 // MODULE file version 1
@@ -63,6 +66,7 @@ type v1Dependency struct {
 
 type v1ModuleFile struct {
 	Dependencies []v1Dependency
+	PersistFlags bool `yaml:"persist-flags"`
 }
 
 // ReadModuleFile reads and parses module Dependencies from a MODULE file.
@@ -71,7 +75,7 @@ func ReadModuleFile(modulePath string) ModuleFile {
 	if !util.FileExists(moduleFilePath) {
 		log.Debug("Module has no %s file.\n", util.ModuleFileName)
 		return ModuleFile{
-			Version:      util.DbtVersion[1],
+			Version:      util.ModuleSyntaxVersion,
 			Dependencies: map[string]Dependency{},
 		}
 	}
@@ -85,28 +89,31 @@ func ReadModuleFile(modulePath string) ModuleFile {
 		return readV1ModuleFile(moduleFilePath)
 	case 2:
 		return readV2ModuleFile(moduleFilePath)
-	case util.DbtVersion[1]:
+	case 3:
 		return readV3ModuleFile(moduleFilePath)
 	default:
-		log.Fatal("MODULE file has version %d that requires a newer version of dbt\n", moduleFileVersion.Version)
+		log.Fatal("MODULE file has unknown syntax version %d. It is either a mistake in the file or a newer version of dbt is required.\n", moduleFileVersion.Version)
 		return ModuleFile{}
 	}
 }
 
 // WriteModuleFile serializes and writes a Module's Dependencies to a MODULE file.
 func WriteModuleFile(modulePath string, moduleFile ModuleFile) {
-	moduleFile.Version = util.DbtVersion[1]
+	moduleFile.Version = util.ModuleSyntaxVersion
 	moduleFilePath := path.Join(modulePath, util.ModuleFileName)
 	util.WriteYaml(moduleFilePath, moduleFile)
 }
 
 func readV1ModuleFile(path string) ModuleFile {
-	var v1ModuleFile v1ModuleFile
+	v1ModuleFile := v1ModuleFile{
+		PersistFlags: config.GetConfig().PersistFlags,
+	}
 	util.ReadYaml(path, &v1ModuleFile)
 
 	moduleFile := ModuleFile{
-		Version:      util.DbtVersion[1],
+		Version:      util.ModuleSyntaxVersion,
 		Dependencies: map[string]Dependency{},
+		PersistFlags: v1ModuleFile.PersistFlags,
 	}
 	for _, dep := range v1ModuleFile.Dependencies {
 		moduleFile.Dependencies[dep.Name] = Dependency{
@@ -119,13 +126,16 @@ func readV1ModuleFile(path string) ModuleFile {
 }
 
 func readV2ModuleFile(path string) ModuleFile {
-	var v2ModuleFile v2ModuleFile
+	v2ModuleFile := v2ModuleFile{
+		PersistFlags: config.GetConfig().PersistFlags,
+	}
 	util.ReadYaml(path, &v2ModuleFile)
 
 	moduleFile := ModuleFile{
-		Version:      util.DbtVersion[1],
+		Version:      util.ModuleSyntaxVersion,
 		Layout:       v2ModuleFile.Layout,
 		Dependencies: map[string]Dependency{},
+		PersistFlags: v2ModuleFile.PersistFlags,
 	}
 	for name, dep := range v2ModuleFile.Dependencies {
 		moduleFile.Dependencies[name] = Dependency{
@@ -138,7 +148,9 @@ func readV2ModuleFile(path string) ModuleFile {
 }
 
 func readV3ModuleFile(path string) ModuleFile {
-	var moduleFile ModuleFile
+	moduleFile := ModuleFile{
+		PersistFlags: config.GetConfig().PersistFlags,
+	}
 	util.ReadYaml(path, &moduleFile)
 
 	// YAML decoding can produce `nil`` maps if the key is present in the YAML file

util/util.go

@@ -16,21 +16,21 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-// DbtVersion is the current version of DBT. The minor version
-// is also used as the MODULE file version.
-var DbtVersion = [3]uint{1, 3, 18}
-
 // ModuleFileName is the name of the file describing each module.
 const (
-	ModuleFileName = "MODULE"
-	BuildDirName   = "BUILD"
+	ModuleFileName      = "MODULE"
+	ModuleSyntaxVersion = 3
+
+	BuildDirName = "BUILD"
 	// DepsDirName is directory that dependencies are stored in.
 	DepsDirName     = "DEPS"
 	WarningFileName = "WARNING.readme.txt"
 )
 
-const fileMode = 0664
-const dirMode = 0775
+const (
+	fileMode = 0664
+	dirMode  = 0775
+)
 
 // Reimplementation of CutPrefix for backwards compatibility with versions < 1.20
 func CutPrefix(str string, prefix string) (string, bool) {
@@ -40,6 +40,20 @@ func CutPrefix(str string, prefix string) (string, bool) {
 	return str, false
 }
 
+func obtainVersion() (string, uint, uint, uint) {
+	return "1.4.0", 1, 4, 0
+}
+
+func VersionTriplet() [3]uint {
+	_, major, minor, patch := obtainVersion()
+	return [3]uint{major, minor, patch}
+}
+
+func Version() string {
+	s, _, _, _ := obtainVersion()
+	return s
+}
+
 // FileExists checks whether some file exists.
 func FileExists(file string) bool {
 	stat, err := os.Stat(file)