diff --git a/cmd/status.go b/cmd/status.go index b12c9e7..6d45cde 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -12,7 +12,6 @@ import ( "github.com/nlewo/comin/internal/deployment" "github.com/nlewo/comin/internal/generation" "github.com/nlewo/comin/internal/manager" - "github.com/nlewo/comin/internal/repository" "github.com/nlewo/comin/internal/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -25,14 +24,14 @@ func generationStatus(g generation.Generation) { fmt.Printf(" Status: initializated\n") case generation.Evaluating: fmt.Printf(" Status: evaluating (since %s)\n", humanize.Time(g.EvalStartedAt)) - case generation.Evaluated: + case generation.EvaluationSucceeded: fmt.Printf(" Status: evaluated (%s)\n", humanize.Time(g.EvalEndedAt)) case generation.Building: fmt.Printf(" Status: building (since %s)\n", humanize.Time(g.BuildStartedAt)) - case generation.Built: + case generation.BuildSucceeded: fmt.Printf(" Status: built (%s)\n", humanize.Time(g.BuildEndedAt)) } - printCommit(g.RepositoryStatus) + printCommit(g.SelectedRemoteName, g.SelectedBranchName, g.SelectedCommitId, g.SelectedCommitMsg) } func deploymentStatus(d deployment.Deployment) { @@ -48,17 +47,17 @@ func deploymentStatus(d deployment.Deployment) { case deployment.Failed: fmt.Printf(" Status: failed (%s)\n", humanize.Time(d.EndAt)) } - printCommit(d.Generation.RepositoryStatus) + printCommit(d.Generation.SelectedRemoteName, d.Generation.SelectedBranchName, d.Generation.SelectedCommitId, d.Generation.SelectedCommitMsg) } -func printCommit(rs repository.RepositoryStatus) { +func printCommit(selectedRemoteName, selectedBranchName, selectedCommitId, selectedCommitMsg string) { fmt.Printf(" Commit %s from '%s/%s'\n", - rs.SelectedCommitId, - rs.SelectedRemoteName, - rs.SelectedBranchName, + selectedCommitId, + selectedRemoteName, + selectedBranchName, ) fmt.Printf(" %s\n", - utils.FormatCommitMsg(rs.SelectedCommitMsg), + utils.FormatCommitMsg(selectedCommitMsg), ) } diff --git a/flake.nix b/flake.nix index 8c9d001..d46d589 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,7 @@ overlay = final: prev: { comin = final.buildGoModule rec { pname = "comin"; - version = "0.0.1"; + version = "0.1.1"; nativeCheckInputs = [ final.git ]; src = final.lib.cleanSourceWith { src = ./.; @@ -26,7 +26,7 @@ p == "README.md" ); }; - vendorHash = "sha256-kyj0CbB3IfRvrNXsO9JEVYJ8Hr5e747i+ZKcbR6WfKM="; + vendorHash = "sha256-7rh1t3DkKfJvUOkPjdi2vqS8JTZpWtI61mTBKDHcPVk="; buildInputs = [ final.makeWrapper ]; postInstall = '' # This is because Nix needs Git at runtime by the go-git library @@ -45,11 +45,7 @@ hostname = cfg.services.comin.hostname; state_dir = "/var/lib/comin"; remotes = cfg.services.comin.remotes; - } // ( - if cfg.services.comin.inotifyRepositoryPath != null - then { inotify.repository_path = cfg.services.comin.inotifyRepositoryPath; } - else { } - ); + }; cominConfigYaml = yaml.generate "comin.yaml" cominConfig; in { options = with lib; with types; { @@ -65,7 +61,10 @@ type = str; default = config.networking.hostName; description = '' - The hostname of the machine. + The name of the NixOS configuration to evaluate and + deploy. This value is used by comin to evaluate the + flake output + nixosConfigurations."".config.system.build.toplevel ''; }; remotes = mkOption { @@ -170,15 +169,6 @@ Note it is only used by comin at evaluation. ''; }; - inotifyRepositoryPath = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - The path of a local repository to watch. On each commit, - the worker is triggered to fetch new commits. This - allows to have fast switch when the repository is local. - ''; - }; }; }; config = lib.mkIf cfg.services.comin.enable { diff --git a/go.mod b/go.mod index e83e41d..d842f7f 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect diff --git a/go.sum b/go.sum index f08e487..6064291 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/internal/deployment/deployment.go b/internal/deployment/deployment.go index 9e0698f..13ac7d2 100644 --- a/internal/deployment/deployment.go +++ b/internal/deployment/deployment.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/google/uuid" "github.com/nlewo/comin/internal/generation" "github.com/sirupsen/logrus" ) @@ -17,9 +18,38 @@ const ( Failed ) +func StatusToString(status Status) string { + switch status { + case Init: + return "init" + case Running: + return "running" + case Done: + return "done" + case Failed: + return "failed" + } + return "" +} + +func StatusFromString(status string) Status { + switch status { + case "init": + return Init + case "running": + return Running + case "done": + return Done + case "failed": + return Failed + } + return Init +} + type DeployFunc func(context.Context, string, string, string) (bool, error) type Deployment struct { + UUID string `json:"uuid"` Generation generation.Generation `json:"generation"` StartAt time.Time `json:"start_at"` EndAt time.Time `json:"end_at"` @@ -42,11 +72,12 @@ type DeploymentResult struct { func New(g generation.Generation, deployerFunc DeployFunc, deploymentCh chan DeploymentResult) Deployment { operation := "switch" - if g.RepositoryStatus.IsTesting() { + if g.SelectedBranchIsTesting { operation = "test" } return Deployment{ + UUID: uuid.NewString(), Generation: g, deployerFunc: deployerFunc, deploymentCh: deploymentCh, diff --git a/internal/generation/generation.go b/internal/generation/generation.go index caa0b7f..cd6f425 100644 --- a/internal/generation/generation.go +++ b/internal/generation/generation.go @@ -5,7 +5,9 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/nlewo/comin/internal/repository" + "github.com/sirupsen/logrus" ) type Status int64 @@ -13,35 +15,85 @@ type Status int64 const ( Init Status = iota Evaluating - Evaluated + EvaluationSucceeded + EvaluationFailed Building - Built + BuildSucceeded + BuildFailed ) +func StatusToString(status Status) string { + switch status { + case Init: + return "init" + case Evaluating: + return "evaluating" + case EvaluationSucceeded: + return "evaluation-succeeded" + case EvaluationFailed: + return "evaluation-failed" + case Building: + return "building" + case BuildSucceeded: + return "build-succeeded" + case BuildFailed: + return "build-failed" + } + return "" +} + +func StatusFromString(status string) Status { + switch status { + case "init": + return Init + case "evaluating": + return Evaluating + case "evaluation-succeeded": + return EvaluationSucceeded + case "evaluation-failed": + return EvaluationFailed + case "building": + return Building + case "build-succeeded": + return BuildSucceeded + case "build-failed": + return BuildFailed + } + return Init +} + // We consider each created genration is legit to be deployed: hard // reset is ensured at RepositoryStatus creation. type Generation struct { - flakeUrl string - hostname string - machineId string + UUID string + FlakeUrl string + Hostname string + MachineId string Status Status - RepositoryStatus repository.RepositoryStatus + SelectedRemoteName string + SelectedBranchName string + SelectedCommitId string + SelectedCommitMsg string + SelectedBranchIsTesting bool EvalStartedAt time.Time evalTimeout time.Duration evalFunc EvalFunc + evalCh chan EvalResult + EvalEndedAt time.Time - EvalErr error + EvalErr error `json:"-"` OutPath string DrvPath string EvalMachineId string BuildStartedAt time.Time BuildEndedAt time.Time - BuildErr error + buildErr error `json:"-"` buildFunc BuildFunc + buildCh chan BuildResult } type EvalFunc func(ctx context.Context, flakeUrl string, hostname string) (drvPath string, outPath string, machineId string, err error) @@ -62,61 +114,90 @@ type EvalResult struct { func New(repositoryStatus repository.RepositoryStatus, flakeUrl, hostname, machineId string, evalFunc EvalFunc, buildFunc BuildFunc) Generation { return Generation{ - RepositoryStatus: repositoryStatus, - evalTimeout: 6 * time.Second, - evalFunc: evalFunc, - buildFunc: buildFunc, - flakeUrl: flakeUrl, - hostname: hostname, - machineId: machineId, - Status: Init, + UUID: uuid.NewString(), + SelectedRemoteName: repositoryStatus.SelectedRemoteName, + SelectedBranchName: repositoryStatus.SelectedBranchName, + SelectedCommitId: repositoryStatus.SelectedCommitId, + SelectedCommitMsg: repositoryStatus.SelectedCommitMsg, + SelectedBranchIsTesting: repositoryStatus.SelectedBranchIsTesting, + evalTimeout: 6 * time.Second, + evalFunc: evalFunc, + buildFunc: buildFunc, + FlakeUrl: flakeUrl, + Hostname: hostname, + MachineId: machineId, + Status: Init, } } +func (g Generation) EvalCh() chan EvalResult { + return g.evalCh +} + +func (g Generation) BuildCh() chan BuildResult { + return g.buildCh +} + func (g Generation) UpdateEval(r EvalResult) Generation { + logrus.Debugf("Eval done with %#v", r) g.EvalEndedAt = r.EndAt g.DrvPath = r.DrvPath g.OutPath = r.OutPath - g.EvalErr = r.Err g.EvalMachineId = r.MachineId - g.Status = Evaluated + g.EvalErr = r.Err + if g.EvalErr == nil { + g.Status = EvaluationSucceeded + } else { + g.Status = EvaluationFailed + } return g } func (g Generation) UpdateBuild(r BuildResult) Generation { + logrus.Debugf("Build done with %#v", r) g.BuildEndedAt = r.EndAt - g.BuildErr = r.Err - g.Status = Built + g.buildErr = r.Err + if g.buildErr == nil { + g.Status = BuildSucceeded + } else { + g.Status = BuildFailed + } return g } -func (g Generation) Eval(ctx context.Context) (result chan EvalResult) { - result = make(chan EvalResult) +func (g Generation) Eval(ctx context.Context) Generation { + g.evalCh = make(chan EvalResult) + g.EvalStartedAt = time.Now() + g.Status = Evaluating + fn := func() { ctx, cancel := context.WithTimeout(ctx, g.evalTimeout) defer cancel() - drvPath, outPath, machineId, err := g.evalFunc(ctx, g.flakeUrl, g.hostname) + drvPath, outPath, machineId, err := g.evalFunc(ctx, g.FlakeUrl, g.Hostname) evaluationResult := EvalResult{ EndAt: time.Now(), } if err == nil { evaluationResult.DrvPath = drvPath evaluationResult.OutPath = outPath - if machineId != "" && g.machineId != machineId { + evaluationResult.MachineId = machineId + if machineId != "" && g.MachineId != machineId { evaluationResult.Err = fmt.Errorf("The evaluated comin.machineId '%s' is different from the /etc/machine-id '%s' of this machine", - g.machineId, machineId) + machineId, g.MachineId) } } else { evaluationResult.Err = err } - result <- evaluationResult + g.evalCh <- evaluationResult } go fn() - return result + return g } -func (g Generation) Build(ctx context.Context) (result chan BuildResult) { - result = make(chan BuildResult) +func (g Generation) Build(ctx context.Context) Generation { + g.buildCh = make(chan BuildResult) + g.BuildStartedAt = time.Now() + g.Status = Building fn := func() { ctx, cancel := context.WithTimeout(ctx, g.evalTimeout) defer cancel() @@ -125,8 +206,8 @@ func (g Generation) Build(ctx context.Context) (result chan BuildResult) { EndAt: time.Now(), } buildResult.Err = err - result <- buildResult + g.buildCh <- buildResult } go fn() - return result + return g } diff --git a/internal/generation/generation_test.go b/internal/generation/generation_test.go index c64d4f5..40b73de 100644 --- a/internal/generation/generation_test.go +++ b/internal/generation/generation_test.go @@ -13,6 +13,7 @@ import ( func TestEval(t *testing.T) { var evalResult EvalResult var ctx context.Context + machineId := "machine-id" evalDone := make(chan struct{}) nixEvalMock := func(ctx context.Context, repositoryPath string, hostname string) (string, string, string, error) { @@ -20,7 +21,7 @@ func TestEval(t *testing.T) { case <-ctx.Done(): return "", "", "", fmt.Errorf("timeout exceeded") case <-evalDone: - return "", "", "", nil + return "", "", machineId, nil } } nixBuildMock := func(ctx context.Context, drv string) error { @@ -29,20 +30,21 @@ func TestEval(t *testing.T) { repositoryPath := "repository/path/" hostname := "machine" - g := New(repository.RepositoryStatus{}, repositoryPath, hostname, "", nixEvalMock, nixBuildMock) + g := New(repository.RepositoryStatus{}, repositoryPath, hostname, machineId, nixEvalMock, nixBuildMock) g.evalTimeout = 1 * time.Second // The eval job never terminates so it should timeout ctx = context.Background() - evalResultCh := g.Eval(ctx) - evalResult = <-evalResultCh + g = g.Eval(ctx) + evalResult = <-g.EvalCh() assert.NotNil(t, evalResult.Err) assert.EqualError(t, evalResult.Err, "timeout exceeded") ctx = context.Background() - evalResultCh = g.Eval(ctx) + g = g.Eval(ctx) // This is to simulate the eval completion close(evalDone) - evalResult = <-evalResultCh + evalResult = <-g.EvalCh() assert.Nil(t, evalResult.Err) + assert.Equal(t, machineId, evalResult.MachineId) } diff --git a/internal/generation/generations.go b/internal/generation/generations.go deleted file mode 100644 index dcc1275..0000000 --- a/internal/generation/generations.go +++ /dev/null @@ -1,26 +0,0 @@ -package generation - -// type Generations struct { -// onBuiltGeneration func() -// } - -// func New() Generations { -// } - -// func (gs Generations) FromRepositoryStatus(rs RepositoryStatus) Generation { -// } - -// func (gs Generations) MostRecentGeneration() { -// } - -// func (gs Generations) IsMostRecentGeneration() { -// } - -// func (gs Generations) BuildGeneration(g Generation) { -// fn := func() { -// g.Eval() -// g.Build() -// } -// go fn() -// gs.onBuiltGeneration() -// } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 7a4c06b..bf1bdc4 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -2,7 +2,7 @@ package manager import ( "context" - "time" + "fmt" "github.com/nlewo/comin/internal/deployment" "github.com/nlewo/comin/internal/generation" @@ -49,6 +49,9 @@ type Manager struct { // The deployment currenly managed deployment deployment.Deployment deployerFunc deployment.DeployFunc + + repositoryStatusCh chan repository.RepositoryStatus + triggerDeploymentCh chan generation.Generation } func New(r repository.Repository, path, hostname, machineId string) Manager { @@ -65,6 +68,8 @@ func New(r repository.Repository, path, hostname, machineId string) Manager { stateResultCh: make(chan State), cominServiceRestartFunc: utils.CominServiceRestart, deploymentResultCh: make(chan deployment.DeploymentResult), + repositoryStatusCh: make(chan repository.RepositoryStatus), + triggerDeploymentCh: make(chan generation.Generation, 1), } } @@ -88,10 +93,81 @@ func (m Manager) toState() State { } } +func (m Manager) onEvaluated(ctx context.Context, evalResult generation.EvalResult) Manager { + m.generation = m.generation.UpdateEval(evalResult) + if evalResult.Err == nil { + m.generation = m.generation.Build(ctx) + } else { + m.isRunning = false + } + return m +} + +func (m Manager) onBuilt(ctx context.Context, buildResult generation.BuildResult) Manager { + m.generation = m.generation.UpdateBuild(buildResult) + if buildResult.Err == nil { + m.triggerDeployment(ctx, m.generation) + } else { + m.isRunning = false + } + return m +} + +func (m Manager) triggerDeployment(ctx context.Context, g generation.Generation) { + m.triggerDeploymentCh <- g +} + +func (m Manager) onTriggerDeployment(ctx context.Context, g generation.Generation) Manager { + m.deployment = deployment.New(g, m.deployerFunc, m.deploymentResultCh) + m.deployment = m.deployment.Deploy(ctx) + return m +} + +func (m Manager) onDeployment(ctx context.Context, deploymentResult deployment.DeploymentResult) Manager { + logrus.Debugf("Deploy done with %#v", deploymentResult) + m.deployment = m.deployment.Update(deploymentResult) + // The comin service is not restart by the switch-to-configuration script in order to let comin terminating properly. Instead, comin restarts itself. + if m.deployment.RestartComin { + m.needToBeRestarted = true + } + m.isRunning = false + return m +} + +func (m Manager) onRepositoryStatus(ctx context.Context, rs repository.RepositoryStatus) Manager { + logrus.Debugf("Fetch done with %#v", rs) + m.isFetching = false + m.repositoryStatus = rs + if rs.SelectedCommitId == m.generation.SelectedCommitId && rs.SelectedBranchIsTesting == m.generation.SelectedBranchIsTesting { + logrus.Debugf("The repository status is the same than the previous one") + m.isRunning = false + } else { + // g.Stop(): this is required once we remove m.IsRunning + flakeUrl := fmt.Sprintf("git+file://%s?rev=%s", m.repositoryPath, m.repositoryStatus.SelectedCommitId) + m.generation = generation.New(rs, flakeUrl, m.hostname, m.machineId, m.evalFunc, m.buildFunc) + m.generation = m.generation.Eval(ctx) + } + return m +} + +func (m Manager) onTriggerRepository(ctx context.Context, remoteName string) Manager { + if m.isFetching { + logrus.Debugf("The manager is already fetching the repository") + return m + } + // FIXME: we will remove this in future versions + if m.isRunning { + logrus.Debugf("The manager is already running: it is currently not able to run tasks in parallel") + return m + } + logrus.Debugf("Trigger fetch and update remote %s", remoteName) + m.isRunning = true + m.isFetching = true + m.repositoryStatusCh = m.repository.FetchAndUpdate(ctx, remoteName) + return m +} + func (m Manager) Run() { - var repositoryStatusCh chan repository.RepositoryStatus - var evalResultCh chan generation.EvalResult - var buildResultCh chan generation.BuildResult ctx := context.TODO() logrus.Info("The manager is started") @@ -103,67 +179,18 @@ func (m Manager) Run() { case <-m.stateRequestCh: m.stateResultCh <- m.toState() case remoteName := <-m.triggerRepository: - if m.isFetching { - logrus.Debugf("The manager is already fetching the repository") - continue - } - // FIXME: we will remove this in future versions - if m.isRunning { - logrus.Debugf("The manager is already running: it is currently not able to run tasks in parallel") - continue - } - logrus.Debugf("Trigger fetch and update remote %s", remoteName) - m.isRunning = true - m.isFetching = true - repositoryStatusCh = m.repository.FetchAndUpdate(ctx, remoteName) - case rs := <-repositoryStatusCh: - logrus.Debugf("Fetch done with %#v", rs) - m.isFetching = false - m.repositoryStatus = rs - if rs.Equal(m.generation.RepositoryStatus) { - logrus.Debugf("The repository status is the same than the previous one") - m.isRunning = false - } else { - // g.Stop(): this is required once we remove m.IsRunning - m.generation = generation.New(rs, m.repositoryPath, m.hostname, m.machineId, m.evalFunc, m.buildFunc) - m.generation.EvalStartedAt = time.Now() - m.generation.Status = generation.Evaluating - - // FIXME: we need to let nix fetching a git commit from the repository instead of using the repository - // directory which an be updated in parallel - evalResultCh = m.generation.Eval(ctx) - } - case evalResult := <-evalResultCh: - logrus.Debugf("Eval done with %#v", evalResult) - m.generation = m.generation.UpdateEval(evalResult) - if evalResult.Err == nil { - m.generation.BuildStartedAt = time.Now() - m.generation.Status = generation.Building - buildResultCh = m.generation.Build(ctx) - } else { - logrus.Infof("Evaluation error: %s", evalResult.Err) - m.isRunning = false - } - case buildResult := <-buildResultCh: - logrus.Debugf("Build done with %#v", buildResult) - m.generation = m.generation.UpdateBuild(buildResult) - if buildResult.Err == nil { - m.deployment = deployment.New(m.generation, m.deployerFunc, m.deploymentResultCh) - m.deployment = m.deployment.Deploy(ctx) - } else { - m.isRunning = false - } + m = m.onTriggerRepository(ctx, remoteName) + case rs := <-m.repositoryStatusCh: + m = m.onRepositoryStatus(ctx, rs) + case evalResult := <-m.generation.EvalCh(): + m = m.onEvaluated(ctx, evalResult) + case buildResult := <-m.generation.BuildCh(): + m = m.onBuilt(ctx, buildResult) + case generation := <-m.triggerDeploymentCh: + m = m.onTriggerDeployment(ctx, generation) case deploymentResult := <-m.deploymentResultCh: - logrus.Debugf("Deploy done with %#v", deploymentResult) - m.deployment = m.deployment.Update(deploymentResult) - // The comin service is not restart by the switch-to-configuration script in order to let comin terminating properly. Instead, comin restarts itself. - if m.deployment.RestartComin { - m.needToBeRestarted = true - break - } - m.isRunning = false + m = m.onDeployment(ctx, deploymentResult) } - if m.needToBeRestarted { // TODO: stop contexts if err := m.cominServiceRestartFunc(); err != nil { diff --git a/internal/repository/repository_status.go b/internal/repository/repository_status.go index 942433d..21ab5dd 100644 --- a/internal/repository/repository_status.go +++ b/internal/repository/repository_status.go @@ -105,7 +105,3 @@ func (r RepositoryStatus) Copy() RepositoryStatus { } return rs.(RepositoryStatus) } - -func (r1 RepositoryStatus) Equal(r2 RepositoryStatus) bool { - return r1.SelectedCommitId == r2.SelectedCommitId && r1.IsTesting() == r2.IsTesting() -}