diff --git a/pkg/cli/admin/nodeimage/basenodeimagecommand.go b/pkg/cli/admin/nodeimage/basenodeimagecommand.go new file mode 100644 index 0000000000..fffa55672c --- /dev/null +++ b/pkg/cli/admin/nodeimage/basenodeimagecommand.go @@ -0,0 +1,405 @@ +package nodeimage + +import ( + "bufio" + "context" + "fmt" + "io" + "regexp" + "time" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "k8s.io/klog/v2" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + kapierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/kubectl/pkg/cmd/exec" + + ocpv1 "github.com/openshift/api/config/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned" + "github.com/openshift/library-go/pkg/operator/resource/retry" + ocrelease "github.com/openshift/oc/pkg/cli/admin/release" + imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" +) + +type BaseNodeImageCommand struct { + genericiooptions.IOStreams + SecurityOptions imagemanifest.SecurityOptions + LogOut io.Writer + + Config *rest.Config + remoteExecutor exec.RemoteExecutor + ConfigClient configclient.Interface + Client kubernetes.Interface + nodeJoinerImage string + nodeJoinerNamespace *corev1.Namespace + nodeJoinerServiceAccount *corev1.ServiceAccount + nodeJoinerRole *rbacv1.ClusterRole + RESTClientGetter genericclioptions.RESTClientGetter + nodeJoinerPod *corev1.Pod + command string +} + +func newBaseNodeImageCommand(streams genericiooptions.IOStreams, command, prefix string) *BaseNodeImageCommand { + cmd := &BaseNodeImageCommand{ + IOStreams: streams, + command: command, + } + cmd.LogOut = cmd.newPrefixWriter(streams.Out, prefix) + return cmd +} + +func (c *BaseNodeImageCommand) newPrefixWriter(out io.Writer, prefix string) io.Writer { + reader, writer := io.Pipe() + scanner := bufio.NewScanner(reader) + go func() { + for scanner.Scan() { + text := scanner.Text() + ts := time.Now().UTC().Format(time.RFC3339) + fmt.Fprintf(out, "%s [node-image %s] %s\n", ts, prefix, text) + } + }() + return writer +} + +func (c *BaseNodeImageCommand) log(format string, a ...interface{}) { + fmt.Fprintf(c.LogOut, format+"\n", a...) +} + +func (c *BaseNodeImageCommand) getNodeJoinerPullSpec(ctx context.Context) error { + // Get the current cluster release version. + releaseImage, err := c.fetchClusterReleaseImage(ctx) + if err != nil { + return err + } + + // Extract the baremetal-installer image pullspec, since it + // provide the node-joiner tool. + opts := ocrelease.NewInfoOptions(c.IOStreams) + opts.SecurityOptions = c.SecurityOptions + release, err := opts.LoadReleaseInfo(releaseImage, false) + if err != nil { + return err + } + + tagName := "baremetal-installer" + for _, tag := range release.References.Spec.Tags { + if tag.Name == tagName { + c.nodeJoinerImage = tag.From.Name + return nil + } + } + + return fmt.Errorf("no image tag %q exists in the release image %s", tagName, releaseImage) +} + +func (c *BaseNodeImageCommand) fetchClusterReleaseImage(ctx context.Context) (string, error) { + cv, err := c.getCurrentClusterVersion(ctx) + if err != nil { + return "", err + } + + image := cv.Status.Desired.Image + if len(image) == 0 && cv.Spec.DesiredUpdate != nil { + image = cv.Spec.DesiredUpdate.Image + } + if len(image) == 0 { + return "", fmt.Errorf("the server is not reporting a release image at this time") + } + + return image, nil +} + +func (c *BaseNodeImageCommand) getCurrentClusterVersion(ctx context.Context) (*ocpv1.ClusterVersion, error) { + cv, err := c.ConfigClient.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{}) + if err != nil { + if kapierrors.IsNotFound(err) || kapierrors.ReasonForError(err) == metav1.StatusReasonUnknown { + klog.V(2).Infof("Unable to find cluster version object from cluster: %v", err) + return nil, fmt.Errorf("command expects a connection to an OpenShift 4.x server") + } + } + return cv, nil +} + +func (c *BaseNodeImageCommand) isClusterVersionLessThan(ctx context.Context, version string) (bool, error) { + cv, err := c.getCurrentClusterVersion(ctx) + if err != nil { + return false, err + } + + currentVersion := cv.Status.Desired.Version + matches := regexp.MustCompile(`^(\d+[.]\d+)[.].*`).FindStringSubmatch(currentVersion) + if len(matches) < 2 { + return false, fmt.Errorf("failed to parse major.minor version from ClusterVersion status.desired.version %q", currentVersion) + } + return matches[1] < version, nil +} + +// Adds a guardrail for node-image commands which is supported only for Openshift version 4.17 and later +func (c *BaseNodeImageCommand) checkMinSupportedVersion(ctx context.Context) error { + notSupported, err := c.isClusterVersionLessThan(ctx, nodeJoinerMinimumSupportedVersion) + if err != nil { + return err + } + if notSupported { + return fmt.Errorf("the 'oc adm node-image' command is only available for OpenShift versions %s and later", nodeJoinerMinimumSupportedVersion) + } + return nil +} + +func (c *BaseNodeImageCommand) createNamespace(ctx context.Context) error { + nsNodeJoiner := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "openshift-node-joiner-", + Annotations: map[string]string{ + "oc.openshift.io/command": c.command, + "openshift.io/node-selector": "", + }, + }, + } + + ns, err := c.Client.CoreV1().Namespaces().Create(ctx, nsNodeJoiner, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("cannot create namespace: %w", err) + } + + c.nodeJoinerNamespace = ns + return nil +} + +func (c *BaseNodeImageCommand) cleanup(ctx context.Context) { + if c.nodeJoinerNamespace == nil { + return + } + + err := c.Client.CoreV1().Namespaces().Delete(ctx, c.nodeJoinerNamespace.GetName(), metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("cannot delete namespace %s: %v\n", c.nodeJoinerNamespace.GetName(), err) + } +} + +func (c *BaseNodeImageCommand) createServiceAccount(ctx context.Context) error { + nodeJoinerServiceAccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "node-joiner-", + Annotations: map[string]string{ + "oc.openshift.io/command": c.command, + }, + Namespace: c.nodeJoinerNamespace.GetName(), + }, + } + + sa, err := c.Client.CoreV1().ServiceAccounts(c.nodeJoinerNamespace.GetName()).Create(ctx, nodeJoinerServiceAccount, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("cannot create service account: %w", err) + } + + c.nodeJoinerServiceAccount = sa + return nil +} + +func (c *BaseNodeImageCommand) clusterRoleBindings() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "node-joiner-monitor-", + Annotations: map[string]string{ + "oc.openshift.io/command": c.command, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Namespace", + Name: c.nodeJoinerNamespace.GetName(), + UID: c.nodeJoinerNamespace.GetUID(), + }, + }, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: c.nodeJoinerServiceAccount.GetName(), + Namespace: c.nodeJoinerNamespace.GetName(), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: c.nodeJoinerRole.GetName(), + }, + } +} + +func (c *BaseNodeImageCommand) waitForRunningPod(ctx context.Context) error { + // Wait for the node-joiner pod to come up + return wait.PollUntilContextTimeout( + ctx, + time.Second*5, + time.Minute*15, + true, + func(ctx context.Context) (done bool, err error) { + klog.V(2).Infof("Waiting for running pod %s/%s", c.nodeJoinerNamespace.GetName(), c.nodeJoinerPod.GetName()) + pod, err := c.Client.CoreV1().Pods(c.nodeJoinerNamespace.GetName()).Get(context.TODO(), c.nodeJoinerPod.GetName(), metav1.GetOptions{}) + if err == nil { + if len(pod.Status.ContainerStatuses) == 0 { + return false, nil + } + state := pod.Status.ContainerStatuses[0].State + if state.Waiting != nil { + switch state.Waiting.Reason { + case "InvalidImageName": + return true, fmt.Errorf("unable to pull image: %v: %v", state.Waiting.Reason, state.Waiting.Message) + case "ErrImagePull", "ImagePullBackOff": + klog.V(1).Infof("Unable to pull image (%s), retrying", state.Waiting.Reason) + return false, nil + } + } + return state.Running != nil || state.Terminated != nil, nil + } + if retry.IsHTTPClientError(err) { + return false, nil + } + return false, err + }) +} + +func (c *BaseNodeImageCommand) createRolesAndBindings(ctx context.Context) error { + nodeJoinerRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "node-joiner-", + Annotations: map[string]string{ + "oc.openshift.io/command": c.command, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Namespace", + Name: c.nodeJoinerNamespace.GetName(), + UID: c.nodeJoinerNamespace.GetUID(), + }, + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "config.openshift.io", + }, + Resources: []string{ + "clusterversions", + "infrastructures", + "proxies", + "imagedigestmirrorsets", + "imagecontentpolicies", + }, + Verbs: []string{ + "get", + "list", + }, + }, + { + APIGroups: []string{ + "machineconfiguration.openshift.io", + }, + Resources: []string{ + "machineconfigs", + }, + Verbs: []string{ + "get", + "list", + }, + }, + { + APIGroups: []string{ + "certificates.k8s.io", + }, + Resources: []string{ + "certificatesigningrequests", + }, + Verbs: []string{ + "get", + "list", + }, + }, + { + APIGroups: []string{ + "", + }, + Resources: []string{ + "configmaps", + "nodes", + "pods", + "nodes", + }, + Verbs: []string{ + "get", + "list", + }, + }, + { + APIGroups: []string{ + "", + }, + Resources: []string{ + "secrets", + }, + Verbs: []string{ + "get", + "list", + "create", + "update", + }, + }, + }, + } + cr, err := c.Client.RbacV1().ClusterRoles().Create(ctx, nodeJoinerRole, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("cannot create role: %w", err) + } + c.nodeJoinerRole = cr + + _, err = c.Client.RbacV1().ClusterRoleBindings().Create(ctx, c.clusterRoleBindings(), metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("cannot create role binding: %w", err) + } + + return nil +} + +func (c *BaseNodeImageCommand) baseComplete(f genericclioptions.RESTClientGetter) error { + c.RESTClientGetter = f + + var err error + if c.Config, err = f.ToRESTConfig(); err != nil { + return err + } + if c.Client, err = kubernetes.NewForConfig(c.Config); err != nil { + return err + } + if c.ConfigClient, err = configclient.NewForConfig(c.Config); err != nil { + return err + } + c.remoteExecutor = &exec.DefaultRemoteExecutor{} + return nil +} + +func (c *BaseNodeImageCommand) addBaseFlags(cmd *cobra.Command) *flag.FlagSet { + f := cmd.Flags() + c.SecurityOptions.Bind(f) + return f +} + +func (o *BaseNodeImageCommand) runNodeJoinerPod(ctx context.Context, tasks []func(context.Context) error) error { + for _, task := range tasks { + if err := task(ctx); err != nil { + return err + } + } + return nil +} diff --git a/pkg/cli/admin/nodeimage/create.go b/pkg/cli/admin/nodeimage/create.go index 55988f8d56..cb814e22b8 100644 --- a/pkg/cli/admin/nodeimage/create.go +++ b/pkg/cli/admin/nodeimage/create.go @@ -3,41 +3,31 @@ package nodeimage import ( "bytes" "context" + "encoding/json" "errors" "fmt" - "io" "io/fs" "os" "path/filepath" - "regexp" "strconv" "strings" "time" "github.com/spf13/cobra" - flag "github.com/spf13/pflag" "k8s.io/klog/v2" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" kapierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/genericiooptions" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" kutils "k8s.io/client-go/util/exec" "k8s.io/kubectl/pkg/cmd/exec" kcmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" "sigs.k8s.io/yaml" - configclient "github.com/openshift/client-go/config/clientset/versioned" - "github.com/openshift/library-go/pkg/operator/resource/retry" - ocrelease "github.com/openshift/oc/pkg/cli/admin/release" - imagemanifest "github.com/openshift/oc/pkg/cli/image/manifest" "github.com/openshift/oc/pkg/cli/rsync" ) @@ -77,6 +67,9 @@ var ( such case the '--mac-address' is the only mandatory flag - while all the others will be optional (note: any eventual configuration file present will be ignored). + + In case of a command failure a report.json file is automatically created + with the error details, and additional troubleshooting information. `) createExample = templates.Examples(` @@ -125,10 +118,7 @@ func NewCreate(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Co // NewCreateOptions creates the options for the create command func NewCreateOptions(streams genericiooptions.IOStreams) *CreateOptions { return &CreateOptions{ - BaseNodeImageCommand: BaseNodeImageCommand{ - IOStreams: streams, - command: createCommand, - }, + BaseNodeImageCommand: *newBaseNodeImageCommand(streams, createCommand, "create"), } } @@ -145,12 +135,15 @@ type CreateOptions struct { OutputName string // GeneratePXEFiles generates files for PXE boot instead of an ISO GeneratePXEFiles bool + // GenerateReport allows to save the report in the asset folder + GenerateReport bool // Simpler interface for creating a single node SingleNodeOpts *singleNodeCreateOptions - nodeJoinerExitCode int - rsyncRshCmd string + report *report + rsyncRshCmd string + fileWriter fileWriter } type singleNodeCreateOptions struct { @@ -169,6 +162,7 @@ func (o *CreateOptions) AddFlags(cmd *cobra.Command) { flags.StringVar(&o.AssetsDir, "dir", o.AssetsDir, "The path containing the configuration file, used also to store the generated artifacts.") flags.StringVarP(&o.OutputName, "output-name", "o", "", "The name of the output image.") flags.BoolVarP(&o.GeneratePXEFiles, "pxe", "p", false, "Instead of an ISO, create files that can be used for PXE boot") + flags.BoolVarP(&o.GenerateReport, "report", "r", false, "When set, the report.json is always generated in the asset folder") flags.StringP(snFlagMacAddress, "m", "", "Single node flag. MAC address used to identify the host to apply the configuration. If specified, the nodes-config.yaml config file will not be used.") usageFmt := "Single node flag. %s. Valid only when `mac-address` is defined." @@ -199,10 +193,18 @@ func (o *CreateOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args [] o.copyStrategy = func(o *rsync.RsyncOptions) rsync.CopyStrategy { return rsync.NewDefaultCopyStrategy(o) } - + o.fileWriter = o return o.completeSingleNodeOptions(cmd) } +type fileWriter interface { + WriteFile(name string, data []byte, perm os.FileMode) error +} + +func (o *CreateOptions) WriteFile(name string, data []byte, perm os.FileMode) error { + return os.WriteFile(name, data, perm) +} + func (o *CreateOptions) completeSingleNodeOptions(cmd *cobra.Command) error { snOpts := &singleNodeCreateOptions{} @@ -281,6 +283,7 @@ func (o *CreateOptions) Run() error { defer o.cleanup(ctx) tasks := []func(context.Context) error{ + o.checkMinSupportedVersion, o.getNodeJoinerPullSpec, o.createNamespace, o.createServiceAccount, @@ -293,15 +296,16 @@ func (o *CreateOptions) Run() error { return err } - err = o.waitForCompletion(ctx) - // Something went wrong during the node-joiner tool execution, - // let's show the logs and return an error - if err != nil || o.nodeJoinerExitCode != 0 { - printErr := o.printLogsInPod(ctx) - if printErr != nil { - return printErr + if err = o.waitForCompletion(ctx); err != nil { + // Something went wrong during the node-joiner tool execution + o.attachPodLogsToReport(ctx, err) + } + if o.report.Result.ExitCode != 0 { + o.log("command execution failed. Reason: %s", o.report.Result.ErrorMessage) + if err = o.saveReport(); err != nil { + return err } - return fmt.Errorf("image generation error: %v (exit code: %d)", err, o.nodeJoinerExitCode) + return kcmdutil.ErrExit } err = o.copyArtifactsFromNodeJoinerPod() @@ -314,28 +318,53 @@ func (o *CreateOptions) Run() error { return err } - klog.V(1).Info("Command successfully completed") + if o.GenerateReport { + if err = o.saveReport(); err != nil { + return err + } + } + + o.log("Command successfully completed") return nil } -func (o *CreateOptions) printLogsInPod(ctx context.Context) error { - klog.V(1).Info("Printing pod logs") +func (o *CreateOptions) attachPodLogsToReport(ctx context.Context, err error) { + o.log("unexpected error caught while running the command, storing pod logs in report") + + // Create a new report if not already present + if o.report == nil { + o.report = &report{ + stageHeader: stageHeader{}, + Result: &reportResult{}, + } + } + + detailedErrorMessage := "" logOptions := &corev1.PodLogOptions{ Container: nodeJoinerContainer, Timestamps: true, } - readCloser, err := o.Client.CoreV1().Pods(o.nodeJoinerNamespace.GetName()).GetLogs(o.nodeJoinerPod.GetName(), logOptions).Stream(ctx) - if err != nil { - return err + readCloser, getLogsErr := o.Client.CoreV1().Pods(o.nodeJoinerNamespace.GetName()).GetLogs(o.nodeJoinerPod.GetName(), logOptions).Stream(ctx) + if getLogsErr != nil { + detailedErrorMessage = getLogsErr.Error() + } else { + defer readCloser.Close() + var buf bytes.Buffer + _, readErr := buf.ReadFrom(readCloser) + if readErr != nil { + detailedErrorMessage = readErr.Error() + } else { + detailedErrorMessage = buf.String() + } } - defer readCloser.Close() - _, err = io.Copy(o.IOStreams.ErrOut, readCloser) - return err + o.report.Result.ExitCode = 1 + o.report.Result.ErrorMessage = err.Error() + o.report.Result.DetailedErrorMessage = detailedErrorMessage } func (o *CreateOptions) copyArtifactsFromNodeJoinerPod() error { - klog.V(2).Infof("Copying artifacts from %s", o.nodeJoinerPod.GetName()) + logMessage := "Saving ISO image to %s" rsyncOptions := &rsync.RsyncOptions{ Namespace: o.nodeJoinerNamespace.GetName(), Source: &rsync.PathSpec{PodName: o.nodeJoinerPod.GetName(), Path: "/assets/"}, @@ -353,8 +382,10 @@ func (o *CreateOptions) copyArtifactsFromNodeJoinerPod() error { if o.GeneratePXEFiles { rsyncOptions.RsyncInclude = []string{"boot-artifacts/*"} rsyncOptions.RsyncExclude = []string{} + logMessage = "Saving PXE artifacts to %s" } rsyncOptions.Strategy = o.copyStrategy(rsyncOptions) + o.log(logMessage, o.AssetsDir) return rsyncOptions.RunRsync() } @@ -402,24 +433,25 @@ func (o *CreateOptions) renameImageIfOutputNameIsSpecified() error { return nil } -func (o *CreateOptions) waitForCompletion(ctx context.Context) error { - klog.V(2).Infof("Starting command in pod %s", o.nodeJoinerPod.GetName()) - // Wait for the node-joiner pod to come up - err := o.waitForContainerRunning(ctx) +func (o *CreateOptions) saveReport() error { + o.log("Saving report file") + data, err := json.MarshalIndent(o.report, "", " ") if err != nil { return err } + return o.fileWriter.WriteFile(filepath.Join(o.AssetsDir, "report.json"), data, 0644) +} - // Wait for the node-joiner cli tool to complete - return wait.PollUntilContextTimeout( +func (o *CreateOptions) nodeJoinerPodExec(ctx context.Context, command ...string) ([]byte, error) { + w := &bytes.Buffer{} + wErr := &bytes.Buffer{} + + err := wait.PollUntilContextTimeout( ctx, time.Second*5, time.Minute*15, true, func(ctx context.Context) (done bool, err error) { - w := &bytes.Buffer{} - wErr := &bytes.Buffer{} - execOptions := &exec.ExecOptions{ StreamOptions: exec.StreamOptions{ Namespace: o.nodeJoinerNamespace.GetName(), @@ -436,9 +468,7 @@ func (o *CreateOptions) waitForCompletion(ctx context.Context) error { Executor: o.remoteExecutor, PodClient: o.Client.CoreV1(), Config: o.Config, - Command: []string{ - "cat", "/assets/exit_code", - }, + Command: command, } err = execOptions.Validate() @@ -446,7 +476,7 @@ func (o *CreateOptions) waitForCompletion(ctx context.Context) error { return false, err } - klog.V(1).Info("Image generation in progress, please wait") + klog.V(2).Infof("Running command on pod %s/%s: %v", o.nodeJoinerNamespace.GetName(), o.nodeJoinerPod.GetName(), command) err = execOptions.Run() if err != nil { var codeExitErr kutils.CodeExitError @@ -459,13 +489,124 @@ func (o *CreateOptions) waitForCompletion(ctx context.Context) error { return false, nil } - // Extract node-joiner tool exit code on completion - o.nodeJoinerExitCode, err = strconv.Atoi(w.String()) + return true, nil + }) + + if err != nil { + return nil, fmt.Errorf("error caught while executing remote command: %w. Error output: %s", err, wErr.String()) + } + + klog.V(2).Infof("Remote command output: %s. Error output: %s", w.String(), wErr.String()) + return w.Bytes(), nil +} + +func (o *CreateOptions) waitForCompletion(ctx context.Context) error { + klog.V(2).Infof("Starting command in pod %s", o.nodeJoinerPod.GetName()) + // Wait for the node-joiner pod to come up + err := o.waitForRunningPod(ctx) + if err != nil { + return err + } + + // Prior than version 4.18, node-joiner tool generated only an + // exit_code file on command completion, to signal it was done and + // to provide the exit result. + useOnlyExitCodeFile, err := o.isClusterVersionLessThan(ctx, "4.18") + if err != nil { + return err + } + if useOnlyExitCodeFile { + return o.monitorExitCodeFile(ctx) + } + return o.monitorWorkflowReport(ctx) +} + +func (o *CreateOptions) monitorExitCodeFile(ctx context.Context) error { + cmdOutput, err := o.nodeJoinerPodExec(ctx, "cat", "/assets/exit_code") + if err != nil { + return err + } + + // Extract node-joiner tool exit code on completion + exitCode, err := strconv.Atoi(string(cmdOutput)) + if err != nil { + return err + } + o.report = &report{ + stageHeader: stageHeader{}, + Result: &reportResult{ + ExitCode: exitCode, + }, + } + + return nil +} + +type report struct { + stageHeader + Stages []*stage `json:"stages,omitempty"` + Result *reportResult `json:"result"` +} + +type stageHeader struct { + Identifier string `json:"id"` + Desc string `json:"description,omitempty"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` +} + +type stage struct { + stageHeader + Result string `json:"result,omitempty"` + SubStages []*stage `json:"sub_stages,omitempty"` +} + +type reportResult struct { + ExitCode int `json:"exit_code"` + ErrorMessage string `json:"error_message,omitempty"` + DetailedErrorMessage string `json:"detailed_error_message,omitempty"` +} + +func (o *CreateOptions) monitorWorkflowReport(ctx context.Context) error { + var r report + shownStages := map[string]bool{} + + err := wait.PollUntilContextTimeout( + ctx, + time.Second*5, + time.Minute*15, + true, + func(ctx context.Context) (done bool, err error) { + cmdOutput, err := o.nodeJoinerPodExec(ctx, "cat", "/assets/report.json") if err != nil { return false, err } - return true, nil + if err := json.Unmarshal(cmdOutput, &r); err != nil { + return false, fmt.Errorf("error while parsing the report: %w", err) + } + + for _, s := range r.Stages { + if _, found := shownStages[s.Identifier]; !found { + shownStages[s.Identifier] = true + o.log("%s", s.Desc) + } + for _, sub := range s.SubStages { + if _, found := shownStages[sub.Identifier]; !found { + shownStages[sub.Identifier] = true + o.log(" %s", sub.Desc) + } + } + } + + // Wait until the report is mark as completed. + return !r.EndTime.IsZero(), nil }) + if err != nil { + return err + } + + o.report = &r + return nil } func (o *CreateOptions) createConfigFileFromFlags() ([]byte, error) { @@ -625,6 +766,7 @@ func (o *CreateOptions) createPod(ctx context.Context) error { return err } + o.log("Launching command") pod, err := o.Client.CoreV1().Pods(o.nodeJoinerNamespace.GetName()).Create(ctx, nodeJoinerPod, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("cannot create pod: %w", err) @@ -659,320 +801,3 @@ func (o *CreateOptions) configurePodProxySetting(ctx context.Context, pod *corev } return nil } - -type BaseNodeImageCommand struct { - genericiooptions.IOStreams - SecurityOptions imagemanifest.SecurityOptions - - Config *rest.Config - remoteExecutor exec.RemoteExecutor - ConfigClient configclient.Interface - Client kubernetes.Interface - nodeJoinerImage string - nodeJoinerNamespace *corev1.Namespace - nodeJoinerServiceAccount *corev1.ServiceAccount - nodeJoinerRole *rbacv1.ClusterRole - RESTClientGetter genericclioptions.RESTClientGetter - nodeJoinerPod *corev1.Pod - command string -} - -func (c *BaseNodeImageCommand) getNodeJoinerPullSpec(ctx context.Context) error { - // Get the current cluster release version. - releaseImage, err := c.fetchClusterReleaseImage(ctx) - if err != nil { - return err - } - - // Extract the baremetal-installer image pullspec, since it - // provide the node-joiner tool. - opts := ocrelease.NewInfoOptions(c.IOStreams) - opts.SecurityOptions = c.SecurityOptions - release, err := opts.LoadReleaseInfo(releaseImage, false) - if err != nil { - return err - } - - tagName := "baremetal-installer" - for _, tag := range release.References.Spec.Tags { - if tag.Name == tagName { - c.nodeJoinerImage = tag.From.Name - return nil - } - } - - return fmt.Errorf("no image tag %q exists in the release image %s", tagName, releaseImage) -} - -func (c *BaseNodeImageCommand) fetchClusterReleaseImage(ctx context.Context) (string, error) { - cv, err := c.ConfigClient.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{}) - if err != nil { - if kapierrors.IsNotFound(err) || kapierrors.ReasonForError(err) == metav1.StatusReasonUnknown { - klog.V(2).Infof("Unable to find cluster version object from cluster: %v", err) - return "", fmt.Errorf("command expects a connection to an OpenShift 4.x server") - } - } - // Adds a guardrail for node-image commands which is supported only for Openshift version 4.17 and later - currentVersion := cv.Status.Desired.Version - matches := regexp.MustCompile(`^(\d+[.]\d+)[.].*`).FindStringSubmatch(currentVersion) - if len(matches) < 2 { - return "", fmt.Errorf("failed to parse major.minor version from ClusterVersion status.desired.version %q", currentVersion) - } else if matches[1] < nodeJoinerMinimumSupportedVersion { - return "", fmt.Errorf("the 'oc adm node-image' command is only available for OpenShift versions %s and later", nodeJoinerMinimumSupportedVersion) - } - image := cv.Status.Desired.Image - if len(image) == 0 && cv.Spec.DesiredUpdate != nil { - image = cv.Spec.DesiredUpdate.Image - } - if len(image) == 0 { - return "", fmt.Errorf("the server is not reporting a release image at this time") - } - - return image, nil -} - -func (c *BaseNodeImageCommand) createNamespace(ctx context.Context) error { - nsNodeJoiner := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "openshift-node-joiner-", - Annotations: map[string]string{ - "oc.openshift.io/command": c.command, - "openshift.io/node-selector": "", - }, - }, - } - - ns, err := c.Client.CoreV1().Namespaces().Create(ctx, nsNodeJoiner, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("cannot create namespace: %w", err) - } - - c.nodeJoinerNamespace = ns - return nil -} - -func (c *BaseNodeImageCommand) cleanup(ctx context.Context) { - if c.nodeJoinerNamespace == nil { - return - } - - err := c.Client.CoreV1().Namespaces().Delete(ctx, c.nodeJoinerNamespace.GetName(), metav1.DeleteOptions{}) - if err != nil { - klog.Errorf("cannot delete namespace %s: %v\n", c.nodeJoinerNamespace.GetName(), err) - } -} - -func (c *BaseNodeImageCommand) createServiceAccount(ctx context.Context) error { - nodeJoinerServiceAccount := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "node-joiner-", - Annotations: map[string]string{ - "oc.openshift.io/command": c.command, - }, - Namespace: c.nodeJoinerNamespace.GetName(), - }, - } - - sa, err := c.Client.CoreV1().ServiceAccounts(c.nodeJoinerNamespace.GetName()).Create(ctx, nodeJoinerServiceAccount, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("cannot create service account: %w", err) - } - - c.nodeJoinerServiceAccount = sa - return nil -} - -func (c *BaseNodeImageCommand) clusterRoleBindings() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "node-joiner-monitor-", - Annotations: map[string]string{ - "oc.openshift.io/command": c.command, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "Namespace", - Name: c.nodeJoinerNamespace.GetName(), - UID: c.nodeJoinerNamespace.GetUID(), - }, - }, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: c.nodeJoinerServiceAccount.GetName(), - Namespace: c.nodeJoinerNamespace.GetName(), - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: c.nodeJoinerRole.GetName(), - }, - } -} - -func (c *BaseNodeImageCommand) waitForContainerRunning(ctx context.Context) error { - // Wait for the node-joiner pod to come up - return wait.PollUntilContextTimeout( - ctx, - time.Second*1, - time.Minute*5, - true, - func(ctx context.Context) (done bool, err error) { - pod, err := c.Client.CoreV1().Pods(c.nodeJoinerNamespace.GetName()).Get(context.TODO(), c.nodeJoinerPod.GetName(), metav1.GetOptions{}) - if err == nil { - klog.V(2).Info("Waiting for pod") - if len(pod.Status.ContainerStatuses) == 0 { - return false, nil - } - state := pod.Status.ContainerStatuses[0].State - if state.Waiting != nil { - switch state.Waiting.Reason { - case "ErrImagePull", "ImagePullBackOff", "InvalidImageName": - return true, fmt.Errorf("unable to pull image: %v: %v", state.Waiting.Reason, state.Waiting.Message) - } - } - return state.Running != nil || state.Terminated != nil, nil - } - if retry.IsHTTPClientError(err) { - return false, nil - } - return false, err - }) -} - -func (c *BaseNodeImageCommand) createRolesAndBindings(ctx context.Context) error { - nodeJoinerRole := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "node-joiner-", - Annotations: map[string]string{ - "oc.openshift.io/command": c.command, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "Namespace", - Name: c.nodeJoinerNamespace.GetName(), - UID: c.nodeJoinerNamespace.GetUID(), - }, - }, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "config.openshift.io", - }, - Resources: []string{ - "clusterversions", - "infrastructures", - "proxies", - "imagedigestmirrorsets", - "imagecontentpolicies", - }, - Verbs: []string{ - "get", - "list", - }, - }, - { - APIGroups: []string{ - "machineconfiguration.openshift.io", - }, - Resources: []string{ - "machineconfigs", - }, - Verbs: []string{ - "get", - "list", - }, - }, - { - APIGroups: []string{ - "certificates.k8s.io", - }, - Resources: []string{ - "certificatesigningrequests", - }, - Verbs: []string{ - "get", - "list", - }, - }, - { - APIGroups: []string{ - "", - }, - Resources: []string{ - "configmaps", - "nodes", - "pods", - "nodes", - }, - Verbs: []string{ - "get", - "list", - }, - }, - { - APIGroups: []string{ - "", - }, - Resources: []string{ - "secrets", - }, - Verbs: []string{ - "get", - "list", - "create", - "update", - }, - }, - }, - } - cr, err := c.Client.RbacV1().ClusterRoles().Create(ctx, nodeJoinerRole, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("cannot create role: %w", err) - } - c.nodeJoinerRole = cr - - _, err = c.Client.RbacV1().ClusterRoleBindings().Create(ctx, c.clusterRoleBindings(), metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("cannot create role binding: %w", err) - } - - return nil -} - -func (c *BaseNodeImageCommand) baseComplete(f genericclioptions.RESTClientGetter) error { - c.RESTClientGetter = f - - var err error - if c.Config, err = f.ToRESTConfig(); err != nil { - return err - } - if c.Client, err = kubernetes.NewForConfig(c.Config); err != nil { - return err - } - if c.ConfigClient, err = configclient.NewForConfig(c.Config); err != nil { - return err - } - c.remoteExecutor = &exec.DefaultRemoteExecutor{} - return nil -} - -func (c *BaseNodeImageCommand) addBaseFlags(cmd *cobra.Command) *flag.FlagSet { - f := cmd.Flags() - c.SecurityOptions.Bind(f) - return f -} - -func (o *BaseNodeImageCommand) runNodeJoinerPod(ctx context.Context, tasks []func(context.Context) error) error { - for _, task := range tasks { - if err := task(ctx); err != nil { - return err - } - } - return nil -} diff --git a/pkg/cli/admin/nodeimage/create_test.go b/pkg/cli/admin/nodeimage/create_test.go index a71639cf99..e96fa6de2d 100644 --- a/pkg/cli/admin/nodeimage/create_test.go +++ b/pkg/cli/admin/nodeimage/create_test.go @@ -19,6 +19,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "slices" "strings" "testing" @@ -34,6 +35,7 @@ import ( restclient "k8s.io/client-go/rest" clientgotesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/remotecommand" + utilexec "k8s.io/utils/exec" "github.com/distribution/distribution/v3/manifest/schema2" configv1 "github.com/openshift/api/config/v1" @@ -106,12 +108,26 @@ func strPtr(s string) *string { return &s } +func createCmdOutput(t *testing.T, r report) string { + t.Helper() + out, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + return string(out) +} + func TestRun(t *testing.T) { ClusterVersion_4_16_ObjectFn := func(repo string, manifestDigest string) []runtime.Object { cvobj := defaultClusterVersionObjectFn(repo, manifestDigest) clusterVersion := cvobj[0].(*configv1.ClusterVersion) clusterVersion.Status.Desired.Version = "4.16.6-x86_64" - + return cvobj + } + ClusterVersion_4_17_ObjectFn := func(repo string, manifestDigest string) []runtime.Object { + cvobj := defaultClusterVersionObjectFn(repo, manifestDigest) + clusterVersion := cvobj[0].(*configv1.ClusterVersion) + clusterVersion.Status.Desired.Version = "4.17.4-x86_64" return cvobj } @@ -124,6 +140,7 @@ func TestRun(t *testing.T) { objects func(string, string) []runtime.Object remoteExecOutput string + expectedErrorCode int expectedError string expectedPod func(t *testing.T, pod *corev1.Pod) expectedRsyncInclude string @@ -145,18 +162,26 @@ func TestRun(t *testing.T) { expectedRsyncInclude: "boot-artifacts/*", }, { - name: "node-joiner tool failure", - nodesConfig: defaultNodesConfigYaml, - objects: defaultClusterVersionObjectFn, - remoteExecOutput: "1", - expectedError: `image generation error: (exit code: 1)`, + name: "node-joiner tool failure", + nodesConfig: defaultNodesConfigYaml, + objects: defaultClusterVersionObjectFn, + remoteExecOutput: createCmdOutput(t, report{ + stageHeader: stageHeader{ + EndTime: time.Date(2024, 11, 14, 0, 0, 0, 0, time.UTC), + }, + Result: &reportResult{ + ExitCode: 127, + ErrorMessage: "Some error message", + }, + }), + expectedErrorCode: 1, + expectedError: `exit`, }, { - name: "node-joiner unsupported prior to 4.17", - nodesConfig: defaultNodesConfigYaml, - objects: ClusterVersion_4_16_ObjectFn, - remoteExecOutput: "1", - expectedError: fmt.Sprintf("the 'oc adm node-image' command is only available for OpenShift versions %s and later", nodeJoinerMinimumSupportedVersion), + name: "node-joiner unsupported prior to 4.17", + nodesConfig: defaultNodesConfigYaml, + objects: ClusterVersion_4_16_ObjectFn, + expectedError: fmt.Sprintf("the 'oc adm node-image' command is only available for OpenShift versions %s and later", nodeJoinerMinimumSupportedVersion), }, { name: "missing cluster connection", @@ -204,6 +229,12 @@ func TestRun(t *testing.T) { } }, }, + { + name: "basic report for ocp < 4.18", + nodesConfig: defaultNodesConfigYaml, + objects: ClusterVersion_4_17_ObjectFn, + remoteExecOutput: "0", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -223,12 +254,23 @@ func TestRun(t *testing.T) { objs = tc.objects(fakeReg.URL()[len("https://"):], fakeReg.fakeManifestDigest) } + fakeRemoteExec.execOut = createCmdOutput(t, report{ + stageHeader: stageHeader{ + Identifier: "report-test", + EndTime: time.Date(2024, 11, 14, 0, 0, 0, 0, time.UTC), + }, + Result: &reportResult{ + ExitCode: 0, + }, + }) if tc.remoteExecOutput != "" { fakeRemoteExec.execOut = tc.remoteExecOutput } // Create another fake for the copy action fakeCp := &fakeCopier{} + var logBuffer bytes.Buffer + // Prepare the command options with all the fakes o := &CreateOptions{ BaseNodeImageCommand: BaseNodeImageCommand{ @@ -238,6 +280,7 @@ func TestRun(t *testing.T) { Client: fakeClient, Config: fakeRestConfig, remoteExecutor: fakeRemoteExec, + LogOut: &logBuffer, }, FSys: fakeFileSystem, copyStrategy: func(o *rsync.RsyncOptions) rsync.CopyStrategy { @@ -247,13 +290,14 @@ func TestRun(t *testing.T) { AssetsDir: tc.assetsDir, GeneratePXEFiles: tc.generatePXEFiles, + fileWriter: mockFileWriter{}, } // Since the fake registry creates a self-signed cert, let's configure // the command options accordingly o.SecurityOptions.Insecure = true err := o.Run() - assertContainerImageAndErrors(t, err, fakeReg, fakeClient, tc.expectedError, nodeJoinerContainer) + assertContainerImageAndErrors(t, err, fakeReg, fakeClient, tc.expectedErrorCode, tc.expectedError, nodeJoinerContainer) // Perform additional checks on the generated node-joiner pod if tc.expectedPod != nil { @@ -276,6 +320,12 @@ func TestRun(t *testing.T) { } } +type mockFileWriter struct{} + +func (mockFileWriter) WriteFile(name string, data []byte, perm os.FileMode) error { + return nil +} + // fakeRegistry creates a fake Docker registry configured to serve the minimum // amount of data required to allow a successfull execution of the command to // retrieve the release info, and to extract the baremetal-installer pullspec. @@ -546,7 +596,7 @@ func getTestPod(fakeClient *fake.Clientset, podName string) *corev1.Pod { return pod } -func assertContainerImageAndErrors(t *testing.T, runErr error, fakeReg *fakeRegistry, fakeClient *fake.Clientset, expectedError, podName string) { +func assertContainerImageAndErrors(t *testing.T, runErr error, fakeReg *fakeRegistry, fakeClient *fake.Clientset, expectedErrorCode int, expectedError, podName string) { if expectedError == "" { if runErr != nil { t.Fatalf("unexpected error: %v", runErr) @@ -561,8 +611,13 @@ func assertContainerImageAndErrors(t *testing.T, runErr error, fakeReg *fakeRegi if runErr == nil { t.Fatalf("expected error not received: %s", expectedError) } - if !strings.Contains(runErr.Error(), expectedError) { - t.Fatalf("expected error: %s, actual: %v", expectedError, runErr.Error()) + if codeExitErr, ok := runErr.(utilexec.CodeExitError); ok { + if codeExitErr.Code != expectedErrorCode { + t.Fatalf("expected error code: %d, actual: %d", expectedErrorCode, codeExitErr.Code) + } + } + if runErr.Error() != expectedError { + t.Fatalf("expected error: %s, actual: %s", expectedError, runErr.Error()) } } } diff --git a/pkg/cli/admin/nodeimage/monitor.go b/pkg/cli/admin/nodeimage/monitor.go index bff3713371..8b3143be68 100644 --- a/pkg/cli/admin/nodeimage/monitor.go +++ b/pkg/cli/admin/nodeimage/monitor.go @@ -146,6 +146,7 @@ func (o *MonitorOptions) Run() error { defer o.cleanup(ctx) tasks := []func(context.Context) error{ + o.checkMinSupportedVersion, o.getNodeJoinerPullSpec, o.createNamespace, o.createServiceAccount, @@ -160,7 +161,7 @@ func (o *MonitorOptions) Run() error { podName := o.nodeJoinerPod.GetName() - if err := o.waitForContainerRunning(ctx); err != nil { + if err := o.waitForRunningPod(ctx); err != nil { klog.Errorf("monitoring did not start: %s", err) return fmt.Errorf("monitoring did not start for pod %s: %s", podName, err) } diff --git a/pkg/cli/admin/nodeimage/monitor_test.go b/pkg/cli/admin/nodeimage/monitor_test.go index d38b5f15d4..a830deb87b 100644 --- a/pkg/cli/admin/nodeimage/monitor_test.go +++ b/pkg/cli/admin/nodeimage/monitor_test.go @@ -123,7 +123,7 @@ func TestMonitorRun(t *testing.T) { o.SecurityOptions.Insecure = true err := o.Run() - assertContainerImageAndErrors(t, err, fakeReg, fakeClient, tc.expectedError, nodeJoinerMonitorContainer) + assertContainerImageAndErrors(t, err, fakeReg, fakeClient, -1, tc.expectedError, nodeJoinerMonitorContainer) if tc.expectedError == "" { if fakeLogContent != logContents.String() { t.Errorf("expected %v, actual %v", fakeLogContent, logContents.String()) diff --git a/pkg/cli/admin/release/image_mapper.go b/pkg/cli/admin/release/image_mapper.go index 9cc0bc6408..6525d85f63 100644 --- a/pkg/cli/admin/release/image_mapper.go +++ b/pkg/cli/admin/release/image_mapper.go @@ -66,7 +66,7 @@ func (p *Payload) Rewrite(allowTags bool, fn func(component string) imagereferen continue } path := filepath.Join(p.path, file.Name()) - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return err } @@ -78,7 +78,7 @@ func (p *Payload) Rewrite(allowTags bool, fn func(component string) imagereferen continue } klog.V(6).Infof("Rewrote\n%s\n\nto\n\n%s\n", string(data), string(out)) - if err := ioutil.WriteFile(path, out, file.Mode()); err != nil { + if err := os.WriteFile(path, out, file.Mode()); err != nil { return err } } @@ -99,7 +99,7 @@ func (p *Payload) References() (*imageapi.ImageStream, error) { } func parseImageStream(path string) (*imageapi.ImageStream, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if os.IsNotExist(err) { return nil, err } @@ -275,7 +275,7 @@ func NewImageMapper(images map[string]ImageReference) (ManifestMapper, error) { ref := images[name] suffix := parts[3] - klog.V(2).Infof("found repository %q with locator %q in the input, switching to %q (from pattern %s)", string(repository), string(suffix), ref.TargetPullSpec, pattern) + klog.V(2).Infof("found repository %q with locator %q in the input, switching to %q (from pattern %s)", repository, string(suffix), ref.TargetPullSpec, pattern) switch { case len(suffix) == 0: // we found a repository, but no tag or digest (implied latest), or we got an exact match @@ -335,9 +335,7 @@ func ComponentReferencesForImageStream(is *imageapi.ImageStream) (func(string) i }, nil } -const ( - componentVersionFormat = `([\W]|^)0\.0\.1-snapshot([a-z0-9\-]*)` -) +var componentVersionRe = regexp.MustCompile(`(\W|^)0\.0\.1-snapshot([a-z0-9\-]*)`) // NewComponentVersionsMapper substitutes strings of the form 0.0.1-snapshot with releaseName and strings // of the form 0.0.1-snapshot-[component] with the version value located in versions, or returns an error. @@ -352,17 +350,12 @@ func NewComponentVersionsMapper(releaseName string, versions ComponentVersions, } else { releaseName = "" } - re, err := regexp.Compile(componentVersionFormat) - if err != nil { - return func([]byte) ([]byte, error) { - return nil, fmt.Errorf("component versions mapper regex: %v", err) - } - } + return func(data []byte) ([]byte, error) { var missing []string var conflicts []string - data = re.ReplaceAllFunc(data, func(part []byte) []byte { - matches := re.FindSubmatch(part) + data = componentVersionRe.ReplaceAllFunc(data, func(part []byte) []byte { + matches := componentVersionRe.FindSubmatch(part) if matches == nil { return part } @@ -415,7 +408,7 @@ var ( // reAllowedVersionKey limits the allowed component name to a strict subset reAllowedVersionKey = regexp.MustCompile(`^[a-z0-9]+[\-a-z0-9]*[a-z0-9]+$`) // reAllowedDisplayNameKey limits the allowed component name to a strict subset - reAllowedDisplayNameKey = regexp.MustCompile(`^[a-zA-Z0-9\-\:\s\(\)]+$`) + reAllowedDisplayNameKey = regexp.MustCompile(`^[a-zA-Z0-9\-:\s()]+$`) ) // ComponentVersion includes the version and optional display name. @@ -434,7 +427,7 @@ func (v ComponentVersion) String() string { // labels removed, but prerelease segments are preserved. type ComponentVersions map[string]ComponentVersion -// OrderedKeys returns the keys in this map in lexigraphic order. +// OrderedKeys returns the keys in this map in lexicographic order. func (v ComponentVersions) OrderedKeys() []string { keys := make([]string, 0, len(v)) for k := range v { diff --git a/pkg/cli/admin/release/image_mapper_test.go b/pkg/cli/admin/release/image_mapper_test.go index 554cd5285e..c5ad01be35 100644 --- a/pkg/cli/admin/release/image_mapper_test.go +++ b/pkg/cli/admin/release/image_mapper_test.go @@ -5,11 +5,11 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" "k8s.io/cli-runtime/pkg/genericiooptions" imageapi "github.com/openshift/api/image/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/diff" ) func TestNewImageMapper(t *testing.T) { @@ -265,8 +265,6 @@ func TestNewExactMapper(t *testing.T) { } func TestNewComponentVersionsMapper(t *testing.T) { - type args struct { - } tests := []struct { name string releaseName string @@ -372,8 +370,6 @@ func TestNewComponentVersionsMapper(t *testing.T) { } func Test_parseComponentVersionsLabel(t *testing.T) { - type args struct { - } tests := []struct { name string label string @@ -700,18 +696,18 @@ func Test_loadImageStreamTransforms(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ioStream := genericiooptions.NewTestIOStreamsDiscard() - got, got1, got2, err := loadImageStreamTransforms(tt.input, tt.local, tt.allowMissingImages, tt.src, ioStream.ErrOut) + got, gotTags, gotRefs, err := loadImageStreamTransforms(tt.input, tt.local, tt.allowMissingImages, tt.src, ioStream.ErrOut) if (err != nil) != tt.wantErr { t.Fatalf("loadImageStreamTransforms() error = %v, wantErr %v", err, tt.wantErr) } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("%s", diff.ObjectReflectDiff(got, tt.want)) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("versions differ (-want +got):\n%s", diff) } - if !reflect.DeepEqual(got1, tt.wantTags) { - t.Errorf("%s", diff.ObjectReflectDiff(got1, tt.wantTags)) + if diff := cmp.Diff(tt.wantTags, gotTags); diff != "" { + t.Errorf("tags differ (-want +got):\n%s", diff) } - if !reflect.DeepEqual(got2, tt.wantRefs) { - t.Errorf("%s", diff.ObjectReflectDiff(got2, tt.wantRefs)) + if diff := cmp.Diff(tt.wantRefs, gotRefs); diff != "" { + t.Errorf("refs differ (-want +got):\n%s", diff) } }) } diff --git a/pkg/cli/admin/release/new.go b/pkg/cli/admin/release/new.go index 79365431bf..d746f92731 100644 --- a/pkg/cli/admin/release/new.go +++ b/pkg/cli/admin/release/new.go @@ -217,7 +217,7 @@ type NewOptions struct { cleanupFns []func() } -func (o *NewOptions) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string) error { +func (o *NewOptions) Complete(f kcmdutil.Factory, _ *cobra.Command, args []string) error { overlap := make(map[string]string) var mappings []Mapping for _, filename := range o.MappingFilenames { @@ -346,10 +346,7 @@ func (o *NewOptions) Run(ctx context.Context) error { o.AlwaysInclude = append(o.AlwaysInclude, o.ToImageBaseTag) } - exclude := sets.NewString() - for _, s := range o.Exclude { - exclude.Insert(s) - } + exclude := sets.New[string](o.Exclude...) metadata := make(map[string]imageData) var ordered []string @@ -795,7 +792,7 @@ func (o *NewOptions) Run(ctx context.Context) error { return nil } -func resolveImageStreamTagsToReferenceMode(inputIS, is *imageapi.ImageStream, referenceMode string, exclude sets.String) error { +func resolveImageStreamTagsToReferenceMode(inputIS, is *imageapi.ImageStream, referenceMode string, exclude sets.Set[string]) error { switch referenceMode { case "public", "", "source": forceExternal := referenceMode == "public" || referenceMode == "" @@ -1497,26 +1494,6 @@ func hasTag(tags []imageapi.TagReference, tag string) *imageapi.TagReference { return nil } -func pruneEmptyDirectories(dir string) error { - return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - return nil - } - names, err := os.ReadDir(path) - if err != nil { - return err - } - if len(names) > 0 { - return nil - } - klog.V(4).Infof("Component %s does not have any manifests", path) - return os.Remove(path) - }) -} - type Mapping struct { Source string Destination string diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.output index ad6ce9e566..497519dc29 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.output @@ -1,8 +1,3 @@ -Upgradeable=False - - Reason: AdminAckRequired - Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. - Upstream: https://api.integration.openshift.com/api/upgrades_info/graph Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) @@ -10,20 +5,24 @@ Updates to 4.13: Version: 4.13.50 Image: quay.io/openshift-release-dev/ocp-release@sha256:6afb11e1cac46fd26476ca134072937115256b9c6360f7a1cd1812992c065f02 - Reason: EvaluationFailed - Message: Exposure to ARODNSWrongBootSequence is unknown due to an evaluation failure: client-side throttling: only 8m30.897651224s has elapsed since the last match call completed for this cluster condition backend; this cached cluster condition request has been queued for later execution + Reason: MultipleReasons + Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. + + Exposure to ARODNSWrongBootSequence is unknown due to an evaluation failure: client-side throttling: only 8m30.897651224s has elapsed since the last match call completed for this cluster condition backend; this cached cluster condition request has been queued for later execution Disconnected ARO clusters or clusters with a UDR 0.0.0.0/0 route definition that are blocking the ARO ACR and quay, are not be able to add or replace nodes after an upgrade https://access.redhat.com/solutions/7074686 Version: 4.13.49 Image: quay.io/openshift-release-dev/ocp-release@sha256:ab6f574665395d809511db9dc57764358278538eaae248c6d199208b3c30ab7d - Reason: EvaluationFailed - Message: Exposure to ARODNSWrongBootSequence is unknown due to an evaluation failure: client-side throttling: only 8m30.897666224s has elapsed since the last match call completed for this cluster condition backend; this cached cluster condition request has been queued for later execution + Reason: MultipleReasons + Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. + + Exposure to ARODNSWrongBootSequence is unknown due to an evaluation failure: client-side throttling: only 8m30.897666224s has elapsed since the last match call completed for this cluster condition backend; this cached cluster condition request has been queued for later execution Disconnected ARO clusters or clusters with a UDR 0.0.0.0/0 route definition that are blocking the ARO ACR and quay, are not be able to add or replace nodes after an upgrade https://access.redhat.com/solutions/7074686 And 45 older 4.13 updates you can see with '--show-outdated-releases' or '--version VERSION'. Updates to 4.12: VERSION ISSUES - 4.12.64 - 4.12.63 + 4.12.64 no known issues relevant to this cluster + 4.12.63 no known issues relevant to this cluster And 43 older 4.12 updates you can see with '--show-outdated-releases' or '--version VERSION'. diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.show-outdated-releases-output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.show-outdated-releases-output index ba11d7f882..ba323e9fb5 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.show-outdated-releases-output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.show-outdated-releases-output @@ -1,51 +1,46 @@ -Upgradeable=False - - Reason: AdminAckRequired - Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. - Upstream: https://api.integration.openshift.com/api/upgrades_info/graph Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) Updates to 4.13: VERSION ISSUES - 4.13.50 EvaluationFailed - 4.13.49 EvaluationFailed - 4.13.48 EvaluationFailed - 4.13.46 EvaluationFailed - 4.13.45 EvaluationFailed - 4.13.44 EvaluationFailed - 4.13.43 EvaluationFailed - 4.13.42 EvaluationFailed - 4.13.41 EvaluationFailed - 4.13.40 EvaluationFailed - 4.13.39 EvaluationFailed - 4.13.38 EvaluationFailed - 4.13.37 EvaluationFailed + 4.13.50 MultipleReasons + 4.13.49 MultipleReasons + 4.13.48 MultipleReasons + 4.13.46 MultipleReasons + 4.13.45 MultipleReasons + 4.13.44 MultipleReasons + 4.13.43 MultipleReasons + 4.13.42 MultipleReasons + 4.13.41 MultipleReasons + 4.13.40 MultipleReasons + 4.13.39 MultipleReasons + 4.13.38 MultipleReasons + 4.13.37 MultipleReasons 4.13.36 MultipleReasons 4.13.35 MultipleReasons 4.13.34 MultipleReasons 4.13.33 MultipleReasons - 4.13.32 EvaluationFailed - 4.13.31 EvaluationFailed - 4.13.30 EvaluationFailed - 4.13.29 EvaluationFailed - 4.13.28 EvaluationFailed - 4.13.27 EvaluationFailed - 4.13.26 EvaluationFailed - 4.13.25 EvaluationFailed - 4.13.24 EvaluationFailed - 4.13.23 EvaluationFailed - 4.13.22 EvaluationFailed - 4.13.21 EvaluationFailed - 4.13.19 EvaluationFailed - 4.13.18 EvaluationFailed - 4.13.17 EvaluationFailed - 4.13.15 EvaluationFailed - 4.13.14 EvaluationFailed - 4.13.13 EvaluationFailed - 4.13.12 EvaluationFailed - 4.13.11 EvaluationFailed - 4.13.10 EvaluationFailed + 4.13.32 MultipleReasons + 4.13.31 MultipleReasons + 4.13.30 MultipleReasons + 4.13.29 MultipleReasons + 4.13.28 MultipleReasons + 4.13.27 MultipleReasons + 4.13.26 MultipleReasons + 4.13.25 MultipleReasons + 4.13.24 MultipleReasons + 4.13.23 MultipleReasons + 4.13.22 MultipleReasons + 4.13.21 MultipleReasons + 4.13.19 MultipleReasons + 4.13.18 MultipleReasons + 4.13.17 MultipleReasons + 4.13.15 MultipleReasons + 4.13.14 MultipleReasons + 4.13.13 MultipleReasons + 4.13.12 MultipleReasons + 4.13.11 MultipleReasons + 4.13.10 MultipleReasons 4.13.9 MultipleReasons 4.13.8 MultipleReasons 4.13.6 MultipleReasons @@ -58,21 +53,21 @@ Updates to 4.13: Updates to 4.12: VERSION ISSUES - 4.12.64 - 4.12.63 - 4.12.61 - 4.12.60 - 4.12.59 - 4.12.58 - 4.12.57 + 4.12.64 no known issues relevant to this cluster + 4.12.63 no known issues relevant to this cluster + 4.12.61 no known issues relevant to this cluster + 4.12.60 no known issues relevant to this cluster + 4.12.59 no known issues relevant to this cluster + 4.12.58 no known issues relevant to this cluster + 4.12.57 no known issues relevant to this cluster 4.12.56 EvaluationFailed 4.12.55 EvaluationFailed 4.12.54 EvaluationFailed - 4.12.53 + 4.12.53 no known issues relevant to this cluster 4.12.51 MultipleReasons 4.12.50 RHELKernelHighLoadIOWait 4.12.49 RHELKernelHighLoadIOWait - 4.12.48 + 4.12.48 no known issues relevant to this cluster 4.12.47 EvaluationFailed 4.12.46 EvaluationFailed 4.12.45 EvaluationFailed @@ -80,20 +75,20 @@ Updates to 4.12: 4.12.43 EvaluationFailed 4.12.42 EvaluationFailed 4.12.41 EvaluationFailed - 4.12.40 - 4.12.39 - 4.12.37 - 4.12.36 - 4.12.35 - 4.12.34 - 4.12.33 - 4.12.32 - 4.12.31 - 4.12.30 - 4.12.29 - 4.12.28 - 4.12.27 - 4.12.26 + 4.12.40 no known issues relevant to this cluster + 4.12.39 no known issues relevant to this cluster + 4.12.37 no known issues relevant to this cluster + 4.12.36 no known issues relevant to this cluster + 4.12.35 no known issues relevant to this cluster + 4.12.34 no known issues relevant to this cluster + 4.12.33 no known issues relevant to this cluster + 4.12.32 no known issues relevant to this cluster + 4.12.31 no known issues relevant to this cluster + 4.12.30 no known issues relevant to this cluster + 4.12.29 no known issues relevant to this cluster + 4.12.28 no known issues relevant to this cluster + 4.12.27 no known issues relevant to this cluster + 4.12.26 no known issues relevant to this cluster 4.12.25 EvaluationFailed 4.12.24 EvaluationFailed 4.12.23 EvaluationFailed @@ -102,4 +97,4 @@ Updates to 4.12: 4.12.20 MultipleReasons 4.12.19 MultipleReasons 4.12.18 MultipleReasons - 4.12.17 + 4.12.17 no known issues relevant to this cluster diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.version-4.12.51-output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.version-4.12.51-output index d240fdd316..e5f68a112c 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.version-4.12.51-output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-not-recommended.version-4.12.51-output @@ -1,14 +1,9 @@ -Upgradeable=False - - Reason: AdminAckRequired - Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. - Upstream: https://api.integration.openshift.com/api/upgrades_info/graph Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) Update to 4.12.51 Recommended=False: Image: quay.io/openshift-release-dev/ocp-release@sha256:158ced797e49f6caf7862acccef58484be63b642fdd2f66e6416295fa7958ab0 -URL: https://access.redhat.com/errata/RHSA-2024:1052 +Release URL: https://access.redhat.com/errata/RHSA-2024:1052 Reason: MultipleReasons Message: An unintended reversion to the default kubelet nodeStatusReportFrequency can cause significant load on the control plane. https://issues.redhat.com/browse/MCO-1094 diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.output index bd4363d5d2..74f99cc399 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.output @@ -1,19 +1,22 @@ -Upgradeable=False +Upstream: https://api.integration.openshift.com/api/upgrades_info/graph +Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) +Updates to 4.13: + + Version: 4.13.50 + Image: quay.io/openshift-release-dev/ocp-release@sha256:6afb11e1cac46fd26476ca134072937115256b9c6360f7a1cd1812992c065f02 Reason: AdminAckRequired Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. -Upstream: https://api.integration.openshift.com/api/upgrades_info/graph -Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) + Version: 4.13.49 + Image: quay.io/openshift-release-dev/ocp-release@sha256:ab6f574665395d809511db9dc57764358278538eaae248c6d199208b3c30ab7d + Reason: AdminAckRequired + Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. -Updates to 4.13: - VERSION ISSUES - 4.13.50 - 4.13.49 And 45 older 4.13 updates you can see with '--show-outdated-releases' or '--version VERSION'. Updates to 4.12: VERSION ISSUES - 4.12.64 - 4.12.63 + 4.12.64 no known issues relevant to this cluster + 4.12.63 no known issues relevant to this cluster And 43 older 4.12 updates you can see with '--show-outdated-releases' or '--version VERSION'. diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.show-outdated-releases-output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.show-outdated-releases-output index c9360104f0..1ecfdd678a 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.show-outdated-releases-output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.show-outdated-releases-output @@ -1,56 +1,51 @@ -Upgradeable=False - - Reason: AdminAckRequired - Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. - Upstream: https://api.integration.openshift.com/api/upgrades_info/graph Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) Updates to 4.13: VERSION ISSUES - 4.13.50 - 4.13.49 - 4.13.48 - 4.13.46 AzureSystemDLooping - 4.13.45 - 4.13.44 - 4.13.43 - 4.13.42 - 4.13.41 - 4.13.40 - 4.13.39 - 4.13.38 - 4.13.37 - 4.13.36 HighNodeStatusReportFrequency - 4.13.35 HighNodeStatusReportFrequency - 4.13.34 HighNodeStatusReportFrequency - 4.13.33 HighNodeStatusReportFrequency - 4.13.32 - 4.13.31 - 4.13.30 - 4.13.29 - 4.13.28 - 4.13.27 - 4.13.26 - 4.13.25 - 4.13.24 - 4.13.23 - 4.13.22 - 4.13.21 - 4.13.19 - 4.13.18 - 4.13.17 - 4.13.15 - 4.13.14 - 4.13.13 - 4.13.12 - 4.13.11 - 4.13.10 - 4.13.9 SeccompFilterErrno524 - 4.13.8 SeccompFilterErrno524 - 4.13.6 SeccompFilterErrno524 - 4.13.5 SeccompFilterErrno524 - 4.13.4 SeccompFilterErrno524 + 4.13.50 AdminAckRequired + 4.13.49 AdminAckRequired + 4.13.48 AdminAckRequired + 4.13.46 MultipleReasons + 4.13.45 AdminAckRequired + 4.13.44 AdminAckRequired + 4.13.43 AdminAckRequired + 4.13.42 AdminAckRequired + 4.13.41 AdminAckRequired + 4.13.40 AdminAckRequired + 4.13.39 AdminAckRequired + 4.13.38 AdminAckRequired + 4.13.37 AdminAckRequired + 4.13.36 MultipleReasons + 4.13.35 MultipleReasons + 4.13.34 MultipleReasons + 4.13.33 MultipleReasons + 4.13.32 AdminAckRequired + 4.13.31 AdminAckRequired + 4.13.30 AdminAckRequired + 4.13.29 AdminAckRequired + 4.13.28 AdminAckRequired + 4.13.27 AdminAckRequired + 4.13.26 AdminAckRequired + 4.13.25 AdminAckRequired + 4.13.24 AdminAckRequired + 4.13.23 AdminAckRequired + 4.13.22 AdminAckRequired + 4.13.21 AdminAckRequired + 4.13.19 AdminAckRequired + 4.13.18 AdminAckRequired + 4.13.17 AdminAckRequired + 4.13.15 AdminAckRequired + 4.13.14 AdminAckRequired + 4.13.13 AdminAckRequired + 4.13.12 AdminAckRequired + 4.13.11 AdminAckRequired + 4.13.10 AdminAckRequired + 4.13.9 MultipleReasons + 4.13.8 MultipleReasons + 4.13.6 MultipleReasons + 4.13.5 MultipleReasons + 4.13.4 MultipleReasons 4.13.3 MultipleReasons 4.13.2 MultipleReasons 4.13.1 MultipleReasons @@ -58,48 +53,48 @@ Updates to 4.13: Updates to 4.12: VERSION ISSUES - 4.12.64 - 4.12.63 - 4.12.61 - 4.12.60 - 4.12.59 - 4.12.58 - 4.12.57 - 4.12.56 - 4.12.55 - 4.12.54 - 4.12.53 + 4.12.64 no known issues relevant to this cluster + 4.12.63 no known issues relevant to this cluster + 4.12.61 no known issues relevant to this cluster + 4.12.60 no known issues relevant to this cluster + 4.12.59 no known issues relevant to this cluster + 4.12.58 no known issues relevant to this cluster + 4.12.57 no known issues relevant to this cluster + 4.12.56 no known issues relevant to this cluster + 4.12.55 no known issues relevant to this cluster + 4.12.54 no known issues relevant to this cluster + 4.12.53 no known issues relevant to this cluster 4.12.51 MultipleReasons 4.12.50 RHELKernelHighLoadIOWait 4.12.49 RHELKernelHighLoadIOWait - 4.12.48 - 4.12.47 - 4.12.46 - 4.12.45 - 4.12.44 - 4.12.43 - 4.12.42 - 4.12.41 - 4.12.40 - 4.12.39 - 4.12.37 - 4.12.36 - 4.12.35 - 4.12.34 - 4.12.33 - 4.12.32 - 4.12.31 - 4.12.30 - 4.12.29 - 4.12.28 - 4.12.27 - 4.12.26 - 4.12.25 - 4.12.24 - 4.12.23 - 4.12.22 - 4.12.21 + 4.12.48 no known issues relevant to this cluster + 4.12.47 no known issues relevant to this cluster + 4.12.46 no known issues relevant to this cluster + 4.12.45 no known issues relevant to this cluster + 4.12.44 no known issues relevant to this cluster + 4.12.43 no known issues relevant to this cluster + 4.12.42 no known issues relevant to this cluster + 4.12.41 no known issues relevant to this cluster + 4.12.40 no known issues relevant to this cluster + 4.12.39 no known issues relevant to this cluster + 4.12.37 no known issues relevant to this cluster + 4.12.36 no known issues relevant to this cluster + 4.12.35 no known issues relevant to this cluster + 4.12.34 no known issues relevant to this cluster + 4.12.33 no known issues relevant to this cluster + 4.12.32 no known issues relevant to this cluster + 4.12.31 no known issues relevant to this cluster + 4.12.30 no known issues relevant to this cluster + 4.12.29 no known issues relevant to this cluster + 4.12.28 no known issues relevant to this cluster + 4.12.27 no known issues relevant to this cluster + 4.12.26 no known issues relevant to this cluster + 4.12.25 no known issues relevant to this cluster + 4.12.24 no known issues relevant to this cluster + 4.12.23 no known issues relevant to this cluster + 4.12.22 no known issues relevant to this cluster + 4.12.21 no known issues relevant to this cluster 4.12.20 DualStackNeedsController 4.12.19 DualStackNeedsController 4.12.18 DualStackNeedsController - 4.12.17 + 4.12.17 no known issues relevant to this cluster diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.version-4.12.51-output b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.version-4.12.51-output index d240fdd316..e5f68a112c 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.version-4.12.51-output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.12.16-longest-recommended.version-4.12.51-output @@ -1,14 +1,9 @@ -Upgradeable=False - - Reason: AdminAckRequired - Message: Kubernetes 1.26 and therefore OpenShift 4.13 remove several APIs which require admin consideration. Please see the knowledge article https://access.redhat.com/articles/6958394 for details and instructions. - Upstream: https://api.integration.openshift.com/api/upgrades_info/graph Channel: stable-4.13 (available channels: candidate-4.12, candidate-4.13, eus-4.12, eus-4.14, fast-4.12, fast-4.13, stable-4.12, stable-4.13) Update to 4.12.51 Recommended=False: Image: quay.io/openshift-release-dev/ocp-release@sha256:158ced797e49f6caf7862acccef58484be63b642fdd2f66e6416295fa7958ab0 -URL: https://access.redhat.com/errata/RHSA-2024:1052 +Release URL: https://access.redhat.com/errata/RHSA-2024:1052 Reason: MultipleReasons Message: An unintended reversion to the default kubelet nodeStatusReportFrequency can cause significant load on the control plane. https://issues.redhat.com/browse/MCO-1094 diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.output b/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.output index dedc5b2bd1..57ed3ab30e 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.output @@ -5,6 +5,6 @@ Channel: candidate-4.14 (available channels: candidate-4.14, candidate-4.15, eus Updates to 4.14: VERSION ISSUES - 4.14.11 - 4.14.10 + 4.14.11 no known issues relevant to this cluster + 4.14.10 no known issues relevant to this cluster And 8 older 4.14 updates you can see with '--show-outdated-releases' or '--version VERSION'. diff --git a/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.show-outdated-releases-output b/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.show-outdated-releases-output index e5bb4cb3f2..07224dbd90 100644 --- a/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.show-outdated-releases-output +++ b/pkg/cli/admin/upgrade/recommend/examples/4.14.1-all-recommended.show-outdated-releases-output @@ -5,13 +5,13 @@ Channel: candidate-4.14 (available channels: candidate-4.14, candidate-4.15, eus Updates to 4.14: VERSION ISSUES - 4.14.11 - 4.14.10 - 4.14.9 - 4.14.8 - 4.14.7 - 4.14.6 - 4.14.5 - 4.14.4 - 4.14.3 - 4.14.2 + 4.14.11 no known issues relevant to this cluster + 4.14.10 no known issues relevant to this cluster + 4.14.9 no known issues relevant to this cluster + 4.14.8 no known issues relevant to this cluster + 4.14.7 no known issues relevant to this cluster + 4.14.6 no known issues relevant to this cluster + 4.14.5 no known issues relevant to this cluster + 4.14.4 no known issues relevant to this cluster + 4.14.3 no known issues relevant to this cluster + 4.14.2 no known issues relevant to this cluster diff --git a/pkg/cli/admin/upgrade/recommend/recommend.go b/pkg/cli/admin/upgrade/recommend/recommend.go index 68f3f6080a..1489459ae3 100644 --- a/pkg/cli/admin/upgrade/recommend/recommend.go +++ b/pkg/cli/admin/upgrade/recommend/recommend.go @@ -146,10 +146,6 @@ func (o *options) Run(ctx context.Context) error { fmt.Fprintf(o.ErrOut, "warning: Cannot refresh available updates:\n Reason: %s\n Message: %s\n\n", c.Reason, strings.ReplaceAll(c.Message, "\n", "\n ")) } - if c := findClusterOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorUpgradeable); c != nil && c.Status == configv1.ConditionFalse { - fmt.Fprintf(o.Out, "%s=%s\n\n Reason: %s\n Message: %s\n\n", c.Type, c.Status, c.Reason, strings.ReplaceAll(c.Message, "\n", "\n ")) - } - if cv.Spec.Channel != "" { if cv.Spec.Upstream == "" { fmt.Fprint(o.Out, "Upstream is unset, so the cluster will use an appropriate default.\n") @@ -206,6 +202,12 @@ func (o *options) Run(ctx context.Context) error { }) } + if c := findClusterOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorUpgradeable); c != nil && c.Status == configv1.ConditionFalse { + if err := injectUpgradeableAsCondition(cv.Status.Desired.Version, c, majorMinorBuckets); err != nil { + fmt.Fprintf(o.ErrOut, "warning: Cannot inject %s=%s as a conditional update risk: %s\n\nReason: %s\n Message: %s\n\n", c.Type, c.Status, err, c.Reason, strings.ReplaceAll(c.Message, "\n", "\n ")) + } + } + if o.version != nil { if len(majorMinorBuckets) == 0 { return fmt.Errorf("no updates available, so cannot display context for the requested release %s", o.version) @@ -220,14 +222,14 @@ func (o *options) Run(ctx context.Context) error { if update.Release.Version == o.version.String() { fmt.Fprintln(o.Out) if c := notRecommendedCondition(update); c == nil { - fmt.Fprintf(o.Out, "Update to %s has no known issues relevant to this cluster.\nImage: %s\nURL: %s\n", update.Release.Version, update.Release.Image, update.Release.URL) + fmt.Fprintf(o.Out, "Update to %s has no known issues relevant to this cluster.\nImage: %s\nRelease URL: %s\n", update.Release.Version, update.Release.Image, update.Release.URL) } else { - fmt.Fprintf(o.Out, "Update to %s %s=%s:\nImage: %s\nURL: %s\nReason: %s\nMessage: %s\n", update.Release.Version, c.Type, c.Status, update.Release.Image, update.Release.URL, c.Reason, strings.ReplaceAll(c.Message, "\n", "\n ")) + fmt.Fprintf(o.Out, "Update to %s %s=%s:\nImage: %s\nRelease URL: %s\nReason: %s\nMessage: %s\n", update.Release.Version, c.Type, c.Status, update.Release.Image, update.Release.URL, c.Reason, strings.ReplaceAll(c.Message, "\n", "\n ")) } return nil } } - return fmt.Errorf("no updates to %d.%d available, so cannot display context for the requested release %s", o.version.Major, o.version.Minor, o.version) + return fmt.Errorf("no update to %s available, so cannot display context for the requested release", o.version) } } @@ -279,7 +281,7 @@ func (o *options) Run(ctx context.Context) error { break } if c == nil { - fmt.Fprintf(w, " %s\t\n", update.Release.Version) + fmt.Fprintf(w, " %s\t%s\n", update.Release.Version, "no known issues relevant to this cluster") if !o.showOutdatedReleases { headerQueued = false w.Flush() @@ -360,3 +362,77 @@ func findClusterOperatorStatusCondition(conditions []configv1.ClusterOperatorSta } return nil } + +func injectUpgradeableAsCondition(version string, condition *configv1.ClusterOperatorStatusCondition, majorMinorBuckets map[uint64]map[uint64][]configv1.ConditionalUpdate) error { + current, err := semver.Parse(version) + if err != nil { + return fmt.Errorf("cannot parse SemVer version %q: %v", version, err) + } + + upgradeableURI := fmt.Sprintf("https://docs.openshift.com/container-platform/%d.%d/updating/preparing_for_updates/updating-cluster-prepare.html#cluster-upgradeable_updating-cluster-prepare", current.Major, current.Minor) + if current.Minor <= 13 { + upgradeableURI = fmt.Sprintf("https://docs.openshift.com/container-platform/%d.%d/updating/index.html#understanding_clusteroperator_conditiontypes_updating-clusters-overview", current.Major, current.Minor) + } + + for major, minors := range majorMinorBuckets { + if major < current.Major { + continue + } + + for minor, targets := range minors { + if major == current.Major && minor <= current.Minor { + continue + } + + for i := 0; i < len(targets); i++ { + majorMinorBuckets[major][minor][i] = ensureUpgradeableRisk(majorMinorBuckets[major][minor][i], condition, upgradeableURI) + } + } + } + + return nil +} + +func ensureUpgradeableRisk(target configv1.ConditionalUpdate, condition *configv1.ClusterOperatorStatusCondition, upgradeableURI string) configv1.ConditionalUpdate { + if hasUpgradeableRisk(target, condition) { + return target + } + + target.Risks = append(target.Risks, configv1.ConditionalUpdateRisk{ + URL: upgradeableURI, + Name: "UpgradeableFalse", + Message: condition.Message, + MatchingRules: []configv1.ClusterCondition{{Type: "Always"}}, + }) + + for i, c := range target.Conditions { + if c.Type == "Recommended" { + if c.Status == metav1.ConditionTrue { + target.Conditions[i].Reason = condition.Reason + target.Conditions[i].Message = condition.Message + } else { + target.Conditions[i].Reason = "MultipleReasons" + target.Conditions[i].Message = fmt.Sprintf("%s\n\n%s", condition.Message, c.Message) + } + target.Conditions[i].Status = metav1.ConditionFalse + return target + } + } + + target.Conditions = append(target.Conditions, metav1.Condition{ + Type: "Recommended", + Status: metav1.ConditionFalse, + Reason: condition.Reason, + Message: condition.Message, + }) + return target +} + +func hasUpgradeableRisk(target configv1.ConditionalUpdate, condition *configv1.ClusterOperatorStatusCondition) bool { + for _, risk := range target.Risks { + if strings.Contains(risk.Message, condition.Message) { + return true + } + } + return false +} diff --git a/pkg/cli/admin/upgrade/status/controlplane.go b/pkg/cli/admin/upgrade/status/controlplane.go index 39692574f4..7c8f24084b 100644 --- a/pkg/cli/admin/upgrade/status/controlplane.go +++ b/pkg/cli/admin/upgrade/status/controlplane.go @@ -5,6 +5,7 @@ import ( "io" "math" "strings" + "text/tabwriter" "text/template" "time" @@ -37,10 +38,15 @@ type operators struct { Degraded int // Updated is the count of operators that updated its version, no matter its conditions Updated int - // Updating is the count of operators that have not updated its version and are with Progressing=True - Updating int // Waiting is the count of operators that have not updated its version and are with Progressing=False Waiting int + // Updating is the collection of cluster operators that are currently being updated + Updating []UpdatingClusterOperator +} + +type UpdatingClusterOperator struct { + Name string + Condition *v1.ClusterOperatorStatusCondition } func (o operators) StatusSummary() string { @@ -223,7 +229,8 @@ func assessControlPlaneStatus(cv *v1.ClusterVersion, operators []v1.ClusterOpera lastObservedProgress = progressing.LastTransitionTime.Time } if !updated && progressing.Status == v1.ConditionTrue { - displayData.Operators.Updating++ + displayData.Operators.Updating = append(displayData.Operators.Updating, + UpdatingClusterOperator{Name: operator.Name, Condition: progressing}) } if !updated && progressing.Status == v1.ConditionFalse { displayData.Operators.Waiting++ @@ -427,23 +434,55 @@ func vagueUnder(actual, estimated time.Duration) string { } } +func commaJoin(elems []UpdatingClusterOperator) string { + var names []string + for _, e := range elems { + names = append(names, e.Name) + } + return strings.Join(names, ", ") +} + var controlPlaneStatusTemplate = template.Must( template.New("controlPlaneStatus"). - Funcs(template.FuncMap{"shortDuration": shortDuration, "vagueUnder": vagueUnder}). + Funcs(template.FuncMap{"shortDuration": shortDuration, "vagueUnder": vagueUnder, "commaJoin": commaJoin}). Parse(controlPlaneStatusTemplateRaw)) -func (d *controlPlaneStatusDisplayData) Write(f io.Writer) error { +func (d *controlPlaneStatusDisplayData) Write(f io.Writer, detailed bool, now time.Time) error { if d.Operators.Updated == d.Operators.Total { _, err := f.Write([]byte(fmt.Sprintf("= Control Plane =\nUpdate to %s successfully completed at %s (duration: %s)\n", d.TargetVersion.target, d.CompletionAt.UTC().Format(time.RFC3339), shortDuration(d.Duration)))) return err } - return controlPlaneStatusTemplate.Execute(f, d) + if err := controlPlaneStatusTemplate.Execute(f, d); err != nil { + return err + } + if detailed && len(d.Operators.Updating) > 0 { + table := tabwriter.NewWriter(f, 0, 0, 3, ' ', 0) + f.Write([]byte("\nUpdating Cluster Operators")) + _, _ = table.Write([]byte("\nNAME\tSINCE\tREASON\tMESSAGE\n")) + for _, o := range d.Operators.Updating { + reason := o.Condition.Reason + if reason == "" { + reason = "-" + } + _, _ = table.Write([]byte(o.Name + "\t")) + _, _ = table.Write([]byte(shortDuration(now.Sub(o.Condition.LastTransitionTime.Time)) + "\t")) + _, _ = table.Write([]byte(reason + "\t")) + _, _ = table.Write([]byte(o.Condition.Message + "\n")) + } + if err := table.Flush(); err != nil { + return err + } + } + return nil } const controlPlaneStatusTemplateRaw = `= Control Plane = Assessment: {{ .Assessment }} Target Version: {{ .TargetVersion }} -Completion: {{ printf "%.0f" .Completion }}% ({{ .Operators.Updated }} operators updated, {{ .Operators.Updating }} updating, {{ .Operators.Waiting }} waiting) +{{ with commaJoin .Operators.Updating -}} +Updating: {{ . }} +{{ end -}} +Completion: {{ printf "%.0f" .Completion }}% ({{ .Operators.Updated }} operators updated, {{ len .Operators.Updating }} updating, {{ .Operators.Waiting }} waiting) Duration: {{ shortDuration .Duration }}{{ if .EstTimeToComplete }} (Est. Time Remaining: {{ vagueUnder .EstTimeToComplete .EstDuration }}){{ end }} Operator Health: {{ .Operators.StatusSummary }} ` diff --git a/pkg/cli/admin/upgrade/status/controlplane_test.go b/pkg/cli/admin/upgrade/status/controlplane_test.go index b8b492a8df..e5c3aad07d 100644 --- a/pkg/cli/admin/upgrade/status/controlplane_test.go +++ b/pkg/cli/admin/upgrade/status/controlplane_test.go @@ -70,7 +70,7 @@ func (c *coBuilder) progressing(status configv1.ConditionStatus, optionFuncs ... c.operator.Status.Conditions[i].Status = status c.operator.Status.Conditions[i].Reason = "ProgressingTowardsDesired" c.operator.Status.Conditions[i].Message = "Operand is operated by operator" - c.operator.Status.Conditions[i].LastTransitionTime = metav1.Now() + c.operator.Status.Conditions[i].LastTransitionTime = metav1.Date(2023, 12, 1, 23, 23, 0, 0, time.UTC) for _, f := range optionFuncs { f(&c.operator.Status.Conditions[i]) } @@ -175,7 +175,14 @@ func TestAssessControlPlaneStatus_Operators(t *testing.T) { co("one").operator, co("two").progressing(configv1.ConditionTrue).operator, }, - expected: operators{Total: 2, Updating: 1, Waiting: 1}, + expected: operators{Total: 2, Waiting: 1, Updating: []UpdatingClusterOperator{{Name: "two", Condition: &configv1.ClusterOperatorStatusCondition{ + Type: "Progressing", + Status: "True", + LastTransitionTime: metav1.Date(2023, 12, 1, 23, 23, 0, 0, time.UTC), + Reason: "ProgressingTowardsDesired", + Message: "Operand is operated by operator", + }, + }}}, }, { name: "one out of two not available", @@ -230,7 +237,14 @@ func TestAssessControlPlaneStatus_Operators(t *testing.T) { available(configv1.ConditionFalse). progressing(configv1.ConditionTrue).operator, }, - expected: operators{Total: 2, Unavailable: 1, Updating: 1, Waiting: 1}, + expected: operators{Total: 2, Unavailable: 1, Waiting: 1, Updating: []UpdatingClusterOperator{{Name: "two", Condition: &configv1.ClusterOperatorStatusCondition{ + Type: "Progressing", + Status: "True", + LastTransitionTime: metav1.Date(2023, 12, 1, 23, 23, 0, 0, time.UTC), + Reason: "ProgressingTowardsDesired", + Message: "Operand is operated by operator", + }, + }}}, }, { name: "one upgraded", diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output index 30a8c0dfe9..0ca82e3f6b 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.detailed-output @@ -1,10 +1,15 @@ = Control Plane = Assessment: Stalled Target Version: 4.14.1 (from 4.14.0-rc.3) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 1h59m (Est. Time Remaining: N/A; estimate duration was 1h24m) Operator Health: 28 Healthy, 1 Unavailable, 4 Available but degraded +Updating Cluster Operators +NAME SINCE REASON MESSAGE +machine-config 1h4m41s - Working towards 4.14.1 + Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? @@ -14,7 +19,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output index 9edaba54d2..36d201e500 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-degraded.output @@ -1,6 +1,7 @@ = Control Plane = Assessment: Stalled Target Version: 4.14.1 (from 4.14.0-rc.3) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 1h59m (Est. Time Remaining: N/A; estimate duration was 1h24m) Operator Health: 28 Healthy, 1 Unavailable, 4 Available but degraded @@ -14,7 +15,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output index dfeb80e182..408170be05 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.detailed-output @@ -14,7 +14,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Excluded 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded +worker Excluded 0% (0/3) 3 Available, 0 Progressing, 0 Draining, 3 Excluded Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output index aa4152a07d..6cedca8dae 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-paused-worker-pool.output @@ -14,7 +14,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Excluded 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 3 Excluded, 0 Degraded +worker Excluded 0% (0/3) 3 Available, 0 Progressing, 0 Draining, 3 Excluded Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output index 4b562068c9..79f82a14f6 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.detailed-output @@ -1,10 +1,15 @@ = Control Plane = Assessment: Progressing Target Version: 4.14.1 (from 4.14.0) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 14m (Est. Time Remaining: <10m) Operator Health: 32 Healthy, 1 Unavailable +Updating Cluster Operators +NAME SINCE REASON MESSAGE +machine-config 1m10s - Working towards 4.14.1 + Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-53-40.us-east-2.compute.internal Progressing Draining 4.14.0 +10m @@ -14,8 +19,8 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Progressing 0% 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded -infra Progressing 0% 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded +worker Progressing 0% (0/2) 1 Available, 1 Progressing, 1 Draining +infra Progressing 0% (0/2) 1 Available, 1 Progressing, 1 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.output b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.output index 4b562068c9..482830f149 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.output +++ b/pkg/cli/admin/upgrade/status/examples/4.14.1-workers-started-updating-multiple-pools.output @@ -1,6 +1,7 @@ = Control Plane = Assessment: Progressing Target Version: 4.14.1 (from 4.14.0) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 14m (Est. Time Remaining: <10m) Operator Health: 32 Healthy, 1 Unavailable @@ -14,8 +15,8 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Progressing 0% 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded -infra Progressing 0% 2 Total, 1 Available, 1 Progressing, 2 Outdated, 1 Draining, 0 Excluded, 0 Degraded +worker Progressing 0% (0/2) 1 Available, 1 Progressing, 1 Draining +infra Progressing 0% (0/2) 1 Available, 1 Progressing, 1 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output index 1b732daea0..b7d24334fa 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.detailed-output @@ -14,7 +14,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining zbeast Empty 0 Total Worker Pool Nodes: worker diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.output index 1b732daea0..b7d24334fa 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-b02-cos-not-annotated.output @@ -14,7 +14,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.15.0-ec.1 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining zbeast Empty 0 Total Worker Pool Nodes: worker diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output index c359d5a5df..434d6b8f57 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.detailed-output @@ -1,10 +1,16 @@ = Control Plane = Assessment: Progressing Target Version: 4.15.0-ec.2 (from incomplete 4.14.1) +Updating: etcd, kube-apiserver Completion: 3% (1 operators updated, 2 updating, 30 waiting) Duration: 1m29s (Est. Time Remaining: 1h25m) Operator Health: 33 Healthy +Updating Cluster Operators +NAME SINCE REASON MESSAGE +etcd 45s NodeInstaller NodeInstallerProgressing: 2 nodes are at revision 33; 1 nodes are at revision 34 +kube-apiserver 6m8s NodeInstaller NodeInstallerProgressing: 3 nodes are at revision 274; 0 nodes have achieved new revision 276 + Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.1 ? @@ -14,7 +20,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.1 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.output index 6cce3322ba..31e4fb5c0c 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-early.output @@ -1,6 +1,7 @@ = Control Plane = Assessment: Progressing Target Version: 4.15.0-ec.2 (from incomplete 4.14.1) +Updating: etcd, kube-apiserver Completion: 3% (1 operators updated, 2 updating, 30 waiting) Duration: 1m29s (Est. Time Remaining: 1h25m) Operator Health: 33 Healthy @@ -14,7 +15,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.1 ? = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output index e4f6f92f7a..53bbc2a893 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.detailed-output @@ -1,10 +1,15 @@ = Control Plane = Assessment: Progressing - Slow Target Version: 4.15.0-ec.2 (from 4.14.0-rc.3) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 59m (Est. Time Remaining: <10m) Operator Health: 30 Healthy, 2 Unavailable, 1 Available but degraded +Updating Cluster Operators +NAME SINCE REASON MESSAGE +machine-config 21m20s - Working towards 4.15.0-ec.2 + Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-30-217.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 ? @@ -14,7 +19,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output index 515993bb80..07e6d5e54b 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output +++ b/pkg/cli/admin/upgrade/status/examples/4.15.0-ec2-unavailable-mco-20m.output @@ -1,6 +1,7 @@ = Control Plane = Assessment: Progressing - Slow Target Version: 4.15.0-ec.2 (from 4.14.0-rc.3) +Updating: machine-config Completion: 97% (32 operators updated, 1 updating, 0 waiting) Duration: 59m (Est. Time Remaining: <10m) Operator Health: 30 Healthy, 2 Unavailable, 1 Available but degraded @@ -14,7 +15,7 @@ ip-10-0-92-180.us-east-2.compute.internal Outdated Pending 4.14.0-rc.3 = Worker Upgrade = WORKER POOL ASSESSMENT COMPLETION STATUS -worker Pending 0% 3 Total, 3 Available, 0 Progressing, 3 Outdated, 0 Draining, 0 Excluded, 0 Degraded +worker Pending 0% (0/3) 3 Available, 0 Progressing, 0 Draining Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output index 55dc33b536..2a75047b9c 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.detailed-output @@ -5,8 +5,8 @@ All control plane nodes successfully updated to 4.16.0-ec.3 = Worker Upgrade = -WORKER POOL ASSESSMENT COMPLETION STATUS -worker Degraded 37% 59 Total, 44 Available, 5 Progressing, 37 Outdated, 12 Draining, 0 Excluded, 7 Degraded +WORKER POOL ASSESSMENT COMPLETION STATUS +worker Degraded 37% (22/59) 44 Available, 5 Progressing, 12 Draining, 7 Degraded Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output index 472eb4336b..f45329b8b5 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-ec2-control-plane-updated-pdb-prohibits-draining.output @@ -5,8 +5,8 @@ All control plane nodes successfully updated to 4.16.0-ec.3 = Worker Upgrade = -WORKER POOL ASSESSMENT COMPLETION STATUS -worker Degraded 37% 59 Total, 44 Available, 5 Progressing, 37 Outdated, 12 Draining, 0 Excluded, 7 Degraded +WORKER POOL ASSESSMENT COMPLETION STATUS +worker Degraded 37% (22/59) 44 Available, 5 Progressing, 12 Draining, 7 Degraded Worker Pool Nodes: worker NAME ASSESSMENT PHASE VERSION EST MESSAGE diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.detailed-output b/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.detailed-output index a0319cc952..5f078534f5 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.detailed-output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.detailed-output @@ -1,10 +1,15 @@ = Control Plane = Assessment: Progressing Target Version: 4.17.0-ec.0 (from 4.16.0-0.nightly-2024-08-01-082745) +Updating: kube-apiserver Completion: 6% (2 operators updated, 1 updating, 30 waiting) Duration: 2m54s (Est. Time Remaining: 1h9m) Operator Health: 32 Healthy, 1 Available but degraded +Updating Cluster Operators +NAME SINCE REASON MESSAGE +kube-apiserver 2m NodeInstaller NodeInstallerProgressing: 1 node is at revision 8; 0 nodes have achieved new revision 10 + Control Plane Nodes NAME ASSESSMENT PHASE VERSION EST MESSAGE ip-10-0-8-37.ec2.internal Outdated Pending 4.16.0-0.nightly-2024-08-01-082745 ? diff --git a/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.output b/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.output index a0319cc952..ac0788ac56 100644 --- a/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.output +++ b/pkg/cli/admin/upgrade/status/examples/4.16.0-single-node.output @@ -1,6 +1,7 @@ = Control Plane = Assessment: Progressing Target Version: 4.17.0-ec.0 (from 4.16.0-0.nightly-2024-08-01-082745) +Updating: kube-apiserver Completion: 6% (2 operators updated, 1 updating, 30 waiting) Duration: 2m54s (Est. Time Remaining: 1h9m) Operator Health: 32 Healthy, 1 Available but degraded diff --git a/pkg/cli/admin/upgrade/status/health.go b/pkg/cli/admin/upgrade/status/health.go index d37c129b65..70dc715f3e 100644 --- a/pkg/cli/admin/upgrade/status/health.go +++ b/pkg/cli/admin/upgrade/status/health.go @@ -180,6 +180,13 @@ func shortDuration(d time.Duration) string { return orig[:len(orig)-2] case strings.HasSuffix(orig, "h0m"): return orig[:len(orig)-2] + case strings.Index(orig, ".") != -1: + dStr := orig[:strings.Index(orig, ".")] + "s" + newD, err := time.ParseDuration(dStr) + if err != nil { + return orig + } + return shortDuration(newD) } return orig } diff --git a/pkg/cli/admin/upgrade/status/health_test.go b/pkg/cli/admin/upgrade/status/health_test.go index 3f22764214..2b12321eb6 100644 --- a/pkg/cli/admin/upgrade/status/health_test.go +++ b/pkg/cli/admin/upgrade/status/health_test.go @@ -261,6 +261,14 @@ func TestShortDuration(t *testing.T) { duration: "0h0m0s", expected: "now", }, + { + duration: "45.000368975s", + expected: "45s", + }, + { + duration: "2m0.000368975s", + expected: "2m", + }, } for _, tc := range testCases { diff --git a/pkg/cli/admin/upgrade/status/status.go b/pkg/cli/admin/upgrade/status/status.go index 9edc2adea8..fecc422661 100644 --- a/pkg/cli/admin/upgrade/status/status.go +++ b/pkg/cli/admin/upgrade/status/status.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "os" - // "sort" "strings" "time" @@ -37,13 +36,14 @@ func newOptions(streams genericiooptions.IOStreams) *options { } const ( - detailedOutputNone = "none" - detailedOutputAll = "all" - detailedOutputNodes = "nodes" - detailedOutputHealth = "health" + detailedOutputNone = "none" + detailedOutputAll = "all" + detailedOutputNodes = "nodes" + detailedOutputHealth = "health" + detailedOutputOperators = "operators" ) -var detailedOutputAllValues = []string{detailedOutputNone, detailedOutputAll, detailedOutputNodes, detailedOutputHealth} +var detailedOutputAllValues = []string{detailedOutputNone, detailedOutputAll, detailedOutputNodes, detailedOutputHealth, detailedOutputOperators} func New(f kcmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { o := newOptions(streams) @@ -308,7 +308,7 @@ func (o *options) Run(ctx context.Context) error { controlPlaneStatusData, insights := assessControlPlaneStatus(cv, operators.Items, now) updateInsights = append(updateInsights, insights...) - _ = controlPlaneStatusData.Write(o.Out) + _ = controlPlaneStatusData.Write(o.Out, o.enabledDetailed(detailedOutputOperators), now) controlPlanePoolStatusData.WriteNodes(o.Out, o.enabledDetailed(detailedOutputNodes)) var workerUpgrade bool diff --git a/pkg/cli/admin/upgrade/status/workerpool.go b/pkg/cli/admin/upgrade/status/workerpool.go index 0093936f3b..acffc7dd59 100644 --- a/pkg/cli/admin/upgrade/status/workerpool.go +++ b/pkg/cli/admin/upgrade/status/workerpool.go @@ -483,10 +483,15 @@ func writePools(w io.Writer, workerPoolsStatusData []poolDisplayData) { _, _ = tabw.Write([]byte(fmt.Sprintf("%d Total", pool.NodesOverview.Total) + "\n")) } else { _, _ = tabw.Write([]byte(pool.Assessment + "\t")) - _, _ = tabw.Write([]byte(fmt.Sprintf("%.0f%%", pool.Completion) + "\t")) - _, _ = tabw.Write([]byte(fmt.Sprintf("%d Total, %d Available, %d Progressing, %d Outdated, %d Draining, %d Excluded, %d Degraded", - pool.NodesOverview.Total, pool.NodesOverview.Available, pool.NodesOverview.Progressing, pool.NodesOverview.Outdated, - pool.NodesOverview.Draining, pool.NodesOverview.Excluded, pool.NodesOverview.Degraded) + "\n")) + _, _ = tabw.Write([]byte(fmt.Sprintf("%.0f%% (%d/%d)", pool.Completion, pool.NodesOverview.Total-pool.NodesOverview.Outdated, pool.NodesOverview.Total) + "\t")) + status := fmt.Sprintf("%d Available, %d Progressing, %d Draining", pool.NodesOverview.Available, pool.NodesOverview.Progressing, pool.NodesOverview.Draining) + for k, v := range map[string]int{"Excluded": pool.NodesOverview.Excluded, "Degraded": pool.NodesOverview.Degraded} { + if v > 0 { + status = fmt.Sprintf("%s, %d %s", status, v, k) + } + } + status = fmt.Sprintf("%s\n", status) + _, _ = tabw.Write([]byte(status)) } } tabw.Flush() diff --git a/pkg/cli/admin/upgrade/upgrade.go b/pkg/cli/admin/upgrade/upgrade.go index 4403694b64..1f61621e2f 100644 --- a/pkg/cli/admin/upgrade/upgrade.go +++ b/pkg/cli/admin/upgrade/upgrade.go @@ -153,8 +153,8 @@ func (o *Options) Complete(f kcmdutil.Factory, cmd *cobra.Command, args []string if o.Clear && (len(o.ToImage) > 0 || len(o.To) > 0 || o.ToLatestAvailable || o.ToMultiArch) { return fmt.Errorf("--clear may not be specified with any other flags") } - if o.ToMultiArch && (len(o.To) > 0 || len(o.ToImage) > 0) { - return fmt.Errorf("--to-multi-arch may not be used with --to or --to-image") + if o.ToMultiArch && (len(o.To) > 0 || len(o.ToImage) > 0 || o.ToLatestAvailable) { + return fmt.Errorf("--to-multi-arch may not be used with --to, --to-image, or --to-latest") } if len(o.To) > 0 && len(o.ToImage) > 0 { return fmt.Errorf("only one of --to or --to-image may be provided") @@ -240,8 +240,17 @@ func (o *Options) Run() error { fmt.Fprintf(o.ErrOut, "warning: --allow-upgrade-with-warnings is bypassing: %s\n", err) } - if err := patchDesiredUpdate(ctx, &configv1.Update{Architecture: configv1.ClusterVersionArchitectureMulti, - Version: cv.Status.Desired.Version}, o.Client, cv.Name); err != nil { + update := &configv1.Update{ + Architecture: configv1.ClusterVersionArchitectureMulti, + Version: cv.Status.Desired.Version, + } + + if o.Force { + update.Force = true + fmt.Fprintln(o.ErrOut, "warning: --force overrides cluster verification of your supplied release image and waives any update precondition failures.") + } + + if err := patchDesiredUpdate(ctx, update, o.Client, cv.Name); err != nil { return err } diff --git a/pkg/helpers/newapp/newapptest/newapp_test.go b/pkg/helpers/newapp/newapptest/newapp_test.go index 051d279ddc..2760127fe8 100644 --- a/pkg/helpers/newapp/newapptest/newapp_test.go +++ b/pkg/helpers/newapp/newapptest/newapp_test.go @@ -1015,14 +1015,14 @@ func TestNewAppRunBuilds(t *testing.T) { name: "successful build from dockerfile", config: &cmd.AppConfig{ GenerationInputs: cmd.GenerationInputs{ - Dockerfile: "FROM openshift/origin:v1.0.6\nUSER foo", + Dockerfile: "FROM centos:centos8\nUSER foo", }, }, expected: map[string][]string{ - "buildConfig": {"origin"}, + "buildConfig": {"centos"}, // There's a single image stream, but different tags: input from // openshift/origin:v1.0.6, output to openshift/origin:latest. - "imageStream": {"origin"}, + "imageStream": {"centos"}, }, }, {