From 72258e42bed7eab6284798b234cf4e7fd7ec362d Mon Sep 17 00:00:00 2001 From: mahao Date: Tue, 18 Jan 2022 16:24:25 +0800 Subject: [PATCH] unify the file format under cmd, alluxio jindo dataset Signed-off-by: mahao --- cmd/alluxio/app/alluxio.go | 132 ++++++++++++++++++++++++++++++++++++ cmd/alluxio/app/init.go | 28 ++++++++ cmd/alluxio/app/version.go | 37 ++++++++++ cmd/alluxio/main.go | 135 +------------------------------------ cmd/dataset/app/dataset.go | 133 ++++++++++++++++++++++++++++++++++++ cmd/dataset/app/init.go | 31 +++++++++ cmd/dataset/app/version.go | 38 +++++++++++ cmd/dataset/main.go | 134 ++---------------------------------- cmd/jindo/app/init.go | 30 +++++++++ cmd/jindo/app/jindo.go | 134 ++++++++++++++++++++++++++++++++++++ cmd/jindo/app/version.go | 38 +++++++++++ cmd/jindo/main.go | 135 ++----------------------------------- 12 files changed, 613 insertions(+), 392 deletions(-) create mode 100644 cmd/alluxio/app/alluxio.go create mode 100644 cmd/alluxio/app/init.go create mode 100644 cmd/alluxio/app/version.go create mode 100644 cmd/dataset/app/dataset.go create mode 100644 cmd/dataset/app/init.go create mode 100644 cmd/dataset/app/version.go create mode 100644 cmd/jindo/app/init.go create mode 100644 cmd/jindo/app/jindo.go create mode 100644 cmd/jindo/app/version.go diff --git a/cmd/alluxio/app/alluxio.go b/cmd/alluxio/app/alluxio.go new file mode 100644 index 00000000000..1acf5364e1e --- /dev/null +++ b/cmd/alluxio/app/alluxio.go @@ -0,0 +1,132 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "os" + // +kubebuilder:scaffold:imports + + "github.com/fluid-cloudnative/fluid" + datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" + alluxioctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/alluxio" + "github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio" + "github.com/fluid-cloudnative/fluid/pkg/ddc/base" + "github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator" + "github.com/fluid-cloudnative/fluid/pkg/utils" + "github.com/spf13/cobra" + zapOpt "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/net" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + // Use compiler to check if the struct implements all the interface + _ base.Implement = (*alluxio.AlluxioEngine)(nil) + + metricsAddr string + enableLeaderElection bool + development bool + portRange string + maxConcurrentReconciles int + pprofAddr string +) + +var alluxioCmd = &cobra.Command{ + Use: "start", + Short: "start alluxioruntime-controller in Kubernetes", + Run: func(cmd *cobra.Command, args []string) { + handle() + }, +} + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + _ = datav1alpha1.AddToScheme(scheme) + + alluxioCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") + alluxioCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + alluxioCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") + alluxioCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "20000-25000", "Set available port range for Alluxio") + alluxioCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for AlluxioRuntime controller") + alluxioCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") +} + +func handle() { + fluid.LogVersion() + + ctrl.SetLogger(zap.New(func(o *zap.Options) { + o.Development = development + }, func(o *zap.Options) { + o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) + }, func(o *zap.Options) { + if !development { + encCfg := zapOpt.NewProductionEncoderConfig() + encCfg.EncodeLevel = zapcore.CapitalLevelEncoder + encCfg.EncodeTime = zapcore.ISO8601TimeEncoder + o.Encoder = zapcore.NewConsoleEncoder(encCfg) + } + })) + + utils.NewPprofServer(setupLog, pprofAddr) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "7857424864.data.fluid.io", + Port: 9443, + }) + if err != nil { + setupLog.Error(err, "unable to start alluxioruntime manager") + os.Exit(1) + } + + controllerOptions := controller.Options{ + MaxConcurrentReconciles: maxConcurrentReconciles, + } + + if err = (alluxioctl.NewRuntimeReconciler(mgr.GetClient(), + ctrl.Log.WithName("alluxioctl").WithName("AlluxioRuntime"), + mgr.GetScheme(), + mgr.GetEventRecorderFor("AlluxioRuntime"), + )).SetupWithManager(mgr, controllerOptions); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AlluxioRuntime") + os.Exit(1) + } + + pr, err := net.ParsePortRange(portRange) + if err != nil { + setupLog.Error(err, "can't parse port range. Port range must be like -") + os.Exit(1) + } + setupLog.Info("port range parsed", "port range", pr.String()) + + portallocator.SetupRuntimePortAllocator(mgr.GetClient(), pr, alluxio.GetReservedPorts) + + setupLog.Info("starting alluxioruntime-controller") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem alluxioruntime-controller") + os.Exit(1) + } +} diff --git a/cmd/alluxio/app/init.go b/cmd/alluxio/app/init.go new file mode 100644 index 00000000000..0e7abcc178f --- /dev/null +++ b/cmd/alluxio/app/init.go @@ -0,0 +1,28 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import "github.com/spf13/cobra" + +func NewAlluxioFSCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "alluxioruntime-controller", + Short: "Controller for alluxioruntime", + } + cmd.AddCommand(versionCmd, alluxioCmd) + + return cmd +} diff --git a/cmd/alluxio/app/version.go b/cmd/alluxio/app/version.go new file mode 100644 index 00000000000..b5823658257 --- /dev/null +++ b/cmd/alluxio/app/version.go @@ -0,0 +1,37 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package app + +import ( + "github.com/fluid-cloudnative/fluid" + "github.com/spf13/cobra" +) + +var ( + short bool +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "print version information", + Run: func(cmd *cobra.Command, args []string) { + fluid.PrintVersion(short) + }, +} + +func init() { + versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") +} diff --git a/cmd/alluxio/main.go b/cmd/alluxio/main.go index 2c95c006476..fe6f1399106 100644 --- a/cmd/alluxio/main.go +++ b/cmd/alluxio/main.go @@ -18,144 +18,15 @@ package main import ( "fmt" - "github.com/fluid-cloudnative/fluid" - alluxioctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/alluxio" - "github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator" - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/net" "os" - "sigs.k8s.io/controller-runtime/pkg/controller" - "go.uber.org/zap/zapcore" - - zapOpt "go.uber.org/zap" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - // +kubebuilder:scaffold:imports - - datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" - "github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio" - "github.com/fluid-cloudnative/fluid/pkg/ddc/base" - "github.com/fluid-cloudnative/fluid/pkg/utils" + "github.com/fluid-cloudnative/fluid/cmd/alluxio/app" ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - // Use compiler to check if the struct implements all the interface - _ base.Implement = (*alluxio.AlluxioEngine)(nil) - - short bool - metricsAddr string - enableLeaderElection bool - development bool - portRange string - maxConcurrentReconciles int - pprofAddr string -) - -var cmd = &cobra.Command{ - Use: "alluxioruntime-controller", - Short: "Controller for alluxioruntime", -} - -var startCmd = &cobra.Command{ - Use: "start", - Short: "start alluxioruntime-controller in Kubernetes", - Run: func(cmd *cobra.Command, args []string) { - handle() - }, -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "print version information", - Run: func(cmd *cobra.Command, args []string) { - fluid.PrintVersion(short) - }, -} - -func init() { - _ = clientgoscheme.AddToScheme(scheme) - _ = datav1alpha1.AddToScheme(scheme) - - startCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") - startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") - startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "20000-25000", "Set available port range for Alluxio") - startCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for AlluxioRuntime controller") - versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") - startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") - cmd.AddCommand(startCmd) - cmd.AddCommand(versionCmd) -} - func main() { - if err := cmd.Execute(); err != nil { + command := app.NewAlluxioFSCommand() + if err := command.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s", err.Error()) os.Exit(1) } } - -func handle() { - fluid.LogVersion() - - ctrl.SetLogger(zap.New(func(o *zap.Options) { - o.Development = development - }, func(o *zap.Options) { - o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) - }, func(o *zap.Options) { - if !development { - encCfg := zapOpt.NewProductionEncoderConfig() - encCfg.EncodeLevel = zapcore.CapitalLevelEncoder - encCfg.EncodeTime = zapcore.ISO8601TimeEncoder - o.Encoder = zapcore.NewConsoleEncoder(encCfg) - } - })) - - utils.NewPprofServer(setupLog, pprofAddr) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "7857424864.data.fluid.io", - Port: 9443, - }) - if err != nil { - setupLog.Error(err, "unable to start alluxioruntime manager") - os.Exit(1) - } - - controllerOptions := controller.Options{ - MaxConcurrentReconciles: maxConcurrentReconciles, - } - - if err = (alluxioctl.NewRuntimeReconciler(mgr.GetClient(), - ctrl.Log.WithName("alluxioctl").WithName("AlluxioRuntime"), - mgr.GetScheme(), - mgr.GetEventRecorderFor("AlluxioRuntime"), - )).SetupWithManager(mgr, controllerOptions); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AlluxioRuntime") - os.Exit(1) - } - - pr, err := net.ParsePortRange(portRange) - if err != nil { - setupLog.Error(err, "can't parse port range. Port range must be like -") - os.Exit(1) - } - setupLog.Info("port range parsed", "port range", pr.String()) - - portallocator.SetupRuntimePortAllocator(mgr.GetClient(), pr, alluxio.GetReservedPorts) - - setupLog.Info("starting alluxioruntime-controller") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem alluxioruntime-controller") - os.Exit(1) - } -} diff --git a/cmd/dataset/app/dataset.go b/cmd/dataset/app/dataset.go new file mode 100644 index 00000000000..4b93dcbe584 --- /dev/null +++ b/cmd/dataset/app/dataset.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "os" + + "github.com/fluid-cloudnative/fluid" + datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" + databackupctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/databackup" + dataloadctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/dataload" + datasetctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/dataset" + "github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio" + "github.com/fluid-cloudnative/fluid/pkg/ddc/base" + "github.com/fluid-cloudnative/fluid/pkg/utils" + "github.com/spf13/cobra" + zapOpt "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + // Use compiler to check if the struct implements all the interface + _ base.Implement = (*alluxio.AlluxioEngine)(nil) + + metricsAddr string + enableLeaderElection bool + development bool + pprofAddr string +) + +var datasetCmd = &cobra.Command{ + Use: "start", + Short: "start dataset-controller in Kubernetes", + Run: func(cmd *cobra.Command, args []string) { + handle() + }, +} + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + _ = datav1alpha1.AddToScheme(scheme) + + datasetCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") + datasetCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + datasetCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") + datasetCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") +} + +func handle() { + fluid.LogVersion() + + ctrl.SetLogger(zap.New(func(o *zap.Options) { + o.Development = development + }, func(o *zap.Options) { + o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) + }, func(o *zap.Options) { + if !development { + encCfg := zapOpt.NewProductionEncoderConfig() + encCfg.EncodeLevel = zapcore.CapitalLevelEncoder + encCfg.EncodeTime = zapcore.ISO8601TimeEncoder + o.Encoder = zapcore.NewConsoleEncoder(encCfg) + } + })) + + utils.NewPprofServer(setupLog, pprofAddr) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "89759796.data.fluid.io", + Port: 9443, + }) + if err != nil { + setupLog.Error(err, "unable to start dataset manager") + os.Exit(1) + } + + if err = (&datasetctl.DatasetReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("datasetctl").WithName("Dataset"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("Dataset"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Dataset") + os.Exit(1) + } + + if err = (dataloadctl.NewDataLoadReconciler(mgr.GetClient(), + ctrl.Log.WithName("dataloadctl").WithName("DataLoad"), + mgr.GetScheme(), + mgr.GetEventRecorderFor("DataLoad"), + )).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DataLoad") + os.Exit(1) + } + + if err = (databackupctl.NewDataBackupReconciler(mgr.GetClient(), + ctrl.Log.WithName("databackupctl").WithName("DataBackup"), + mgr.GetScheme(), + mgr.GetEventRecorderFor("DataBackup"), + )).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DataBackup") + os.Exit(1) + } + + setupLog.Info("starting dataset-controller") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running dataset-controller") + os.Exit(1) + } +} diff --git a/cmd/dataset/app/init.go b/cmd/dataset/app/init.go new file mode 100644 index 00000000000..4b19926f98b --- /dev/null +++ b/cmd/dataset/app/init.go @@ -0,0 +1,31 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import "github.com/spf13/cobra" + +func NewDatasetCommand() *cobra.Command { + command := &cobra.Command{ + Use: "dataset-controller", + Short: "controller for dataset", + } + + command.AddCommand(versionCmd, datasetCmd) + + return command + +} diff --git a/cmd/dataset/app/version.go b/cmd/dataset/app/version.go new file mode 100644 index 00000000000..c1003351628 --- /dev/null +++ b/cmd/dataset/app/version.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/fluid-cloudnative/fluid" + "github.com/spf13/cobra" +) + +var ( + short bool +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "print version information", + Run: func(cmd *cobra.Command, args []string) { + fluid.PrintVersion(short) + }, +} + +func init() { + versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") +} diff --git a/cmd/dataset/main.go b/cmd/dataset/main.go index 6e3f8dfd950..2f8a6d79907 100644 --- a/cmd/dataset/main.go +++ b/cmd/dataset/main.go @@ -1,4 +1,5 @@ /* +Copyright 2021 The Fluid Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,142 +18,15 @@ package main import ( "fmt" - "github.com/fluid-cloudnative/fluid" - "github.com/fluid-cloudnative/fluid/pkg/utils" - "github.com/spf13/cobra" - zapOpt "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "os" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" - databackupctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/databackup" - dataloadctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/dataload" - datasetctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/dataset" - "github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio" - "github.com/fluid-cloudnative/fluid/pkg/ddc/base" + "github.com/fluid-cloudnative/fluid/cmd/dataset/app" ) -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - // Use compiler to check if the struct implements all the interface - _ base.Implement = (*alluxio.AlluxioEngine)(nil) - - short bool - metricsAddr string - enableLeaderElection bool - development bool - pprofAddr string -) - -var cmd = &cobra.Command{ - Use: "dataset-controller", - Short: "controller for dataset", -} - -var startCmd = &cobra.Command{ - Use: "start", - Short: "start dataset-controller in Kubernetes", - Run: func(cmd *cobra.Command, args []string) { - handle() - }, -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "print version information", - Run: func(cmd *cobra.Command, args []string) { - fluid.PrintVersion(short) - }, -} - -func init() { - _ = clientgoscheme.AddToScheme(scheme) - _ = datav1alpha1.AddToScheme(scheme) - - startCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") - startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") - versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") - startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") - cmd.AddCommand(startCmd) - cmd.AddCommand(versionCmd) - -} - func main() { - if err := cmd.Execute(); err != nil { + command := app.NewDatasetCommand() + if err := command.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s", err.Error()) os.Exit(1) } } - -func handle() { - fluid.LogVersion() - - ctrl.SetLogger(zap.New(func(o *zap.Options) { - o.Development = development - }, func(o *zap.Options) { - o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) - }, func(o *zap.Options) { - if !development { - encCfg := zapOpt.NewProductionEncoderConfig() - encCfg.EncodeLevel = zapcore.CapitalLevelEncoder - encCfg.EncodeTime = zapcore.ISO8601TimeEncoder - o.Encoder = zapcore.NewConsoleEncoder(encCfg) - } - })) - - utils.NewPprofServer(setupLog, pprofAddr) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "89759796.data.fluid.io", - Port: 9443, - }) - if err != nil { - setupLog.Error(err, "unable to start dataset manager") - os.Exit(1) - } - - if err = (&datasetctl.DatasetReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("datasetctl").WithName("Dataset"), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("Dataset"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Dataset") - os.Exit(1) - } - - if err = (dataloadctl.NewDataLoadReconciler(mgr.GetClient(), - ctrl.Log.WithName("dataloadctl").WithName("DataLoad"), - mgr.GetScheme(), - mgr.GetEventRecorderFor("DataLoad"), - )).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "DataLoad") - os.Exit(1) - } - - if err = (databackupctl.NewDataBackupReconciler(mgr.GetClient(), - ctrl.Log.WithName("databackupctl").WithName("DataBackup"), - mgr.GetScheme(), - mgr.GetEventRecorderFor("DataBackup"), - )).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "DataBackup") - os.Exit(1) - } - - setupLog.Info("starting dataset-controller") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running dataset-controller") - os.Exit(1) - } -} diff --git a/cmd/jindo/app/init.go b/cmd/jindo/app/init.go new file mode 100644 index 00000000000..c6de5b47a70 --- /dev/null +++ b/cmd/jindo/app/init.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import "github.com/spf13/cobra" + +func NewJindoFSCommand() *cobra.Command { + command := &cobra.Command{ + Use: "jindoruntime-controller", + Short: "Controller for jindoruntime", + } + + command.AddCommand(versionCmd, jindoCmd) + + return command +} diff --git a/cmd/jindo/app/jindo.go b/cmd/jindo/app/jindo.go new file mode 100644 index 00000000000..d38ac767354 --- /dev/null +++ b/cmd/jindo/app/jindo.go @@ -0,0 +1,134 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "os" + + "github.com/fluid-cloudnative/fluid" + datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" + jindoctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/jindo" + "github.com/fluid-cloudnative/fluid/pkg/ddc/base" + "github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator" + "github.com/fluid-cloudnative/fluid/pkg/ddc/jindo" + "github.com/fluid-cloudnative/fluid/pkg/utils" + "github.com/spf13/cobra" + zapOpt "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/net" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + // Use compiler to check if the struct implements all the interface + _ base.Implement = (*jindo.JindoEngine)(nil) + + metricsAddr string + enableLeaderElection bool + development bool + // The new mode + eventDriven bool + portRange string + maxConcurrentReconciles int + pprofAddr string +) + +var jindoCmd = &cobra.Command{ + Use: "start", + Short: "start jindoruntime-controller in Kubernetes", + Run: func(cmd *cobra.Command, args []string) { + handle() + }, +} + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + _ = datav1alpha1.AddToScheme(scheme) + + jindoCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") + jindoCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + jindoCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") + jindoCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "18000-19999", "Set available port range for Jindo") + jindoCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for JindoRuntime controller") + jindoCmd.Flags().BoolVar(&eventDriven, "event-driven", true, "The reconciler's loop strategy. if it's false, it indicates period driven.") + jindoCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") +} + +func handle() { + fluid.LogVersion() + + ctrl.SetLogger(zap.New(func(o *zap.Options) { + o.Development = development + }, func(o *zap.Options) { + o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) + }, func(o *zap.Options) { + if !development { + encCfg := zapOpt.NewProductionEncoderConfig() + encCfg.EncodeLevel = zapcore.CapitalLevelEncoder + encCfg.EncodeTime = zapcore.ISO8601TimeEncoder + o.Encoder = zapcore.NewConsoleEncoder(encCfg) + } + })) + + utils.NewPprofServer(setupLog, pprofAddr) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "5688274864.data.fluid.io", + Port: 9443, + }) + if err != nil { + setupLog.Error(err, "unable to start jindoruntime manager") + os.Exit(1) + } + + controllerOptions := controller.Options{ + MaxConcurrentReconciles: maxConcurrentReconciles, + } + + if err = (jindoctl.NewRuntimeReconciler(mgr.GetClient(), + ctrl.Log.WithName("jindoctl").WithName("JindoRuntime"), + mgr.GetScheme(), + mgr.GetEventRecorderFor("JindoRuntime"), + )).SetupWithManager(mgr, controllerOptions, eventDriven); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "JindoRuntime") + os.Exit(1) + } + + pr, err := net.ParsePortRange(portRange) + if err != nil { + setupLog.Error(err, "can't parse port range. Port range must be like -max") + os.Exit(1) + } + setupLog.Info("port range parsed", "port range", pr.String()) + + portallocator.SetupRuntimePortAllocator(mgr.GetClient(), pr, jindo.GetReservedPorts) + + setupLog.Info("starting jindoruntime-controller") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem jindoruntime-controller") + os.Exit(1) + } +} diff --git a/cmd/jindo/app/version.go b/cmd/jindo/app/version.go new file mode 100644 index 00000000000..c1003351628 --- /dev/null +++ b/cmd/jindo/app/version.go @@ -0,0 +1,38 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/fluid-cloudnative/fluid" + "github.com/spf13/cobra" +) + +var ( + short bool +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "print version information", + Run: func(cmd *cobra.Command, args []string) { + fluid.PrintVersion(short) + }, +} + +func init() { + versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") +} diff --git a/cmd/jindo/main.go b/cmd/jindo/main.go index 932df4005b4..f8f22b62d44 100644 --- a/cmd/jindo/main.go +++ b/cmd/jindo/main.go @@ -1,4 +1,5 @@ /* +Copyright 2021 The Fluid Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,142 +18,16 @@ package main import ( "fmt" - "github.com/fluid-cloudnative/fluid" - datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" - jindoctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/jindo" - "github.com/fluid-cloudnative/fluid/pkg/ddc/base" - "github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator" - "github.com/fluid-cloudnative/fluid/pkg/ddc/jindo" - "github.com/fluid-cloudnative/fluid/pkg/utils" - "github.com/spf13/cobra" - zapOpt "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/net" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" "os" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/log/zap" -) - -var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - // Use compiler to check if the struct implements all the interface - _ base.Implement = (*jindo.JindoEngine)(nil) - short bool - metricsAddr string - enableLeaderElection bool - development bool - // The new mode - eventDriven bool - portRange string - maxConcurrentReconciles int - pprofAddr string + "github.com/fluid-cloudnative/fluid/cmd/jindo/app" ) -var cmd = &cobra.Command{ - Use: "jindoruntime-controller", - Short: "Controller for jindoruntime", -} - -var startCmd = &cobra.Command{ - Use: "start", - Short: "start jindoruntime-controller in Kubernetes", - Run: func(cmd *cobra.Command, args []string) { - handle() - }, -} - -var versionCmd = &cobra.Command{ - Use: "version", - Short: "print version information", - Run: func(cmd *cobra.Command, args []string) { - fluid.PrintVersion(short) - }, -} - -func init() { - _ = clientgoscheme.AddToScheme(scheme) - _ = datav1alpha1.AddToScheme(scheme) - - startCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.") - startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.") - startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "18000-19999", "Set available port range for Jindo") - startCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for JindoRuntime controller") - startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results") - startCmd.Flags().BoolVar(&eventDriven, "event-driven", true, "The reconciler's loop strategy. if it's false, it indicates period driven.") - versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info") - - cmd.AddCommand(startCmd) - cmd.AddCommand(versionCmd) -} - func main() { - if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "%s", err.Error()) - os.Exit(1) - } -} -func handle() { - fluid.LogVersion() - - ctrl.SetLogger(zap.New(func(o *zap.Options) { - o.Development = development - }, func(o *zap.Options) { - o.ZapOpts = append(o.ZapOpts, zapOpt.AddCaller()) - }, func(o *zap.Options) { - if !development { - encCfg := zapOpt.NewProductionEncoderConfig() - encCfg.EncodeLevel = zapcore.CapitalLevelEncoder - encCfg.EncodeTime = zapcore.ISO8601TimeEncoder - o.Encoder = zapcore.NewConsoleEncoder(encCfg) - } - })) - - utils.NewPprofServer(setupLog, pprofAddr) - - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "5688274864.data.fluid.io", - Port: 9443, - }) - if err != nil { - setupLog.Error(err, "unable to start jindoruntime manager") - os.Exit(1) - } - - controllerOptions := controller.Options{ - MaxConcurrentReconciles: maxConcurrentReconciles, - } - - if err = (jindoctl.NewRuntimeReconciler(mgr.GetClient(), - ctrl.Log.WithName("jindoctl").WithName("JindoRuntime"), - mgr.GetScheme(), - mgr.GetEventRecorderFor("JindoRuntime"), - )).SetupWithManager(mgr, controllerOptions, eventDriven); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "JindoRuntime") - os.Exit(1) - } - - pr, err := net.ParsePortRange(portRange) - if err != nil { - setupLog.Error(err, "can't parse port range. Port range must be like -max") - os.Exit(1) - } - setupLog.Info("port range parsed", "port range", pr.String()) - - portallocator.SetupRuntimePortAllocator(mgr.GetClient(), pr, jindo.GetReservedPorts) - - setupLog.Info("starting jindoruntime-controller") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem jindoruntime-controller") + command := app.NewJindoFSCommand() + if err := command.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%s", err.Error()) os.Exit(1) } }