From ed3b2d586d0b1181c0db744283886e61130aab38 Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Fri, 30 Sep 2022 12:05:50 +0200 Subject: [PATCH] Kubevirt switch to instancetype Signed-off-by: Helene Durand --- examples/kubevirt-machinedeployment.yaml | 9 + go.mod | 3 +- go.sum | 22 +- .../provider/kubevirt/provider.go | 173 ++++++++---- .../provider/kubevirt/provider_test.go | 247 ++++++++++++++++++ .../provider/kubevirt/testdata/affinity.yaml | 81 ++++++ .../kubevirt/testdata/custom-local-disk.yaml | 73 ++++++ .../instancetype-preference-custom.yaml | 71 +++++ .../instancetype-preference-standard.yaml | 71 +++++ .../kubevirt/testdata/nominal-case.yaml | 72 +++++ .../kubevirt/testdata/secondary-disks.yaml | 110 ++++++++ .../testdata/topologyspreadconstraints.yaml | 78 ++++++ .../provider/kubevirt/types/types.go | 37 ++- 13 files changed, 994 insertions(+), 53 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index f2eecf71dd..d540dc72e1 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -32,6 +32,15 @@ spec: # If instead specified directly, this value should be a base64 encoded kubeconfig. value: "<< KUBECONFIG_BASE64 >>" virtualMachine: + instancetype: + name: "standard-2" + category: "standard" # Allowed values: "standard" (namespaced), "custom" (cluster-wide) + preference: + name: "sockets-advantage" + category: "standard" # Allowed values: "standard" (namespaced), "custom" (cluster-wide) + # Deprecated: in favor instancetype and preference + flavor: + name: "kubermatic-standard" template: cpus: "1" memory: "2048M" diff --git a/go.mod b/go.mod index 26511e312d..c3060215a4 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed kubevirt.io/api v0.57.1 - kubevirt.io/containerized-data-importer-api v1.50.0 + kubevirt.io/containerized-data-importer-api v1.54.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 ) @@ -130,6 +130,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/peterhellberg/link v1.1.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go.sum b/go.sum index e5ecf23c73..f51932718e 100644 --- a/go.sum +++ b/go.sum @@ -236,6 +236,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -452,6 +457,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -698,6 +704,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4= +github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -959,6 +968,7 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1117,6 +1127,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1279,6 +1290,7 @@ golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1511,6 +1523,7 @@ gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= @@ -1548,12 +1561,14 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8c.io/operating-system-manager v1.0.0 h1:E1dCaLHypgaaLNgm50jcT3uwk3vok3xWYOnFcspXJ38= k8c.io/operating-system-manager v1.0.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= +k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= @@ -1561,6 +1576,7 @@ k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= +k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= @@ -1584,14 +1600,15 @@ k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHU k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kubevirt.io/api v0.57.1 h1:z6ImWKCQL2efFYqMWmxEsDNyt8c6mbWk7oCY6ZAa06U= kubevirt.io/api v0.57.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= -kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= +kubevirt.io/containerized-data-importer-api v1.54.0 h1:0nIFScuAQNtD2OHNM3hNyBRrZwgOKIOUlD1JIG0PWxI= +kubevirt.io/containerized-data-importer-api v1.54.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1605,6 +1622,7 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index fab9276420..3315217cc7 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -97,6 +97,8 @@ type Config struct { StorageClassName string PVCSize resource.Quantity FlavorName string + Instancetype *kubevirtv1.InstancetypeMatcher + Preference *kubevirtv1.PreferenceMatcher SecondaryDisks []SecondaryDisks PodAffinityPreset AffinityType PodAntiAffinityPreset AffinityType @@ -265,9 +267,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } - config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) + // Instancetype and Preference + config.Instancetype, err = p.parseInstancetype(rawConfig.VirtualMachine.Instancetype) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) + return nil, nil, fmt.Errorf(`failed to parse "instancetype" field: %w`, err) + } + config.Preference, err = p.parsePreference(kubevirttypes.Instancetype(rawConfig.VirtualMachine.Preference)) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "preference" field: %w`, err) } dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) @@ -317,6 +324,56 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &config, pconfig, nil } +func (p *provider) parseInstancetype(instancetype kubevirttypes.Instancetype) (*kubevirtv1.InstancetypeMatcher, error) { + name, err := p.configVarResolver.GetConfigVarStringValue(instancetype.Name) + if err != nil { + return nil, fmt.Errorf(`failed to parse "instancetype.name" field: %w`, err) + } + category, err := p.configVarResolver.GetConfigVarStringValue(instancetype.Category) + if err != nil { + return nil, fmt.Errorf(`failed to parse "instancetype.category" field: %w`, err) + } + switch category { + case string(kubevirttypes.InstancetypeCustom): + return &kubevirtv1.InstancetypeMatcher{ + Name: name, + Kind: "VirtualMachineClusterInstancetype", + }, nil + case string(kubevirttypes.InstancetypeStandard): + return &kubevirtv1.InstancetypeMatcher{ + Name: name, + Kind: "VirtualMachineInstancetype", + }, nil + } + + return nil, nil +} + +func (p *provider) parsePreference(preference kubevirttypes.Instancetype) (*kubevirtv1.PreferenceMatcher, error) { + name, err := p.configVarResolver.GetConfigVarStringValue(preference.Name) + if err != nil { + return nil, fmt.Errorf(`failed to parse "preference.name" field: %w`, err) + } + category, err := p.configVarResolver.GetConfigVarStringValue(preference.Category) + if err != nil { + return nil, fmt.Errorf(`failed to parse "preference.category" field: %w`, err) + } + switch category { + case string(kubevirttypes.InstancetypeCustom): + return &kubevirtv1.PreferenceMatcher{ + Name: name, + Kind: "VirtualMachineClusterPreference", + }, nil + case string(kubevirttypes.InstancetypeStandard): + return &kubevirtv1.PreferenceMatcher{ + Name: name, + Kind: "VirtualMachinePreference", + }, nil + } + + return nil, nil +} + func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { nodeAffinity := NodeAffinityPreset{} var err error @@ -328,7 +385,7 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) } - nodeAffinity.Values = make([]string, len(nodeAffinityPreset.Values)) + nodeAffinity.Values = make([]string, 0, len(nodeAffinityPreset.Values)) for _, v := range nodeAffinityPreset.Values { valueString, err := p.configVarResolver.GetConfigVarStringValue(v) if err != nil { @@ -445,8 +502,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to parse config: %w", err) } - // If VMIPreset is specified, skip CPU and Memory validation. - if c.FlavorName == "" { + // If instancetype is specified, skip CPU and Memory validation. + // Values will come from instancetype. + if c.Instancetype == nil { if _, err := parseResources(c.CPUs, c.Memory); err != nil { return err } @@ -504,6 +562,15 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } +type machineDeploymentNameGetter func() (string, error) + +func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) machineDeploymentNameGetter { + mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, c) + return func() (string, error) { + return mdName, err + } +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -513,34 +580,52 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } + sigClient, err := client.New(c.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) + } + + userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) + + virtualMachine, err := p.newVirtualMachine(c, pc, machine, userDataSecretName, userdata, + machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter) + if err != nil { + return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) + } + + if err := sigClient.Create(ctx, virtualMachine); err != nil { + return nil, fmt.Errorf("failed to create vmi: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: virtualMachine.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, + }, + Data: map[string][]byte{"userdata": []byte(userdata)}, + } + if err := sigClient.Create(ctx, secret); err != nil { + return nil, fmt.Errorf("failed to create secret for userdata: %w", err) + } + return &kubeVirtServer{}, nil +} + +func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, + userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) - userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} - // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, data.Client); err == nil { + //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). + if mdName, err := mdNameGetter(); err == nil { labels[machineDeploymentLabelKey] = mdName } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - // Add VMIPreset label if specified - if c.FlavorName != "" { - vmiPreset := kubevirtv1.VirtualMachineInstancePreset{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.FlavorName}, &vmiPreset); err != nil { - return nil, err - } - for key, val := range vmiPreset.Spec.Selector.MatchLabels { - labels[key] = val - } - } else { + if c.Instancetype == nil { requestsAndLimits, err := parseResources(c.CPUs, c.Memory) if err != nil { return nil, err @@ -562,7 +647,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - defaultBridgeNetwork, err := defaultBridgeNetwork() + defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") } @@ -574,7 +659,9 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - Running: utilpointer.BoolPtr(true), + Running: utilpointer.BoolPtr(true), + Instancetype: c.Instancetype, + Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, @@ -593,7 +680,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, Affinity: getAffinity(c, machineDeploymentLabelKey, labels[machineDeploymentLabelKey]), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), + Volumes: getVMVolumes(c, dataVolumeName, userdataSecretName), DNSPolicy: c.DNSPolicy, DNSConfig: c.DNSConfig, TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), @@ -602,23 +689,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), }, } - - if err := sigClient.Create(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to create vmi: %w", err) - } - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: userDataSecretName, - Namespace: virtualMachine.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, - }, - Data: map[string][]byte{"userdata": []byte(userdata)}, - } - if err := sigClient.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret for userdata: %w", err) - } - return &kubeVirtServer{}, nil + return virtualMachine, nil } func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { @@ -700,13 +771,23 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { return disks } -func defaultBridgeNetwork() (*kubevirtv1.Interface, error) { - defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() +type macAddressGetter func() (string, error) + +func randomMacAddressGetter() (string, error) { mac, err := netutil.GenerateRandMAC() + if err != nil { + return "", err + } + return mac.String(), nil +} + +func defaultBridgeNetwork(macAddressGetter macAddressGetter) (*kubevirtv1.Interface, error) { + defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() + mac, err := macAddressGetter() if err != nil { return nil, err } - defaultBridgeNetwork.MacAddress = mac.String() + defaultBridgeNetwork.MacAddress = mac return defaultBridgeNetwork, nil } diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index b651783178..5740c80a80 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -17,13 +17,260 @@ limitations under the License. package kubevirt import ( + "bytes" + "context" + "embed" + "html/template" + "path" "reflect" "testing" + kubevirtv1 "kubevirt.io/api/core/v1" + + kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" + cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/diff" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + //go:embed testdata + vmManifestsFS embed.FS + vmDir = "testdata" + fakeclient ctrlruntimeclient.WithWatch + expectedVms map[string]*kubevirtv1.VirtualMachine +) + +func init() { + fakeclient = fakectrlruntimeclient.NewClientBuilder().Build() + objs := runtimeFromYaml(fakeclient, vmManifestsFS, vmDir) + expectedVms = toVirtualMachines(objs) +} + +type kubevirtProviderSpecConf struct { + OsImageDV string // if OsImage from DV and not from http source + InstancetypeName string + InstancetypeCategory kubevirttypes.VirtualMachineInstancetypeCategory + PreferenceName string + PreferenceCategory kubevirttypes.VirtualMachineInstancetypeCategory + OperatingSystem string + TopologySpreadConstraint bool + Affinity bool + SecondaryDisks bool +} + +func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { + var out bytes.Buffer + tmpl, err := template.New("test").Parse(`{ + "cloudProvider": "kubevirt", + "cloudProviderSpec": { + "auth": { + "kubeconfig": "eyJhcGlWZXJzaW9uIjoidjEiLCJjbHVzdGVycyI6W3siY2x1c3RlciI6eyJjZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YSI6IiIsInNlcnZlciI6Imh0dHBzOi8vOTUuMjE2LjIwLjE0Njo2NDQzIn0sIm5hbWUiOiJrdWJlcm5ldGVzIn1dLCJjb250ZXh0cyI6W3siY29udGV4dCI6eyJjbHVzdGVyIjoia3ViZXJuZXRlcyIsIm5hbWVzcGFjZSI6Imt1YmUtc3lzdGVtIiwidXNlciI6Imt1YmVybmV0ZXMtYWRtaW4ifSwibmFtZSI6Imt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcyJ9XSwiY3VycmVudC1jb250ZXh0Ijoia3ViZXJuZXRlcy1hZG1pbkBrdWJlcm5ldGVzIiwia2luZCI6IkNvbmZpZyIsInByZWZlcmVuY2VzIjp7fSwidXNlcnMiOlt7Im5hbWUiOiJrdWJlcm5ldGVzLWFkbWluIiwidXNlciI6eyJjbGllbnQtY2VydGlmaWNhdGUtZGF0YSI6IiIsImNsaWVudC1rZXktZGF0YSI6IiJ9fV19" + }, + {{- if .TopologySpreadConstraint }} + "topologySpreadConstraints": [{ + "maxSkew": "2", + "topologyKey": "key1", + "whenUnsatisfiable": "DoNotSchedule"},{ + "maxSkew": "3", + "topologyKey": "key2", + "whenUnsatisfiable": "ScheduleAnyway"}], + {{- end }} + {{- if .Affinity }} + "affinity": { + "nodeAffinityPreset": { + "type": "hard", + "key": "key1", + "values": [ + "foo1", "foo2" ] + } + }, + {{- end }} + "virtualMachine": { + {{- if .InstancetypeName }} + "instancetype": { + "name": "{{ .InstancetypeName }}", + "category": "{{ .InstancetypeCategory }}" + }, + {{- end }} + {{- if .PreferenceName }} + "preference": { + "name": "{{ .PreferenceName }}", + "category": "{{ .PreferenceCategory }}" + }, + {{- end }} + "template": { + "cpus": "2", + "memory": "2Gi", + {{- if .SecondaryDisks }} + "secondaryDisks": [{ + "size": "20Gi", + "storageClassName": "longhorn2"},{ + "size": "30Gi", + "storageClassName": "longhorn3"}], + {{- end }} + "primaryDisk": { + {{- if .OsImageDV }} + "osImage": "{{ .OsImageDV }}", + {{- else }} + "osImage": "http://x.y.z.t/ubuntu.img", + {{- end }} + "size": "10Gi", + "storageClassName": "longhorn" + } + } + } + }, + "operatingSystem": "ubuntu", + "operatingSystemSpec": { + "disableAutoUpdate": false, + "disableLocksmithD": true, + "disableUpdateEngine": false + } +}`) + if err != nil { + t.Fatalf("Error occurred while parsing openstack provider spec template: %v", err) + } + err = tmpl.Execute(&out, k) + if err != nil { + t.Fatalf("Error occurred while executing openstack provider spec template: %v", err) + } + t.Logf("Generated providerSpec: %s", out.String()) + return out.Bytes() +} + +var ( + userdata = "fake-userdata" + testNamespace = "test-namespace" ) +func TestNewVirtualMachine(t *testing.T) { + tests := []struct { + name string + specConf kubevirtProviderSpecConf + }{ + { + name: "nominal-case", + specConf: kubevirtProviderSpecConf{}, + }, + { + name: "instancetype-preference-standard", + specConf: kubevirtProviderSpecConf{ + InstancetypeName: "standard-it", + InstancetypeCategory: kubevirttypes.InstancetypeStandard, + PreferenceName: "standard-pref", + PreferenceCategory: kubevirttypes.InstancetypeStandard, + }, + }, + { + name: "instancetype-preference-custom", + specConf: kubevirtProviderSpecConf{ + InstancetypeName: "custom-it", + InstancetypeCategory: kubevirttypes.InstancetypeCustom, + PreferenceName: "custom-pref", + PreferenceCategory: kubevirttypes.InstancetypeCustom, + }, + }, + { + name: "topologyspreadconstraints", + specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, + }, + { + name: "affinity", + specConf: kubevirtProviderSpecConf{Affinity: true}, + }, + { + name: "secondary-disks", + specConf: kubevirtProviderSpecConf{SecondaryDisks: true}, + }, + { + name: "custom-local-disk", + specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &provider{ + // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient), + } + + machine := cloudprovidertesting.Creator{ + Name: tt.name, + Namespace: "kubevirt", + ProviderSpecGetter: tt.specConf.rawProviderSpec, + }.CreateMachine(t) + + c, pc, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + t.Fatalf("provider.getConfig() error %v", err) + } + // Do not rely on POD_NAMESPACE env variable, force to known value + c.Namespace = testNamespace + + // Check the created VirtualMachine + vm, _ := p.newVirtualMachine(c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter) + vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() + + if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { + t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) + } + }) + } +} + +func fakeMachineDeploymentNameAndRevisionForMachineGetter() machineDeploymentNameGetter { + return func() (string, error) { + return "md-name", nil + } +} + +func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualMachine { + vms := make(map[string]*kubevirtv1.VirtualMachine) + for _, o := range objects { + if vm, ok := o.(*kubevirtv1.VirtualMachine); ok { + vms[vm.Name] = vm + } + } + return vms +} + +func fixedMacAddressGetter() (string, error) { + return "b6:f5:b4:fe:45:1d", nil +} + +// runtimeFromYaml returns a list of Kubernetes runtime objects from their yaml templates. +// It returns the objects for all files included in the ManifestFS folder, skipping (with error log) the yaml files +// that would not contain correct yaml files. +func runtimeFromYaml(client ctrlruntimeclient.Client, fs embed.FS, dir string) []runtime.Object { + decode := serializer.NewCodecFactory(client.Scheme()).UniversalDeserializer().Decode + + files, _ := fs.ReadDir(dir) + objects := make([]runtime.Object, 0, len(files)) + + for _, f := range files { + manifest, err := fs.ReadFile(path.Join(dir, f.Name())) + if err != nil { + continue + } + obj, _, err := decode(manifest, nil, nil) + // Skip and log but continue with others + if err != nil { + continue + } + objects = append(objects, obj) + } + + return objects +} func TestTopologySpreadConstraint(t *testing.T) { tests := []struct { desc string diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml new file mode 100644 index 0000000000..4c1ffe4707 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -0,0 +1,81 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: affinity + md: md-name + name: affinity + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: affinity + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + labels: + kubevirt.io/vm: affinity + md: md-name + spec: + affinity: + nodeAffinity: # Section present if nodeAffinityPreset.type != "" + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: + - foo1 + - foo2 + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: affinity + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml new file mode 100644 index 0000000000..a84d7c91ab --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -0,0 +1,73 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: custom-local-disk + md: md-name + name: custom-local-disk + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: custom-local-disk + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + pvc: + namespace: ns + name: dvname + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: custom-local-disk + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: custom-local-disk + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml new file mode 100644 index 0000000000..8d90a87470 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -0,0 +1,71 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: instancetype-preference-custom + md: md-name + name: instancetype-preference-custom + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: instancetype-preference-custom + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineClusterInstancetype + name: custom-it + preference: + kind: VirtualMachineClusterPreference + name: custom-pref + template: + metadata: + labels: + kubevirt.io/vm: instancetype-preference-custom + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-custom + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml new file mode 100644 index 0000000000..709de11999 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -0,0 +1,71 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: instancetype-preference-standard + md: md-name + name: instancetype-preference-standard + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: instancetype-preference-standard + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineInstancetype + name: standard-it + preference: + kind: VirtualMachinePreference + name: standard-pref + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: instancetype-preference-standard + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-standard + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml new file mode 100644 index 0000000000..bdb107a142 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -0,0 +1,72 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: nominal-case + md: md-name + name: nominal-case + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: nominal-case + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: nominal-case + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: nominal-case + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml new file mode 100644 index 0000000000..b1137331a2 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -0,0 +1,110 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: secondary-disks + md: md-name + name: secondary-disks + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: secondary-disks + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk0 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: longhorn2 + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk1 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + storageClassName: longhorn3 + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: secondary-disks + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: + bus: virtio + name: secondary-disks-secondarydisk0 + - disk: + bus: virtio + name: secondary-disks-secondarydisk1 + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: secondary-disks + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + - dataVolume: + name: secondary-disks-secondarydisk0 + name: secondary-disks-secondarydisk0 + - dataVolume: + name: secondary-disks-secondarydisk1 + name: secondary-disks-secondarydisk1 diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml new file mode 100644 index 0000000000..4f51eeb63b --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: topologyspreadconstraints + md: md-name + name: topologyspreadconstraints + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: topologyspreadconstraints + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: topologyspreadconstraints + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 2 + topologykey: key1 + whenunsatisfiable: DoNotSchedule + labelselector: + matchlabels: + md: md-name + - maxskew: 3 + topologykey: key2 + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: topologyspreadconstraints + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 7cca0a96a0..fe92f696a1 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -37,18 +37,47 @@ type Auth struct { // VirtualMachine. type VirtualMachine struct { - Flavor Flavor `json:"flavor,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + // Deprecated: in favour of Instancetype and Preference. + Flavor Flavor `json:"flavor,omitempty"` + Instancetype Instancetype `json:"instancetype,omitempty"` + Preference Preference `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } // Flavor. +// Deprecated: in favour of Instancetype and Preference. type Flavor struct { Name providerconfigtypes.ConfigVarString `json:"name,omitempty"` Profile providerconfigtypes.ConfigVarString `json:"profile,omitempty"` } +// Instancetype provide a way to define a set of resource, performance and other runtime characteristics, +// allowing users to reuse these definitions across multiple VirtualMachines. +// Anything provided within an instancetype cannot be overridden within the VirtualMachine. +type Instancetype struct { + // Category defines a category of VirtualMachineInstancetype (possible values VirtualMachineInstancetypeCategory). + Category providerconfigtypes.ConfigVarString `json:"category,omitempty"` + Name providerconfigtypes.ConfigVarString `json:"name,omitempty"` +} + +// Preference are like Instancetype defining runtime characteristics. But unlike Instancetypes, +// Preferences only represent the preferred values and as such can be overridden by values in the VirtualMachine. +type Preference Instancetype + +// VirtualMachineInstancetypeCategory defines a category of VirtualMachineInstancetype/VirtualMachinePreference. +type VirtualMachineInstancetypeCategory string + +const ( + + // cluster-wide resources (KubeVirt VirtualMachineClusterInstancetype/VirtualMachineClusterPreference). + InstancetypeCustom VirtualMachineInstancetypeCategory = "custom" + + // namespaced resources (KubeVirt VirtualMachineInstancetype/VirtualMachinePrefence). + InstancetypeStandard VirtualMachineInstancetypeCategory = "standard" +) + // Template. type Template struct { CPUs providerconfigtypes.ConfigVarString `json:"cpus,omitempty"`