From a064d04a664b13a5137497ee128a6bffbe28d0a2 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 13:40:31 +0100 Subject: [PATCH 01/10] Update go dependencies to kubernetes v1.30.2 Signed-off-by: Kevin Klues --- go.mod | 51 ++++++++++---------- go.sum | 148 +++++++++++++++++++++++---------------------------------- 2 files changed, 85 insertions(+), 114 deletions(-) diff --git a/go.mod b/go.mod index 7f03ec3b..c68d3716 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/spf13/pflag v1.0.5 github.com/urfave/cli/v2 v2.25.3 - k8s.io/api v0.28.0 - k8s.io/apimachinery v0.28.0 - k8s.io/client-go v0.28.0 - k8s.io/component-base v0.28.0 - k8s.io/dynamic-resource-allocation v0.28.0 - k8s.io/klog/v2 v2.100.1 - k8s.io/kubelet v0.28.0 + k8s.io/api v0.30.2 + k8s.io/apimachinery v0.30.2 + k8s.io/client-go v0.30.2 + k8s.io/component-base v0.30.2 + k8s.io/dynamic-resource-allocation v0.30.2 + k8s.io/klog/v2 v2.120.1 + k8s.io/kubelet v0.30.2 ) require ( @@ -23,19 +23,19 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -57,26 +57,25 @@ require ( github.com/spf13/cobra v1.7.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.21.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.13.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/mod v0.15.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.54.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 9116b8a1..83501f1a 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -22,19 +20,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= @@ -52,14 +48,15 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -81,7 +78,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -103,10 +99,10 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= -github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg= github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -117,7 +113,6 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go. github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -148,13 +143,11 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -173,46 +166,33 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRT github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -221,50 +201,43 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -274,30 +247,29 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= -k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= -k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= -k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= -k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= -k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= -k8s.io/component-base v0.28.0 h1:HQKy1enJrOeJlTlN4a6dU09wtmXaUvThC0irImfqyxI= -k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk= -k8s.io/dynamic-resource-allocation v0.28.0 h1:LQjb8kRQcvorEHlJVfHLSGued8taykmqROMGozMpHsY= -k8s.io/dynamic-resource-allocation v0.28.0/go.mod h1:x8xyQvoo22hfBEZlKt3nqkzfoTcMOJxiD9mSgDgrd2w= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U= -k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= +k8s.io/component-base v0.30.2 h1:pqGBczYoW1sno8q9ObExUqrYSKhtE5rW3y6gX88GZII= +k8s.io/component-base v0.30.2/go.mod h1:yQLkQDrkK8J6NtP+MGJOws+/PPeEXNpwFixsUI7h/OE= +k8s.io/dynamic-resource-allocation v0.30.2 h1:wEhjNbVPymPEY5Db4UXPiQkioHV/4MHDzAkf+1TLaNM= +k8s.io/dynamic-resource-allocation v0.30.2/go.mod h1:J5gKMh7FcGcWziX6ugeNfyFM8j1mvxBgYWrLfRDZ38k= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kubelet v0.30.2 h1:Ck4E/pHndI20IzDXxS57dElhDGASPO5pzXF7BcKfmCY= +k8s.io/kubelet v0.30.2/go.mod h1:DSwwTbLQmdNkebAU7ypIALR4P9aXZNFwgRmedojUE94= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 6b1df1b43d9a7c682531fa704322df61d9906d40 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 10 Jul 2024 15:39:03 +0000 Subject: [PATCH 02/10] Update to kindest/node:v1.30.2 Signed-off-by: Kevin Klues --- demo/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo/scripts/common.sh b/demo/scripts/common.sh index b3aa0ac0..121ea619 100644 --- a/demo/scripts/common.sh +++ b/demo/scripts/common.sh @@ -33,7 +33,7 @@ SCRIPTS_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)" # The kubernetes tag to build the kind cluster from # From https://github.com/kubernetes/kubernetes/tags -: ${KIND_K8S_TAG:="v1.28.0"} +: ${KIND_K8S_TAG:="v1.30.2"} # At present, kind has a new enough node image that we don't need to build our # own. This won't always be true and we may need to set the variable below to From f35a0cd82b7c3aec65904d09e9b4abf3a1ddc70d Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 14:21:35 +0100 Subject: [PATCH 03/10] Add support to advertise allocatable GPUs as a ResourceSlice Signed-off-by: Kevin Klues --- cmd/dra-example-kubeletplugin/driver.go | 24 ++++++++++++++++++++++++ cmd/dra-example-kubeletplugin/state.go | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/cmd/dra-example-kubeletplugin/driver.go b/cmd/dra-example-kubeletplugin/driver.go index 7aa351eb..9673407c 100644 --- a/cmd/dra-example-kubeletplugin/driver.go +++ b/cmd/dra-example-kubeletplugin/driver.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + resourceapi "k8s.io/api/resource/v1alpha2" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1alpha3" @@ -31,6 +32,7 @@ import ( var _ drapbv1.NodeServer = &driver{} type driver struct { + doneCh chan struct{} nascrd *nascrd.NodeAllocationState nasclient *nasclient.Client state *DeviceState @@ -86,6 +88,8 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) { } func (d *driver) Shutdown(ctx context.Context) error { + defer close(d.doneCh) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { err := d.nasclient.Get(ctx) if err != nil { @@ -95,6 +99,26 @@ func (d *driver) Shutdown(ctx context.Context) error { }) } +func (d *driver) NodeListAndWatchResources(req *drapbv1.NodeListAndWatchResourcesRequest, stream drapbv1.Node_NodeListAndWatchResourcesServer) error { + model := d.state.getResourceModelFromAllocatableDevices() + resp := &drapbv1.NodeListAndWatchResourcesResponse{ + Resources: []*resourceapi.ResourceModel{&model}, + } + + if err := stream.Send(resp); err != nil { + return err + } + + //nolint:all,S1000: should use for range instead of for { select {} } (gosimple) + for { + select { + case <-d.doneCh: + return nil + } + // TODO: Update with case for when GPUs go unhealthy + } +} + func (d *driver) NodePrepareResources(ctx context.Context, req *drapbv1.NodePrepareResourcesRequest) (*drapbv1.NodePrepareResourcesResponse, error) { logger := klog.FromContext(ctx) logger.Info("NodePrepareResource", "numClaims", len(req.Claims)) diff --git a/cmd/dra-example-kubeletplugin/state.go b/cmd/dra-example-kubeletplugin/state.go index 1163740d..d754a3ba 100644 --- a/cmd/dra-example-kubeletplugin/state.go +++ b/cmd/dra-example-kubeletplugin/state.go @@ -18,8 +18,11 @@ package main import ( "fmt" + "strings" "sync" + resourceapi "k8s.io/api/resource/v1alpha2" + nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" ) @@ -192,6 +195,22 @@ func (s *DeviceState) unprepareGpus(claimUID string, devices *PreparedDevices) e return nil } +func (s *DeviceState) getResourceModelFromAllocatableDevices() resourceapi.ResourceModel { + var instances []resourceapi.NamedResourcesInstance + for _, device := range s.allocatable { + instance := resourceapi.NamedResourcesInstance{ + Name: strings.ToLower(device.uuid), + } + instances = append(instances, instance) + } + + model := resourceapi.ResourceModel{ + NamedResources: &resourceapi.NamedResourcesResources{Instances: instances}, + } + + return model +} + func (s *DeviceState) syncAllocatableDevicesToCRDSpec(spec *nascrd.NodeAllocationStateSpec) error { gpus := make(map[string]nascrd.AllocatableDevice) for _, device := range s.allocatable { From c82ba3035511f9a48871091a7f574d7251bd6b09 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 14:22:23 +0100 Subject: [PATCH 04/10] Add support for {Prepare,Unprepare}Resources with stuctured parameters Signed-off-by: Kevin Klues --- cmd/dra-example-kubeletplugin/driver.go | 177 ++++++++++++++++-------- 1 file changed, 120 insertions(+), 57 deletions(-) diff --git a/cmd/dra-example-kubeletplugin/driver.go b/cmd/dra-example-kubeletplugin/driver.go index 9673407c..124be3d7 100644 --- a/cmd/dra-example-kubeletplugin/driver.go +++ b/cmd/dra-example-kubeletplugin/driver.go @@ -33,7 +33,7 @@ var _ drapbv1.NodeServer = &driver{} type driver struct { doneCh chan struct{} - nascrd *nascrd.NodeAllocationState + nascr *nascrd.NodeAllocationState nasclient *nasclient.Client state *DeviceState } @@ -73,7 +73,7 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) { } d = &driver{ - nascrd: config.nascr, + nascr: config.nascr, nasclient: client, state: state, } @@ -120,8 +120,7 @@ func (d *driver) NodeListAndWatchResources(req *drapbv1.NodeListAndWatchResource } func (d *driver) NodePrepareResources(ctx context.Context, req *drapbv1.NodePrepareResourcesRequest) (*drapbv1.NodePrepareResourcesResponse, error) { - logger := klog.FromContext(ctx) - logger.Info("NodePrepareResource", "numClaims", len(req.Claims)) + klog.Infof("NodePrepareResource is called: number of claims: %d", len(req.Claims)) preparedResources := &drapbv1.NodePrepareResourcesResponse{Claims: map[string]*drapbv1.NodePrepareResourceResponse{}} // In production version some common operations of d.nodeUnprepareResources @@ -135,51 +134,29 @@ func (d *driver) NodePrepareResources(ctx context.Context, req *drapbv1.NodePrep } func (d *driver) nodePrepareResource(ctx context.Context, claim *drapbv1.Claim) *drapbv1.NodePrepareResourceResponse { - logger := klog.FromContext(ctx) - var err error - var prepared []string - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - prepared, err = d.prepare(ctx, claim.Uid) - if err != nil { - return fmt.Errorf("error allocating devices for claim '%v': %v", claim.Uid, err) - } - - updatedSpec, err := d.state.GetUpdatedSpec(&d.nascrd.Spec) - if err != nil { - return fmt.Errorf("error getting updated CR spec: %v", err) - } - - err = d.nasclient.Update(ctx, updatedSpec) - if err != nil { - if err := d.state.Unprepare(claim.Uid); err != nil { - logger.Error(err, "Failed to unprepare after Update", "claim", claim.Uid) + if len(claim.StructuredResourceHandle) > 0 { + if err := d.allocateDevices(ctx, claim); err != nil { + return &drapbv1.NodePrepareResourceResponse{ + Error: fmt.Sprintf("error allocating devices for claim %v: %v", claim.Uid, err), } - return err } + } - return nil - }) - + prepared, err := d.prepare(ctx, claim.Uid) if err != nil { return &drapbv1.NodePrepareResourceResponse{ - Error: fmt.Sprintf("error preparing resource: %v", err), + Error: fmt.Sprintf("error preparing devices for claim %v: %v", claim.Uid, err), } } - klog.FromContext(ctx).Info("Prepared devices", "claim", claim.Uid) + klog.Infof("Returning newly prepared devices for claim '%v': %s", claim.Uid, prepared) return &drapbv1.NodePrepareResourceResponse{CDIDevices: prepared} } func (d *driver) NodeUnprepareResources(ctx context.Context, req *drapbv1.NodeUnprepareResourcesRequest) (*drapbv1.NodeUnprepareResourcesResponse, error) { - logger := klog.FromContext(ctx) - logger.Info("NodeUnprepareResource", "numClaims", len(req.Claims)) - unpreparedResources := &drapbv1.NodeUnprepareResourcesResponse{ - Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{}, - } + klog.Infof("NodeUnPrepareResource is called: number of claims: %d", len(req.Claims)) + unpreparedResources := &drapbv1.NodeUnprepareResourcesResponse{Claims: map[string]*drapbv1.NodeUnprepareResourceResponse{}} - // In production version some common operations of d.nodeUnprepareResources - // should be done outside of the loop, for instance updating the CR could - // be done once after all HW was unprepared. for _, claim := range req.Claims { unpreparedResources.Claims[claim.Uid] = d.nodeUnprepareResource(ctx, claim) } @@ -188,13 +165,38 @@ func (d *driver) NodeUnprepareResources(ctx context.Context, req *drapbv1.NodeUn } func (d *driver) nodeUnprepareResource(ctx context.Context, claim *drapbv1.Claim) *drapbv1.NodeUnprepareResourceResponse { - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := d.unprepare(ctx, claim.Uid) + if len(claim.StructuredResourceHandle) > 0 { + if err := d.deallocateDevices(ctx, claim); err != nil { + return &drapbv1.NodeUnprepareResourceResponse{ + Error: fmt.Sprintf("error deallocating devices for claim %v: %v", claim.Uid, err), + } + } + } + + if err := d.unprepare(ctx, claim.Uid); err != nil { + return &drapbv1.NodeUnprepareResourceResponse{ + Error: fmt.Sprintf("error unpreparing devices for claim %v: %v", claim.Uid, err), + } + } + + return &drapbv1.NodeUnprepareResourceResponse{} +} + +func (d *driver) prepare(ctx context.Context, claimUID string) ([]string, error) { + var err error + var prepared []string + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + err = d.nasclient.Get(ctx) if err != nil { - return fmt.Errorf("error unpreparing devices for claim '%v': %v", claim.Uid, err) + return err } - updatedSpec, err := d.state.GetUpdatedSpec(&d.nascrd.Spec) + prepared, err = d.state.Prepare(claimUID, d.nascr.Spec.AllocatedClaims[claimUID]) + if err != nil { + return err + } + + updatedSpec, err := d.state.GetUpdatedSpec(&d.nascr.Spec) if err != nil { return fmt.Errorf("error getting updated CR spec: %v", err) } @@ -207,33 +209,94 @@ func (d *driver) nodeUnprepareResource(ctx context.Context, claim *drapbv1.Claim return nil }) if err != nil { - return &drapbv1.NodeUnprepareResourceResponse{ - Error: fmt.Sprintf("error unpreparing resource: %v", err), - } + return nil, err } - - klog.FromContext(ctx).Info("Unprepared devices", "claim", claim.Uid) - return &drapbv1.NodeUnprepareResourceResponse{} + return prepared, nil } -func (d *driver) prepare(ctx context.Context, claimUID string) ([]string, error) { - err := d.nasclient.Get(ctx) - if err != nil { - return nil, err - } - prepared, err := d.state.Prepare(claimUID, d.nascrd.Spec.AllocatedClaims[claimUID]) +func (d *driver) unprepare(ctx context.Context, claimUID string) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := d.nasclient.Get(ctx) + if err != nil { + return err + } + + err = d.state.Unprepare(claimUID) + if err != nil { + return err + } + + updatedSpec, err := d.state.GetUpdatedSpec(&d.nascr.Spec) + if err != nil { + return fmt.Errorf("error getting updated CR spec: %v", err) + } + + err = d.nasclient.Update(ctx, updatedSpec) + if err != nil { + return err + } + + return nil + }) if err != nil { - return nil, err + return err } - return prepared, nil + return nil } -func (d *driver) unprepare(ctx context.Context, claimUID string) error { - err := d.nasclient.Get(ctx) +func (d *driver) allocateDevices(ctx context.Context, claim *drapbv1.Claim) error { + allocated := nascrd.AllocatedDevices{ + Gpu: &nascrd.AllocatedGpus{}, + } + + for _, r := range claim.StructuredResourceHandle[0].Results { + name := r.AllocationResultModel.NamedResources.Name + gpu := nascrd.AllocatedGpu{ + UUID: fmt.Sprintf("GPU-%s", name[4:]), + } + allocated.Gpu.Devices = append(allocated.Gpu.Devices, gpu) + } + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := d.nasclient.Get(ctx) + if err != nil { + return err + } + + if len(d.nascr.Spec.AllocatedClaims) == 0 { + d.nascr.Spec.AllocatedClaims = make(map[string]nascrd.AllocatedDevices) + } + d.nascr.Spec.AllocatedClaims[claim.Uid] = allocated + + err = d.nasclient.Update(ctx, &d.nascr.Spec) + if err != nil { + return err + } + + return nil + }) if err != nil { return err } - err = d.state.Unprepare(claimUID) + return nil +} + +func (d *driver) deallocateDevices(ctx context.Context, claim *drapbv1.Claim) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := d.nasclient.Get(ctx) + if err != nil { + return err + } + + delete(d.nascr.Spec.AllocatedClaims, claim.Uid) + + err = d.nasclient.Update(ctx, &d.nascr.Spec) + if err != nil { + return err + } + + return nil + }) if err != nil { return err } From 41ca66b09d903b52bd82211314baccf871f58217 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 14:24:36 +0100 Subject: [PATCH 05/10] Turn the GpuClaimParameters.Count field into a pointer type Signed-off-by: Kevin Klues --- api/example.com/resource/gpu/v1alpha1/api.go | 4 +++- api/example.com/resource/gpu/v1alpha1/gpuclaim.go | 2 +- .../resource/gpu/v1alpha1/zz_generated.deepcopy.go | 7 ++++++- cmd/dra-example-controller/gpu.go | 6 +++--- go.mod | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/api/example.com/resource/gpu/v1alpha1/api.go b/api/example.com/resource/gpu/v1alpha1/api.go index e6a7e598..eae4b011 100644 --- a/api/example.com/resource/gpu/v1alpha1/api.go +++ b/api/example.com/resource/gpu/v1alpha1/api.go @@ -17,6 +17,8 @@ package v1alpha1 import ( + "k8s.io/utils/ptr" + nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" ) @@ -40,6 +42,6 @@ func DefaultDeviceClassParametersSpec() *DeviceClassParametersSpec { func DefaultGpuClaimParametersSpec() *GpuClaimParametersSpec { return &GpuClaimParametersSpec{ - Count: 1, + Count: ptr.To(1), } } diff --git a/api/example.com/resource/gpu/v1alpha1/gpuclaim.go b/api/example.com/resource/gpu/v1alpha1/gpuclaim.go index a3d1e499..aabf6f75 100644 --- a/api/example.com/resource/gpu/v1alpha1/gpuclaim.go +++ b/api/example.com/resource/gpu/v1alpha1/gpuclaim.go @@ -22,7 +22,7 @@ import ( // GpuClaimParametersSpec is the spec for the GpuClaimParameters CRD. type GpuClaimParametersSpec struct { - Count int `json:"count,omitempty"` + Count *int `json:"count,omitempty"` } // +genclient diff --git a/api/example.com/resource/gpu/v1alpha1/zz_generated.deepcopy.go b/api/example.com/resource/gpu/v1alpha1/zz_generated.deepcopy.go index 16aa6fa2..9d816f0d 100644 --- a/api/example.com/resource/gpu/v1alpha1/zz_generated.deepcopy.go +++ b/api/example.com/resource/gpu/v1alpha1/zz_generated.deepcopy.go @@ -122,7 +122,7 @@ func (in *GpuClaimParameters) DeepCopyInto(out *GpuClaimParameters) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GpuClaimParameters. @@ -178,6 +178,11 @@ func (in *GpuClaimParametersList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GpuClaimParametersSpec) DeepCopyInto(out *GpuClaimParametersSpec) { *out = *in + if in.Count != nil { + in, out := &in.Count, &out.Count + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GpuClaimParametersSpec. diff --git a/cmd/dra-example-controller/gpu.go b/cmd/dra-example-controller/gpu.go index ad92ab94..308779b2 100644 --- a/cmd/dra-example-controller/gpu.go +++ b/cmd/dra-example-controller/gpu.go @@ -38,7 +38,7 @@ func NewGpuDriver() *gpudriver { } func (g *gpudriver) ValidateClaimParameters(claimParams *gpucrd.GpuClaimParametersSpec) error { - if claimParams.Count < 1 { + if *claimParams.Count < 1 { return fmt.Errorf("invalid number of GPUs requested: %v", claimParams.Count) } return nil @@ -78,7 +78,7 @@ func (g *gpudriver) UnsuitableNode(crd *nascrd.NodeAllocationState, pod *corev1. claimUID := string(ca.Claim.UID) claimParams, _ := ca.ClaimParameters.(*gpucrd.GpuClaimParametersSpec) - if claimParams.Count != len(allocated[claimUID]) { + if *claimParams.Count != len(allocated[claimUID]) { for _, ca := range allcas { ca.UnsuitableNodes = append(ca.UnsuitableNodes, potentialNode) } @@ -141,7 +141,7 @@ func (g *gpudriver) allocate(crd *nascrd.NodeAllocationState, pod *corev1.Pod, g claimParams, _ := ca.ClaimParameters.(*gpucrd.GpuClaimParametersSpec) var devices []string - for i := 0; i < claimParams.Count; i++ { + for i := 0; i < *claimParams.Count; i++ { for _, device := range available { devices = append(devices, device.UUID) delete(available, device.UUID) diff --git a/go.mod b/go.mod index c68d3716..edc3af00 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( k8s.io/dynamic-resource-allocation v0.30.2 k8s.io/klog/v2 v2.120.1 k8s.io/kubelet v0.30.2 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) require ( @@ -74,7 +75,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect From f46eea8ed7002c20dd2124daa7af19415b265b33 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 14:26:31 +0100 Subject: [PATCH 06/10] Add ability to generate ResourceClaimParameters from GpuClaimParameters Signed-off-by: Kevin Klues --- .../claimparametersgen.go | 238 ++++++++++++++++++ cmd/dra-example-controller/driver.go | 1 + cmd/dra-example-controller/main.go | 5 + 3 files changed, 244 insertions(+) create mode 100644 cmd/dra-example-controller/claimparametersgen.go diff --git a/cmd/dra-example-controller/claimparametersgen.go b/cmd/dra-example-controller/claimparametersgen.go new file mode 100644 index 00000000..d769bdd9 --- /dev/null +++ b/cmd/dra-example-controller/claimparametersgen.go @@ -0,0 +1,238 @@ +/* + * Copyright 2024 The Kubernetes Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + resourceapi "k8s.io/api/resource/v1alpha2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" +) + +func StartClaimParametersGenerator(ctx context.Context, config *Config) error { + // Build a client set config + csconfig, err := config.flags.kubeClientConfig.NewClientSetConfig() + if err != nil { + return fmt.Errorf("error creating client set config: %w", err) + } + + // Create a new dynamic client + dynamicClient, err := dynamic.NewForConfig(csconfig) + if err != nil { + return fmt.Errorf("error creating dynamic client: %w", err) + } + + klog.Info("Starting ResourceClaimParamaters generator") + + // Set up informer to watch for GpuClaimParameters objects + gpuClaimParametersInformer := newGpuClaimParametersInformer(ctx, dynamicClient) + + // Set up handler for events + _, err = gpuClaimParametersInformer.AddEventHandler(newGpuClaimParametersHandler(ctx, config.clientSets.Core, dynamicClient)) + if err != nil { + return fmt.Errorf("error adding event handler: %w", err) + } + + // Start informer + go gpuClaimParametersInformer.Run(ctx.Done()) + + return nil +} + +func newGpuClaimParametersInformer(ctx context.Context, dynamicClient dynamic.Interface) cache.SharedIndexInformer { + // Set up shared index informer for GpuClaimParameters objects + gvr := schema.GroupVersionResource{ + Group: gpucrd.GroupName, + Version: gpucrd.Version, + Resource: strings.ToLower(gpucrd.GpuClaimParametersKind), + } + + informer := cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{}) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return dynamicClient.Resource(gvr).Watch(ctx, metav1.ListOptions{}) + }, + }, + &unstructured.Unstructured{}, + 0, // resyncPeriod + cache.Indexers{}, + ) + + return informer +} + +func newGpuClaimParametersHandler(ctx context.Context, clientset kubernetes.Interface, dynamicClient dynamic.Interface) cache.ResourceEventHandler { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + unstructured, ok := obj.(*unstructured.Unstructured) + if !ok { + klog.Errorf("Error converting object to *unstructured.Unstructured: %v", obj) + } + + var gpuClaimParameters gpucrd.GpuClaimParameters + err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.Object, &gpuClaimParameters) + if err != nil { + klog.Errorf("Error converting *unstructured.Unstructured to GpuClaimParameters: %v", err) + return + } + + if err := createOrUpdateResourceClaimParameters(ctx, clientset, &gpuClaimParameters); err != nil { + klog.Errorf("Error creating ResourceClaimParameters: %v", err) + return + } + }, + UpdateFunc: func(oldObj any, newObj any) { + unstructured, ok := newObj.(*unstructured.Unstructured) + if !ok { + klog.Errorf("Error converting object to *unstructured.Unstructured: %v", newObj) + } + + var gpuClaimParameters gpucrd.GpuClaimParameters + err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.Object, &gpuClaimParameters) + if err != nil { + klog.Errorf("Error converting *unstructured.Unstructured to GpuClaimParameters: %v", err) + return + } + + if err := createOrUpdateResourceClaimParameters(ctx, clientset, &gpuClaimParameters); err != nil { + klog.Errorf("Error updating ResourceClaimParameters: %v", err) + return + } + }, + } +} + +func newResourceClaimParametersFromGpuClaimParameters(gpuClaimParameters *gpucrd.GpuClaimParameters) (*resourceapi.ResourceClaimParameters, error) { + rawSpec, err := json.Marshal(gpuClaimParameters.Spec) + if err != nil { + return nil, fmt.Errorf("error marshaling GpuClaimParamaters to JSON: %w", err) + } + + resourceCount := 1 + if gpuClaimParameters.Spec.Count != nil { + resourceCount = *gpuClaimParameters.Spec.Count + } + + selector := "true" + shareable := true + + var resourceRequests []resourceapi.ResourceRequest + for i := 0; i < resourceCount; i++ { + request := resourceapi.ResourceRequest{ + ResourceRequestModel: resourceapi.ResourceRequestModel{ + NamedResources: &resourceapi.NamedResourcesRequest{ + Selector: selector, + }, + }, + } + resourceRequests = append(resourceRequests, request) + } + + resourceClaimParameters := &resourceapi.ResourceClaimParameters{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "resource-claim-parameters-", + Namespace: gpuClaimParameters.Namespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: gpuClaimParameters.APIVersion, + Kind: gpuClaimParameters.Kind, + Name: gpuClaimParameters.Name, + UID: gpuClaimParameters.UID, + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + GeneratedFrom: &resourceapi.ResourceClaimParametersReference{ + APIGroup: gpucrd.GroupName, + Kind: gpuClaimParameters.Kind, + Name: gpuClaimParameters.Name, + }, + DriverRequests: []resourceapi.DriverRequests{ + { + DriverName: DriverName, + VendorParameters: runtime.RawExtension{Raw: rawSpec}, + Requests: resourceRequests, + }, + }, + Shareable: shareable, + } + + return resourceClaimParameters, nil +} + +func createOrUpdateResourceClaimParameters(ctx context.Context, clientset kubernetes.Interface, gpuClaimParameters *gpucrd.GpuClaimParameters) error { + namespace := gpuClaimParameters.Namespace + + // Get a list of existing ResourceClaimParameters in the same namespace as the incoming GpuClaimParameters + existing, err := clientset.ResourceV1alpha2().ResourceClaimParameters(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("error listing existing ResourceClaimParameters: %w", err) + } + + // Build a new ResourceClaimParameters object from the incoming GpuClaimParameters object + resourceClaimParameters, err := newResourceClaimParametersFromGpuClaimParameters(gpuClaimParameters) + if err != nil { + return fmt.Errorf("error building new ResourceClaimParameters object from a GpuClaimParameters object: %w", err) + } + + // If there is an existing ResourceClaimParameters generated from the incoming GpuClaimParameters object, then update it + if len(existing.Items) > 0 { + for _, item := range existing.Items { + if (item.GeneratedFrom.APIGroup == gpucrd.GroupName) && + (item.GeneratedFrom.Kind == gpuClaimParameters.Kind) && + (item.GeneratedFrom.Name == gpuClaimParameters.Name) { + klog.Infof("ResourceClaimParameters already exists for GpuClaimParameters %s/%s, updating it", namespace, gpuClaimParameters.Name) + + // Copy the matching ResourceClaimParameters metadata into the new ResourceClaimParameters object before updating it + resourceClaimParameters.ObjectMeta = *item.ObjectMeta.DeepCopy() + + _, err = clientset.ResourceV1alpha2().ResourceClaimParameters(namespace).Update(ctx, resourceClaimParameters, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error updating ResourceClaimParameters object: %w", err) + } + + return nil + } + } + } + + // Otherwise create a new ResourceClaimParameters object from the incoming GpuClaimParameters object + _, err = clientset.ResourceV1alpha2().ResourceClaimParameters(namespace).Create(ctx, resourceClaimParameters, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("error creating ResourceClaimParameters object from GpuClaimParameters object: %w", err) + } + + klog.Infof("Created ResourceClaimParameters for GpuClaimParameters %s/%s", namespace, gpuClaimParameters.Name) + return nil +} diff --git a/cmd/dra-example-controller/driver.go b/cmd/dra-example-controller/driver.go index 03ecfc53..b2f445cd 100644 --- a/cmd/dra-example-controller/driver.go +++ b/cmd/dra-example-controller/driver.go @@ -33,6 +33,7 @@ import ( const ( DriverAPIGroup = gpucrd.GroupName + DriverName = gpucrd.GroupName ) type OnSuccessCallback func() diff --git a/cmd/dra-example-controller/main.go b/cmd/dra-example-controller/main.go index de55547b..ddd9d949 100644 --- a/cmd/dra-example-controller/main.go +++ b/cmd/dra-example-controller/main.go @@ -144,6 +144,11 @@ func newApp() *cli.App { } } + err = StartClaimParametersGenerator(ctx, config) + if err != nil { + return fmt.Errorf("start claim parameters generator: %w", err) + } + err = StartController(ctx, config) if err != nil { return fmt.Errorf("start controller: %v", err) From 0a0a8bda016b4a484f8c7a4c7e11ff4b55ae84b9 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 14:27:15 +0100 Subject: [PATCH 07/10] Force ResourceClass deployed with helm to use structured parameters Signed-off-by: Kevin Klues --- deployments/helm/dra-example-driver/templates/resourceclass.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/helm/dra-example-driver/templates/resourceclass.yaml b/deployments/helm/dra-example-driver/templates/resourceclass.yaml index da337cca..9417a1a3 100644 --- a/deployments/helm/dra-example-driver/templates/resourceclass.yaml +++ b/deployments/helm/dra-example-driver/templates/resourceclass.yaml @@ -4,3 +4,4 @@ kind: ResourceClass metadata: name: gpu.example.com driverName: gpu.resource.example.com +structuredParameters: true From 62a4696d8aa2a2ea779a8b52624e262890424ea7 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 10 Jul 2024 15:32:46 +0000 Subject: [PATCH 08/10] Remove support for "classic" DRA Signed-off-by: Kevin Klues --- .gitignore | 1 - .../resource/gpu/nas/v1alpha1/api.go | 53 --- .../gpu/nas/v1alpha1/client/client.go | 101 ------ .../resource/gpu/nas/v1alpha1/doc.go | 20 -- .../resource/gpu/nas/v1alpha1/nas.go | 119 ------- .../resource/gpu/nas/v1alpha1/register.go | 49 --- .../gpu/nas/v1alpha1/zz_generated.deepcopy.go | 285 --------------- api/example.com/resource/gpu/v1alpha1/api.go | 7 +- cmd/dra-example-controller/allocations.go | 112 ------ .../claimparametersgen.go | 5 + cmd/dra-example-controller/driver.go | 324 ------------------ cmd/dra-example-controller/gpu.go | 155 --------- cmd/dra-example-controller/main.go | 25 +- cmd/dra-example-controller/mutex.go | 41 --- cmd/dra-example-kubeletplugin/cdi.go | 6 +- cmd/dra-example-kubeletplugin/driver.go | 211 ++---------- cmd/dra-example-kubeletplugin/main.go | 12 +- cmd/dra-example-kubeletplugin/state.go | 127 ++----- cmd/set-nas-status/main.go | 114 ------ common.mk | 2 +- deployments/container/Dockerfile | 1 - ...urce.example.com_nodeallocationstates.yaml | 113 ------ .../templates/clusterrole.yaml | 1 - .../templates/kubeletplugin.yaml | 22 -- .../resource/clientset/versioned/clientset.go | 13 - .../versioned/fake/clientset_generated.go | 7 - .../clientset/versioned/fake/register.go | 2 - .../clientset/versioned/scheme/register.go | 2 - .../versioned/typed/nas/v1alpha1/doc.go | 20 -- .../versioned/typed/nas/v1alpha1/fake/doc.go | 20 -- .../nas/v1alpha1/fake/fake_nas_client.go | 40 --- .../v1alpha1/fake/fake_nodeallocationstate.go | 129 ------- .../typed/nas/v1alpha1/generated_expansion.go | 21 -- .../typed/nas/v1alpha1/nas_client.go | 107 ------ .../typed/nas/v1alpha1/nodeallocationstate.go | 178 ---------- pkg/flags/{nodeallocationstate.go => crds.go} | 32 +- 36 files changed, 78 insertions(+), 2399 deletions(-) delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/api.go delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/client/client.go delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/doc.go delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/nas.go delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/register.go delete mode 100644 api/example.com/resource/gpu/nas/v1alpha1/zz_generated.deepcopy.go delete mode 100644 cmd/dra-example-controller/allocations.go delete mode 100644 cmd/dra-example-controller/driver.go delete mode 100644 cmd/dra-example-controller/gpu.go delete mode 100644 cmd/dra-example-controller/mutex.go delete mode 100644 cmd/set-nas-status/main.go delete mode 100644 deployments/helm/dra-example-driver/crds/nas.gpu.resource.example.com_nodeallocationstates.yaml delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/doc.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/doc.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nas_client.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nodeallocationstate.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/generated_expansion.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nas_client.go delete mode 100644 pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nodeallocationstate.go rename pkg/flags/{nodeallocationstate.go => crds.go} (58%) diff --git a/.gitignore b/.gitignore index 0150cd71..5e8272e1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,3 @@ vendor/ ./dra-example-controller ./dra-example-kubeletplugin -./set-nas-status diff --git a/api/example.com/resource/gpu/nas/v1alpha1/api.go b/api/example.com/resource/gpu/nas/v1alpha1/api.go deleted file mode 100644 index 3e723eab..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/api.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - GroupName = "nas.gpu.resource.example.com" - Version = "v1alpha1" - - GpuDeviceType = "gpu" - UnknownDeviceType = "unknown" - - NodeAllocationStateStatusReady = "Ready" - NodeAllocationStateStatusNotReady = "NotReady" -) - -type NodeAllocationStateConfig struct { - Name string - Namespace string - Owner *metav1.OwnerReference -} - -func NewNodeAllocationState(config *NodeAllocationStateConfig) *NodeAllocationState { - nascrd := &NodeAllocationState{ - ObjectMeta: metav1.ObjectMeta{ - Name: config.Name, - Namespace: config.Namespace, - }, - } - - if config.Owner != nil { - nascrd.OwnerReferences = []metav1.OwnerReference{*config.Owner} - } - - return nascrd -} diff --git a/api/example.com/resource/gpu/nas/v1alpha1/client/client.go b/api/example.com/resource/gpu/nas/v1alpha1/client/client.go deleted file mode 100644 index 77ab3367..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/client/client.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package client - -import ( - "context" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - nasclient "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1" -) - -type Client struct { - nas *nascrd.NodeAllocationState - client nasclient.NasV1alpha1Interface -} - -func New(nas *nascrd.NodeAllocationState, client nasclient.NasV1alpha1Interface) *Client { - return &Client{ - nas, - client, - } -} - -func (c *Client) GetOrCreate(ctx context.Context) error { - err := c.Get(ctx) - if err == nil { - return nil - } - if errors.IsNotFound(err) { - return c.Create(ctx) - } - return err -} - -func (c *Client) Create(ctx context.Context) error { - crd := c.nas.DeepCopy() - crd, err := c.client.NodeAllocationStates(c.nas.Namespace).Create(ctx, crd, metav1.CreateOptions{}) - if err != nil { - return err - } - *c.nas = *crd - return nil -} - -func (c *Client) Delete(ctx context.Context) error { - deletePolicy := metav1.DeletePropagationForeground - deleteOptions := metav1.DeleteOptions{PropagationPolicy: &deletePolicy} - err := c.client.NodeAllocationStates(c.nas.Namespace).Delete(ctx, c.nas.Name, deleteOptions) - if err != nil && !errors.IsNotFound(err) { - return err - } - return nil -} - -func (c *Client) Update(ctx context.Context, spec *nascrd.NodeAllocationStateSpec) error { - crd := c.nas.DeepCopy() - crd.Spec = *spec - crd, err := c.client.NodeAllocationStates(c.nas.Namespace).Update(ctx, crd, metav1.UpdateOptions{}) - if err != nil { - return err - } - *c.nas = *crd - return nil -} - -func (c *Client) UpdateStatus(ctx context.Context, status string) error { - crd := c.nas.DeepCopy() - crd.Status = status - crd, err := c.client.NodeAllocationStates(c.nas.Namespace).Update(ctx, crd, metav1.UpdateOptions{}) - if err != nil { - return err - } - *c.nas = *crd - return nil -} - -func (c *Client) Get(ctx context.Context) error { - crd, err := c.client.NodeAllocationStates(c.nas.Namespace).Get(ctx, c.nas.Name, metav1.GetOptions{}) - if err != nil { - return err - } - *c.nas = *crd - return nil -} diff --git a/api/example.com/resource/gpu/nas/v1alpha1/doc.go b/api/example.com/resource/gpu/nas/v1alpha1/doc.go deleted file mode 100644 index f93bc357..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// +k8s:deepcopy-gen=package -// +groupName=nas.gpu.resource.example.com - -package v1alpha1 diff --git a/api/example.com/resource/gpu/nas/v1alpha1/nas.go b/api/example.com/resource/gpu/nas/v1alpha1/nas.go deleted file mode 100644 index 436c2227..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/nas.go +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// AllocatableGpu represents an allocatable GPU on a node. -type AllocatableGpu struct { - UUID string `json:"uuid"` - ProductName string `json:"productName"` -} - -// AllocatableDevice represents an allocatable device on a node. -type AllocatableDevice struct { - Gpu *AllocatableGpu `json:"gpu,omitempty"` -} - -// Type returns the type of AllocatableDevice this represents. -func (d AllocatableDevice) Type() string { - if d.Gpu != nil { - return GpuDeviceType - } - return UnknownDeviceType -} - -// AllocatedGpu represents an allocated GPU. -type AllocatedGpu struct { - UUID string `json:"uuid,omitempty"` -} - -// AllocatedGpus represents a set of allocated GPUs. -type AllocatedGpus struct { - Devices []AllocatedGpu `json:"devices"` -} - -// AllocatedDevices represents a set of allocated devices. -type AllocatedDevices struct { - Gpu *AllocatedGpus `json:"gpu,omitempty"` -} - -// Type returns the type of AllocatedDevices this represents. -func (r AllocatedDevices) Type() string { - if r.Gpu != nil { - return GpuDeviceType - } - return UnknownDeviceType -} - -// PreparedGpu represents a prepared GPU on a node. -type PreparedGpu struct { - UUID string `json:"uuid"` -} - -// PreparedGpus represents a set of prepared GPUs on a node. -type PreparedGpus struct { - Devices []PreparedGpu `json:"devices"` -} - -// PreparedDevices represents a set of prepared devices on a node. -type PreparedDevices struct { - Gpu *PreparedGpus `json:"gpu,omitempty"` -} - -// Type returns the type of PreparedDevices this represents. -func (d PreparedDevices) Type() string { - if d.Gpu != nil { - return GpuDeviceType - } - return UnknownDeviceType -} - -// NodeAllocationStateSpec is the spec for the NodeAllocationState CRD. -type NodeAllocationStateSpec struct { - AllocatableDevices []AllocatableDevice `json:"allocatableDevices,omitempty"` - AllocatedClaims map[string]AllocatedDevices `json:"allocatedClaims,omitempty"` - PreparedClaims map[string]PreparedDevices `json:"preparedClaims,omitempty"` -} - -// +genclient -// +genclient:noStatus -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// +k8s:openapi-gen=true -// +kubebuilder:resource:scope=Namespaced -// +kubebuilder:resource:singular=nas - -// NodeAllocationState holds the state required for allocation on a node. -type NodeAllocationState struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec NodeAllocationStateSpec `json:"spec,omitempty"` - Status string `json:"status,omitempty"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// NodeAllocationStateList represents the "plural" of a NodeAllocationState CRD object. -type NodeAllocationStateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - - Items []NodeAllocationState `json:"items"` -} diff --git a/api/example.com/resource/gpu/nas/v1alpha1/register.go b/api/example.com/resource/gpu/nas/v1alpha1/register.go deleted file mode 100644 index 4d584c4d..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/register.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -var ( - // SchemeBuilder initializes a scheme builder. - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - // AddToScheme is a global function that registers this API group & version to a scheme. - AddToScheme = SchemeBuilder.AddToScheme -) - -// SchemeGroupVersion is group version used to register these objects. -var SchemeGroupVersion = schema.GroupVersion{ - Group: GroupName, - Version: Version, -} - -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &NodeAllocationState{}, - &NodeAllocationStateList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/api/example.com/resource/gpu/nas/v1alpha1/zz_generated.deepcopy.go b/api/example.com/resource/gpu/nas/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 5eaee00b..00000000 --- a/api/example.com/resource/gpu/nas/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,285 +0,0 @@ -//go:build !ignore_autogenerated - -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AllocatableDevice) DeepCopyInto(out *AllocatableDevice) { - *out = *in - if in.Gpu != nil { - in, out := &in.Gpu, &out.Gpu - *out = new(AllocatableGpu) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatableDevice. -func (in *AllocatableDevice) DeepCopy() *AllocatableDevice { - if in == nil { - return nil - } - out := new(AllocatableDevice) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AllocatableGpu) DeepCopyInto(out *AllocatableGpu) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatableGpu. -func (in *AllocatableGpu) DeepCopy() *AllocatableGpu { - if in == nil { - return nil - } - out := new(AllocatableGpu) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AllocatedDevices) DeepCopyInto(out *AllocatedDevices) { - *out = *in - if in.Gpu != nil { - in, out := &in.Gpu, &out.Gpu - *out = new(AllocatedGpus) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedDevices. -func (in *AllocatedDevices) DeepCopy() *AllocatedDevices { - if in == nil { - return nil - } - out := new(AllocatedDevices) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AllocatedGpu) DeepCopyInto(out *AllocatedGpu) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedGpu. -func (in *AllocatedGpu) DeepCopy() *AllocatedGpu { - if in == nil { - return nil - } - out := new(AllocatedGpu) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AllocatedGpus) DeepCopyInto(out *AllocatedGpus) { - *out = *in - if in.Devices != nil { - in, out := &in.Devices, &out.Devices - *out = make([]AllocatedGpu, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatedGpus. -func (in *AllocatedGpus) DeepCopy() *AllocatedGpus { - if in == nil { - return nil - } - out := new(AllocatedGpus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeAllocationState) DeepCopyInto(out *NodeAllocationState) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeAllocationState. -func (in *NodeAllocationState) DeepCopy() *NodeAllocationState { - if in == nil { - return nil - } - out := new(NodeAllocationState) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NodeAllocationState) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeAllocationStateConfig) DeepCopyInto(out *NodeAllocationStateConfig) { - *out = *in - if in.Owner != nil { - in, out := &in.Owner, &out.Owner - *out = new(v1.OwnerReference) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeAllocationStateConfig. -func (in *NodeAllocationStateConfig) DeepCopy() *NodeAllocationStateConfig { - if in == nil { - return nil - } - out := new(NodeAllocationStateConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeAllocationStateList) DeepCopyInto(out *NodeAllocationStateList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]NodeAllocationState, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeAllocationStateList. -func (in *NodeAllocationStateList) DeepCopy() *NodeAllocationStateList { - if in == nil { - return nil - } - out := new(NodeAllocationStateList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *NodeAllocationStateList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodeAllocationStateSpec) DeepCopyInto(out *NodeAllocationStateSpec) { - *out = *in - if in.AllocatableDevices != nil { - in, out := &in.AllocatableDevices, &out.AllocatableDevices - *out = make([]AllocatableDevice, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.AllocatedClaims != nil { - in, out := &in.AllocatedClaims, &out.AllocatedClaims - *out = make(map[string]AllocatedDevices, len(*in)) - for key, val := range *in { - (*out)[key] = *val.DeepCopy() - } - } - if in.PreparedClaims != nil { - in, out := &in.PreparedClaims, &out.PreparedClaims - *out = make(map[string]PreparedDevices, len(*in)) - for key, val := range *in { - (*out)[key] = *val.DeepCopy() - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeAllocationStateSpec. -func (in *NodeAllocationStateSpec) DeepCopy() *NodeAllocationStateSpec { - if in == nil { - return nil - } - out := new(NodeAllocationStateSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PreparedDevices) DeepCopyInto(out *PreparedDevices) { - *out = *in - if in.Gpu != nil { - in, out := &in.Gpu, &out.Gpu - *out = new(PreparedGpus) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedDevices. -func (in *PreparedDevices) DeepCopy() *PreparedDevices { - if in == nil { - return nil - } - out := new(PreparedDevices) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PreparedGpu) DeepCopyInto(out *PreparedGpu) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedGpu. -func (in *PreparedGpu) DeepCopy() *PreparedGpu { - if in == nil { - return nil - } - out := new(PreparedGpu) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PreparedGpus) DeepCopyInto(out *PreparedGpus) { - *out = *in - if in.Devices != nil { - in, out := &in.Devices, &out.Devices - *out = make([]PreparedGpu, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreparedGpus. -func (in *PreparedGpus) DeepCopy() *PreparedGpus { - if in == nil { - return nil - } - out := new(PreparedGpus) - in.DeepCopyInto(out) - return out -} diff --git a/api/example.com/resource/gpu/v1alpha1/api.go b/api/example.com/resource/gpu/v1alpha1/api.go index eae4b011..fa07563a 100644 --- a/api/example.com/resource/gpu/v1alpha1/api.go +++ b/api/example.com/resource/gpu/v1alpha1/api.go @@ -18,14 +18,15 @@ package v1alpha1 import ( "k8s.io/utils/ptr" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" ) const ( GroupName = "gpu.resource.example.com" Version = "v1alpha1" + GpuDeviceType = "gpu" + UnknownDeviceType = "unknown" + GpuClaimParametersKind = "GpuClaimParameters" ) @@ -33,7 +34,7 @@ func DefaultDeviceClassParametersSpec() *DeviceClassParametersSpec { return &DeviceClassParametersSpec{ DeviceSelector: []DeviceSelector{ { - Type: nascrd.GpuDeviceType, + Type: GpuDeviceType, Name: "*", }, }, diff --git a/cmd/dra-example-controller/allocations.go b/cmd/dra-example-controller/allocations.go deleted file mode 100644 index bebe0428..00000000 --- a/cmd/dra-example-controller/allocations.go +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "sync" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" -) - -type PerNodeAllocatedClaims struct { - sync.RWMutex - allocations map[string]map[string]nascrd.AllocatedDevices -} - -func NewPerNodeAllocatedClaims() *PerNodeAllocatedClaims { - return &PerNodeAllocatedClaims{ - allocations: make(map[string]map[string]nascrd.AllocatedDevices), - } -} - -func (p *PerNodeAllocatedClaims) Exists(claimUID, node string) bool { - p.RLock() - defer p.RUnlock() - - _, exists := p.allocations[claimUID] - if !exists { - return false - } - - _, exists = p.allocations[claimUID][node] - return exists -} - -func (p *PerNodeAllocatedClaims) Get(claimUID, node string) nascrd.AllocatedDevices { - p.RLock() - defer p.RUnlock() - - if !p.Exists(claimUID, node) { - return nascrd.AllocatedDevices{} - } - return p.allocations[claimUID][node] -} - -func (p *PerNodeAllocatedClaims) VisitNode(node string, visitor func(claimUID string, allocation nascrd.AllocatedDevices)) { - p.RLock() - for claimUID := range p.allocations { - if allocation, exists := p.allocations[claimUID][node]; exists { - p.RUnlock() - visitor(claimUID, allocation) - p.RLock() - } - } - p.RUnlock() -} - -func (p *PerNodeAllocatedClaims) Visit(visitor func(claimUID, node string, allocation nascrd.AllocatedDevices)) { - p.RLock() - for claimUID := range p.allocations { - for node, allocation := range p.allocations[claimUID] { - p.RUnlock() - visitor(claimUID, node, allocation) - p.RLock() - } - } - p.RUnlock() -} - -func (p *PerNodeAllocatedClaims) Set(claimUID, node string, devices nascrd.AllocatedDevices) { - p.Lock() - defer p.Unlock() - - _, exists := p.allocations[claimUID] - if !exists { - p.allocations[claimUID] = make(map[string]nascrd.AllocatedDevices) - } - - p.allocations[claimUID][node] = devices -} - -func (p *PerNodeAllocatedClaims) RemoveNode(claimUID, node string) { - p.Lock() - defer p.Unlock() - - _, exists := p.allocations[claimUID] - if !exists { - return - } - - delete(p.allocations[claimUID], node) -} - -func (p *PerNodeAllocatedClaims) Remove(claimUID string) { - p.Lock() - defer p.Unlock() - - delete(p.allocations, claimUID) -} diff --git a/cmd/dra-example-controller/claimparametersgen.go b/cmd/dra-example-controller/claimparametersgen.go index d769bdd9..db740342 100644 --- a/cmd/dra-example-controller/claimparametersgen.go +++ b/cmd/dra-example-controller/claimparametersgen.go @@ -37,6 +37,11 @@ import ( gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) +const ( + DriverAPIGroup = gpucrd.GroupName + DriverName = gpucrd.GroupName +) + func StartClaimParametersGenerator(ctx context.Context, config *Config) error { // Build a client set config csconfig, err := config.flags.kubeClientConfig.NewClientSetConfig() diff --git a/cmd/dra-example-controller/driver.go b/cmd/dra-example-controller/driver.go deleted file mode 100644 index b2f445cd..00000000 --- a/cmd/dra-example-controller/driver.go +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1alpha2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/dynamic-resource-allocation/controller" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - nasclient "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1/client" - gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" - clientset "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned" -) - -const ( - DriverAPIGroup = gpucrd.GroupName - DriverName = gpucrd.GroupName -) - -type OnSuccessCallback func() - -type driver struct { - lock *PerNodeMutex - namespace string - clientset clientset.Interface - gpu *gpudriver -} - -var _ controller.Driver = &driver{} - -func NewDriver(config *Config) *driver { - return &driver{ - lock: NewPerNodeMutex(), - namespace: config.namespace, - clientset: config.clientSets.Example, - gpu: NewGpuDriver(), - } -} - -func (d driver) GetClassParameters(ctx context.Context, class *resourcev1.ResourceClass) (interface{}, error) { - if class.ParametersRef == nil { - return gpucrd.DefaultDeviceClassParametersSpec(), nil - } - if class.ParametersRef.APIGroup != DriverAPIGroup { - return nil, fmt.Errorf("incorrect API group: %v", class.ParametersRef.APIGroup) - } - dc, err := d.clientset.GpuV1alpha1().DeviceClassParameters().Get(ctx, class.ParametersRef.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("error getting DeviceClassParameters called '%v': %v", class.ParametersRef.Name, err) - } - return &dc.Spec, nil -} - -func (d driver) GetClaimParameters(ctx context.Context, claim *resourcev1.ResourceClaim, class *resourcev1.ResourceClass, classParameters interface{}) (interface{}, error) { - if claim.Spec.ParametersRef == nil { - return gpucrd.DefaultGpuClaimParametersSpec(), nil - } - if claim.Spec.ParametersRef.APIGroup != DriverAPIGroup { - return nil, fmt.Errorf("incorrect API group: %v", claim.Spec.ParametersRef.APIGroup) - } - - switch claim.Spec.ParametersRef.Kind { - case gpucrd.GpuClaimParametersKind: - gc, err := d.clientset.GpuV1alpha1().GpuClaimParameters(claim.Namespace).Get(ctx, claim.Spec.ParametersRef.Name, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("error getting GpuClaimParameters called '%v' in namespace '%v': %v", claim.Spec.ParametersRef.Name, claim.Namespace, err) - } - err = d.gpu.ValidateClaimParameters(&gc.Spec) - if err != nil { - return nil, fmt.Errorf("error validating GpuClaimParameters called '%v' in namespace '%v': %v", claim.Spec.ParametersRef.Name, claim.Namespace, err) - } - return &gc.Spec, nil - default: - return nil, fmt.Errorf("unknown ResourceClaim.ParametersRef.Kind: %v", claim.Spec.ParametersRef.Kind) - } -} - -func (d driver) Allocate(ctx context.Context, cas []*controller.ClaimAllocation, selectedNode string) { - // In production version of the driver the common operations for every - // d.allocate looped call should be done prior this loop, and can be reused - // for every d.allocate() looped call. - // E.g.: selectedNode=="" check, client stup and CRD fetching. - for _, ca := range cas { - ca.Allocation, ca.Error = d.allocate(ctx, ca.Claim, ca.ClaimParameters, ca.Class, ca.ClassParameters, selectedNode) - } -} - -func (d driver) allocate(ctx context.Context, claim *resourcev1.ResourceClaim, claimParameters interface{}, class *resourcev1.ResourceClass, classParameters interface{}, selectedNode string) (*resourcev1.AllocationResult, error) { - - if selectedNode == "" { - return nil, fmt.Errorf("TODO: immediate allocations is not yet supported") - } - - d.lock.Get(selectedNode).Lock() - defer d.lock.Get(selectedNode).Unlock() - - crdconfig := &nascrd.NodeAllocationStateConfig{ - Name: selectedNode, - Namespace: d.namespace, - } - crd := nascrd.NewNodeAllocationState(crdconfig) - - client := nasclient.New(crd, d.clientset.NasV1alpha1()) - err := client.Get(ctx) - if err != nil { - return nil, fmt.Errorf("error retrieving node specific Gpu CRD: %v", err) - } - - if crd.Status != nascrd.NodeAllocationStateStatusReady { - return nil, fmt.Errorf("NodeAllocationStateStatus: %v", crd.Status) - } - - if crd.Spec.AllocatedClaims == nil { - crd.Spec.AllocatedClaims = make(map[string]nascrd.AllocatedDevices) - } - - if _, exists := crd.Spec.AllocatedClaims[string(claim.UID)]; exists { - return buildAllocationResult(selectedNode, true), nil - } - - var onSuccess OnSuccessCallback - classParams, _ := classParameters.(*gpucrd.DeviceClassParametersSpec) - - switch claimParams := claimParameters.(type) { - case *gpucrd.GpuClaimParametersSpec: - onSuccess, err = d.gpu.Allocate(crd, claim, claimParams, class, classParams, selectedNode) - default: - err = fmt.Errorf("unknown ResourceClaim.ParametersRef.Kind: %v", claim.Spec.ParametersRef.Kind) - } - if err != nil { - return nil, fmt.Errorf("unable to allocate devices on node '%v': %v", selectedNode, err) - } - - err = client.Update(ctx, &crd.Spec) - if err != nil { - return nil, fmt.Errorf("error updating NodeAllocationState CRD: %v", err) - } - - onSuccess() - - return buildAllocationResult(selectedNode, true), nil -} - -func (d driver) Deallocate(ctx context.Context, claim *resourcev1.ResourceClaim) error { - selectedNode := getSelectedNode(claim) - if selectedNode == "" { - return nil - } - - d.lock.Get(selectedNode).Lock() - defer d.lock.Get(selectedNode).Unlock() - - crdconfig := &nascrd.NodeAllocationStateConfig{ - Name: selectedNode, - Namespace: d.namespace, - } - crd := nascrd.NewNodeAllocationState(crdconfig) - - client := nasclient.New(crd, d.clientset.NasV1alpha1()) - err := client.Get(ctx) - if err != nil { - return fmt.Errorf("error retrieving node specific Gpu CRD: %v", err) - } - - if crd.Spec.AllocatedClaims == nil { - return nil - } - - if _, exists := crd.Spec.AllocatedClaims[string(claim.UID)]; !exists { - return nil - } - - devices := crd.Spec.AllocatedClaims[string(claim.UID)] - switch devices.Type() { - case nascrd.GpuDeviceType: - err = d.gpu.Deallocate(crd, claim) - default: - err = fmt.Errorf("unknown AllocatedDevices.Type(): %v", devices.Type()) - } - if err != nil { - return fmt.Errorf("unable to deallocate devices '%v': %v", devices, err) - } - - delete(crd.Spec.AllocatedClaims, string(claim.UID)) - - err = client.Update(ctx, &crd.Spec) - if err != nil { - return fmt.Errorf("error updating NodeAllocationState CRD: %v", err) - } - - return nil -} - -func (d driver) UnsuitableNodes(ctx context.Context, pod *corev1.Pod, cas []*controller.ClaimAllocation, potentialNodes []string) error { - for _, node := range potentialNodes { - err := d.unsuitableNode(ctx, pod, cas, node) - if err != nil { - return fmt.Errorf("error processing node '%v': %v", node, err) - } - } - - for _, ca := range cas { - ca.UnsuitableNodes = unique(ca.UnsuitableNodes) - } - - return nil -} - -func (d driver) unsuitableNode(ctx context.Context, pod *corev1.Pod, allcas []*controller.ClaimAllocation, potentialNode string) error { - d.lock.Get(potentialNode).Lock() - defer d.lock.Get(potentialNode).Unlock() - - crdconfig := &nascrd.NodeAllocationStateConfig{ - Name: potentialNode, - Namespace: d.namespace, - } - crd := nascrd.NewNodeAllocationState(crdconfig) - - client := nasclient.New(crd, d.clientset.NasV1alpha1()) - err := client.Get(ctx) - if err != nil { - for _, ca := range allcas { - ca.UnsuitableNodes = append(ca.UnsuitableNodes, potentialNode) - } - return nil - } - - if crd.Status != nascrd.NodeAllocationStateStatusReady { - for _, ca := range allcas { - ca.UnsuitableNodes = append(ca.UnsuitableNodes, potentialNode) - } - return nil - } - - if crd.Spec.AllocatedClaims == nil { - crd.Spec.AllocatedClaims = make(map[string]nascrd.AllocatedDevices) - } - - perKindCas := make(map[string][]*controller.ClaimAllocation) - for _, ca := range allcas { - switch ca.ClaimParameters.(type) { - case *gpucrd.GpuClaimParametersSpec: - perKindCas[gpucrd.GpuClaimParametersKind] = append(perKindCas[gpucrd.GpuClaimParametersKind], ca) - default: - return fmt.Errorf("unknown ResourceClaimParameters kind: %T", ca.ClaimParameters) - } - } - for _, kind := range []string{gpucrd.GpuClaimParametersKind} { - var err error - switch kind { - case gpucrd.GpuClaimParametersKind: - err = d.gpu.UnsuitableNode(crd, pod, perKindCas[kind], allcas, potentialNode) - default: - err = fmt.Errorf("unknown ResourceClaimParameters kind: %+v", kind) - } - if err != nil { - return fmt.Errorf("error processing '%v': %v", kind, err) - } - } - - return nil -} - -func buildAllocationResult(selectedNode string, shareable bool) *resourcev1.AllocationResult { - nodeSelector := &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchFields: []corev1.NodeSelectorRequirement{ - { - Key: "metadata.name", - Operator: "In", - Values: []string{selectedNode}, - }, - }, - }, - }, - } - allocation := &resourcev1.AllocationResult{ - AvailableOnNodes: nodeSelector, - Shareable: shareable, - } - return allocation -} - -func getSelectedNode(claim *resourcev1.ResourceClaim) string { - if claim.Status.Allocation == nil { - return "" - } - if claim.Status.Allocation.AvailableOnNodes == nil { - return "" - } - return claim.Status.Allocation.AvailableOnNodes.NodeSelectorTerms[0].MatchFields[0].Values[0] -} - -func unique(s []string) []string { - set := make(map[string]struct{}) - var news []string - for _, str := range s { - if _, exists := set[str]; !exists { - set[str] = struct{}{} - news = append(news, str) - } - } - return news -} diff --git a/cmd/dra-example-controller/gpu.go b/cmd/dra-example-controller/gpu.go deleted file mode 100644 index 308779b2..00000000 --- a/cmd/dra-example-controller/gpu.go +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - resourcev1 "k8s.io/api/resource/v1alpha2" - "k8s.io/dynamic-resource-allocation/controller" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" -) - -type gpudriver struct { - PendingAllocatedClaims *PerNodeAllocatedClaims -} - -func NewGpuDriver() *gpudriver { - return &gpudriver{ - PendingAllocatedClaims: NewPerNodeAllocatedClaims(), - } -} - -func (g *gpudriver) ValidateClaimParameters(claimParams *gpucrd.GpuClaimParametersSpec) error { - if *claimParams.Count < 1 { - return fmt.Errorf("invalid number of GPUs requested: %v", claimParams.Count) - } - return nil -} - -func (g *gpudriver) Allocate(crd *nascrd.NodeAllocationState, claim *resourcev1.ResourceClaim, claimParams *gpucrd.GpuClaimParametersSpec, class *resourcev1.ResourceClass, classParams *gpucrd.DeviceClassParametersSpec, selectedNode string) (OnSuccessCallback, error) { - claimUID := string(claim.UID) - - if !g.PendingAllocatedClaims.Exists(claimUID, selectedNode) { - return nil, fmt.Errorf("no allocations generated for claim '%v' on node '%v' yet", claim.UID, selectedNode) - } - - crd.Spec.AllocatedClaims[claimUID] = g.PendingAllocatedClaims.Get(claimUID, selectedNode) - onSuccess := func() { - g.PendingAllocatedClaims.Remove(claimUID) - } - - return onSuccess, nil -} - -func (g *gpudriver) Deallocate(crd *nascrd.NodeAllocationState, claim *resourcev1.ResourceClaim) error { - g.PendingAllocatedClaims.Remove(string(claim.UID)) - return nil -} - -func (g *gpudriver) UnsuitableNode(crd *nascrd.NodeAllocationState, pod *corev1.Pod, gpucas []*controller.ClaimAllocation, allcas []*controller.ClaimAllocation, potentialNode string) error { - g.PendingAllocatedClaims.VisitNode(potentialNode, func(claimUID string, allocation nascrd.AllocatedDevices) { - if _, exists := crd.Spec.AllocatedClaims[claimUID]; exists { - g.PendingAllocatedClaims.Remove(claimUID) - } else { - crd.Spec.AllocatedClaims[claimUID] = allocation - } - }) - - allocated := g.allocate(crd, pod, gpucas, allcas, potentialNode) - for _, ca := range gpucas { - claimUID := string(ca.Claim.UID) - claimParams, _ := ca.ClaimParameters.(*gpucrd.GpuClaimParametersSpec) - - if *claimParams.Count != len(allocated[claimUID]) { - for _, ca := range allcas { - ca.UnsuitableNodes = append(ca.UnsuitableNodes, potentialNode) - } - return nil - } - - var devices []nascrd.AllocatedGpu - for _, gpu := range allocated[claimUID] { - device := nascrd.AllocatedGpu{ - UUID: gpu, - } - devices = append(devices, device) - } - - allocatedDevices := nascrd.AllocatedDevices{ - Gpu: &nascrd.AllocatedGpus{ - Devices: devices, - }, - } - - g.PendingAllocatedClaims.Set(claimUID, potentialNode, allocatedDevices) - } - - return nil -} - -func (g *gpudriver) allocate(crd *nascrd.NodeAllocationState, pod *corev1.Pod, gpucas []*controller.ClaimAllocation, allcas []*controller.ClaimAllocation, node string) map[string][]string { - available := make(map[string]*nascrd.AllocatableGpu) - - for _, device := range crd.Spec.AllocatableDevices { - switch device.Type() { - case nascrd.GpuDeviceType: - available[device.Gpu.UUID] = device.Gpu - default: - // skip other devices - } - } - - for _, allocation := range crd.Spec.AllocatedClaims { - switch allocation.Type() { - case nascrd.GpuDeviceType: - for _, device := range allocation.Gpu.Devices { - delete(available, device.UUID) - } - default: - // skip other devices - } - } - - allocated := make(map[string][]string) - for _, ca := range gpucas { - claimUID := string(ca.Claim.UID) - if _, exists := crd.Spec.AllocatedClaims[claimUID]; exists { - devices := crd.Spec.AllocatedClaims[claimUID].Gpu.Devices - for _, device := range devices { - allocated[claimUID] = append(allocated[claimUID], device.UUID) - } - continue - } - - claimParams, _ := ca.ClaimParameters.(*gpucrd.GpuClaimParametersSpec) - var devices []string - for i := 0; i < *claimParams.Count; i++ { - for _, device := range available { - devices = append(devices, device.UUID) - delete(available, device.UUID) - break - } - } - allocated[claimUID] = devices - } - - return allocated -} diff --git a/cmd/dra-example-controller/main.go b/cmd/dra-example-controller/main.go index ddd9d949..fcc8e26a 100644 --- a/cmd/dra-example-controller/main.go +++ b/cmd/dra-example-controller/main.go @@ -29,9 +29,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" - "k8s.io/client-go/informers" "k8s.io/component-base/metrics/legacyregistry" - "k8s.io/dynamic-resource-allocation/controller" "k8s.io/klog/v2" _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration @@ -44,7 +42,7 @@ import ( type Flags struct { kubeClientConfig flags.KubeClientConfig loggingConfig *flags.LoggingConfig - nasConfig flags.NasConfig + crdConfig flags.CRDConfig workers int @@ -106,8 +104,8 @@ func newApp() *cli.App { cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...) cliFlags = append(cliFlags, flags.loggingConfig.Flags()...) - flags.nasConfig.HideNodeName = true - cliFlags = append(cliFlags, flags.nasConfig.Flags()...) + flags.crdConfig.HideNodeName = true + cliFlags = append(cliFlags, flags.crdConfig.Flags()...) app := &cli.App{ Name: "dra-example-controller", @@ -133,7 +131,7 @@ func newApp() *cli.App { config := &Config{ mux: mux, flags: flags, - namespace: flags.nasConfig.Namespace, + namespace: flags.crdConfig.Namespace, clientSets: clientSets, } @@ -149,11 +147,7 @@ func newApp() *cli.App { return fmt.Errorf("start claim parameters generator: %w", err) } - err = StartController(ctx, config) - if err != nil { - return fmt.Errorf("start controller: %v", err) - } - + <-ctx.Done() return nil }, } @@ -211,12 +205,3 @@ func SetupHTTPEndpoint(ctx context.Context, config *Config) error { return nil } - -func StartController(ctx context.Context, config *Config) error { - driver := NewDriver(config) - informerFactory := informers.NewSharedInformerFactory(config.clientSets.Core, 0 /* resync period */) - ctrl := controller.New(ctx, DriverAPIGroup, driver, config.clientSets.Core, informerFactory) - informerFactory.Start(ctx.Done()) - ctrl.Run(config.flags.workers) - return nil -} diff --git a/cmd/dra-example-controller/mutex.go b/cmd/dra-example-controller/mutex.go deleted file mode 100644 index 50a4597e..00000000 --- a/cmd/dra-example-controller/mutex.go +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "sync" -) - -type PerNodeMutex struct { - sync.Mutex - submutex map[string]*sync.Mutex -} - -func NewPerNodeMutex() *PerNodeMutex { - return &PerNodeMutex{ - submutex: make(map[string]*sync.Mutex), - } -} - -func (pnm *PerNodeMutex) Get(node string) *sync.Mutex { - pnm.Mutex.Lock() - defer pnm.Mutex.Unlock() - if pnm.submutex[node] == nil { - pnm.submutex[node] = &sync.Mutex{} - } - return pnm.submutex[node] -} diff --git a/cmd/dra-example-kubeletplugin/cdi.go b/cmd/dra-example-kubeletplugin/cdi.go index 9f915bf1..fe787c3b 100644 --- a/cmd/dra-example-kubeletplugin/cdi.go +++ b/cmd/dra-example-kubeletplugin/cdi.go @@ -23,7 +23,7 @@ import ( cdiapi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" cdispec "github.com/container-orchestrated-devices/container-device-interface/specs-go" - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" + gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) const ( @@ -99,7 +99,7 @@ func (cdi *CDIHandler) CreateClaimSpecFile(claimUID string, devices *PreparedDev gpuIdx := 0 switch devices.Type() { - case nascrd.GpuDeviceType: + case gpucrd.GpuDeviceType: for _, device := range devices.Gpu.Devices { cdiDevice := cdispec.Device{ Name: device.uuid, @@ -136,7 +136,7 @@ func (cdi *CDIHandler) GetClaimDevices(claimUID string, devices *PreparedDevices } switch devices.Type() { - case nascrd.GpuDeviceType: + case gpucrd.GpuDeviceType: for _, device := range devices.Gpu.Devices { cdiDevice := cdiapi.QualifiedName(cdiVendor, cdiClass, device.uuid) cdiDevices = append(cdiDevices, cdiDevice) diff --git a/cmd/dra-example-kubeletplugin/driver.go b/cmd/dra-example-kubeletplugin/driver.go index 124be3d7..5888be00 100644 --- a/cmd/dra-example-kubeletplugin/driver.go +++ b/cmd/dra-example-kubeletplugin/driver.go @@ -21,82 +21,33 @@ import ( "fmt" resourceapi "k8s.io/api/resource/v1alpha2" - "k8s.io/client-go/util/retry" "k8s.io/klog/v2" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1alpha3" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - nasclient "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1/client" ) var _ drapbv1.NodeServer = &driver{} type driver struct { - doneCh chan struct{} - nascr *nascrd.NodeAllocationState - nasclient *nasclient.Client - state *DeviceState + doneCh chan struct{} + state *DeviceState } func NewDriver(ctx context.Context, config *Config) (*driver, error) { - var d *driver - client := nasclient.New(config.nascr, config.exampleclient.NasV1alpha1()) - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := client.GetOrCreate(ctx) - if err != nil { - return err - } - - err = client.UpdateStatus(ctx, nascrd.NodeAllocationStateStatusNotReady) - if err != nil { - return err - } - - state, err := NewDeviceState(config) - if err != nil { - return err - } - - updatedSpec, err := state.GetUpdatedSpec(&config.nascr.Spec) - if err != nil { - return fmt.Errorf("error getting updated CR spec: %v", err) - } - - err = client.Update(ctx, updatedSpec) - if err != nil { - return err - } - - err = client.UpdateStatus(ctx, nascrd.NodeAllocationStateStatusReady) - if err != nil { - return err - } - - d = &driver{ - nascr: config.nascr, - nasclient: client, - state: state, - } - - return nil - }) + state, err := NewDeviceState(config) if err != nil { return nil, err } + d := &driver{ + state: state, + } + return d, nil } func (d *driver) Shutdown(ctx context.Context) error { - defer close(d.doneCh) - - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := d.nasclient.Get(ctx) - if err != nil { - return err - } - return d.nasclient.UpdateStatus(ctx, nascrd.NodeAllocationStateStatusNotReady) - }) + close(d.doneCh) + return nil } func (d *driver) NodeListAndWatchResources(req *drapbv1.NodeListAndWatchResourcesRequest, stream drapbv1.Node_NodeListAndWatchResourcesServer) error { @@ -134,15 +85,20 @@ func (d *driver) NodePrepareResources(ctx context.Context, req *drapbv1.NodePrep } func (d *driver) nodePrepareResource(ctx context.Context, claim *drapbv1.Claim) *drapbv1.NodePrepareResourceResponse { - if len(claim.StructuredResourceHandle) > 0 { - if err := d.allocateDevices(ctx, claim); err != nil { - return &drapbv1.NodePrepareResourceResponse{ - Error: fmt.Sprintf("error allocating devices for claim %v: %v", claim.Uid, err), - } + if len(claim.StructuredResourceHandle) == 0 { + return &drapbv1.NodePrepareResourceResponse{ + Error: "driver only supports structured parameters", + } + } + + allocated, err := d.getAllocatedDevices(ctx, claim) + if err != nil { + return &drapbv1.NodePrepareResourceResponse{ + Error: fmt.Sprintf("error allocating devices for claim %v: %v", claim.Uid, err), } } - prepared, err := d.prepare(ctx, claim.Uid) + prepared, err := d.state.Prepare(claim.Uid, allocated) if err != nil { return &drapbv1.NodePrepareResourceResponse{ Error: fmt.Sprintf("error preparing devices for claim %v: %v", claim.Uid, err), @@ -165,15 +121,13 @@ func (d *driver) NodeUnprepareResources(ctx context.Context, req *drapbv1.NodeUn } func (d *driver) nodeUnprepareResource(ctx context.Context, claim *drapbv1.Claim) *drapbv1.NodeUnprepareResourceResponse { - if len(claim.StructuredResourceHandle) > 0 { - if err := d.deallocateDevices(ctx, claim); err != nil { - return &drapbv1.NodeUnprepareResourceResponse{ - Error: fmt.Sprintf("error deallocating devices for claim %v: %v", claim.Uid, err), - } + if len(claim.StructuredResourceHandle) == 0 { + return &drapbv1.NodeUnprepareResourceResponse{ + Error: "driver only supports structured parameters", } } - if err := d.unprepare(ctx, claim.Uid); err != nil { + if err := d.state.Unprepare(claim.Uid); err != nil { return &drapbv1.NodeUnprepareResourceResponse{ Error: fmt.Sprintf("error unpreparing devices for claim %v: %v", claim.Uid, err), } @@ -182,123 +136,16 @@ func (d *driver) nodeUnprepareResource(ctx context.Context, claim *drapbv1.Claim return &drapbv1.NodeUnprepareResourceResponse{} } -func (d *driver) prepare(ctx context.Context, claimUID string) ([]string, error) { - var err error - var prepared []string - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - err = d.nasclient.Get(ctx) - if err != nil { - return err - } - - prepared, err = d.state.Prepare(claimUID, d.nascr.Spec.AllocatedClaims[claimUID]) - if err != nil { - return err - } - - updatedSpec, err := d.state.GetUpdatedSpec(&d.nascr.Spec) - if err != nil { - return fmt.Errorf("error getting updated CR spec: %v", err) - } - - err = d.nasclient.Update(ctx, updatedSpec) - if err != nil { - return err - } - - return nil - }) - if err != nil { - return nil, err - } - return prepared, nil -} - -func (d *driver) unprepare(ctx context.Context, claimUID string) error { - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := d.nasclient.Get(ctx) - if err != nil { - return err - } - - err = d.state.Unprepare(claimUID) - if err != nil { - return err - } - - updatedSpec, err := d.state.GetUpdatedSpec(&d.nascr.Spec) - if err != nil { - return fmt.Errorf("error getting updated CR spec: %v", err) - } - - err = d.nasclient.Update(ctx, updatedSpec) - if err != nil { - return err - } - - return nil - }) - if err != nil { - return err - } - return nil -} - -func (d *driver) allocateDevices(ctx context.Context, claim *drapbv1.Claim) error { - allocated := nascrd.AllocatedDevices{ - Gpu: &nascrd.AllocatedGpus{}, +func (d *driver) getAllocatedDevices(ctx context.Context, claim *drapbv1.Claim) (AllocatedDevices, error) { + allocated := AllocatedDevices{ + Gpu: &AllocatedGpus{}, } for _, r := range claim.StructuredResourceHandle[0].Results { name := r.AllocationResultModel.NamedResources.Name - gpu := nascrd.AllocatedGpu{ - UUID: fmt.Sprintf("GPU-%s", name[4:]), - } + gpu := fmt.Sprintf("GPU-%s", name[4:]) allocated.Gpu.Devices = append(allocated.Gpu.Devices, gpu) } - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := d.nasclient.Get(ctx) - if err != nil { - return err - } - - if len(d.nascr.Spec.AllocatedClaims) == 0 { - d.nascr.Spec.AllocatedClaims = make(map[string]nascrd.AllocatedDevices) - } - d.nascr.Spec.AllocatedClaims[claim.Uid] = allocated - - err = d.nasclient.Update(ctx, &d.nascr.Spec) - if err != nil { - return err - } - - return nil - }) - if err != nil { - return err - } - return nil -} - -func (d *driver) deallocateDevices(ctx context.Context, claim *drapbv1.Claim) error { - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := d.nasclient.Get(ctx) - if err != nil { - return err - } - - delete(d.nascr.Spec.AllocatedClaims, claim.Uid) - - err = d.nasclient.Update(ctx, &d.nascr.Spec) - if err != nil { - return err - } - - return nil - }) - if err != nil { - return err - } - return nil + return allocated, nil } diff --git a/cmd/dra-example-kubeletplugin/main.go b/cmd/dra-example-kubeletplugin/main.go index cd50af07..cfd3eb9a 100644 --- a/cmd/dra-example-kubeletplugin/main.go +++ b/cmd/dra-example-kubeletplugin/main.go @@ -28,7 +28,6 @@ import ( plugin "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/klog/v2" - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" exampleclientset "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned" "sigs.k8s.io/dra-example-driver/pkg/flags" @@ -44,7 +43,7 @@ const ( type Flags struct { kubeClientConfig flags.KubeClientConfig - nasConfig flags.NasConfig + crdConfig flags.CRDConfig loggingConfig *flags.LoggingConfig cdiRoot string @@ -52,7 +51,6 @@ type Flags struct { type Config struct { flags *Flags - nascr *nascrd.NodeAllocationState exampleclient exampleclientset.Interface } @@ -77,7 +75,7 @@ func newApp() *cli.App { }, } cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...) - cliFlags = append(cliFlags, flags.nasConfig.Flags()...) + cliFlags = append(cliFlags, flags.crdConfig.Flags()...) cliFlags = append(cliFlags, flags.loggingConfig.Flags()...) app := &cli.App{ @@ -99,14 +97,8 @@ func newApp() *cli.App { return fmt.Errorf("create client: %v", err) } - nascr, err := flags.nasConfig.NewNodeAllocationState(ctx, clientSets.Core) - if err != nil { - return fmt.Errorf("create NodeAllocationState CR: %v", err) - } - config := &Config{ flags: flags, - nascr: nascr, exampleclient: clientSets.Example, } diff --git a/cmd/dra-example-kubeletplugin/state.go b/cmd/dra-example-kubeletplugin/state.go index d754a3ba..c4d96b76 100644 --- a/cmd/dra-example-kubeletplugin/state.go +++ b/cmd/dra-example-kubeletplugin/state.go @@ -23,7 +23,7 @@ import ( resourceapi "k8s.io/api/resource/v1alpha2" - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" + gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) type AllocatableDevices map[string]*AllocatableDeviceInfo @@ -34,6 +34,14 @@ type GpuInfo struct { model string } +type AllocatedGpus struct { + Devices []string +} + +type AllocatedDevices struct { + Gpu *AllocatedGpus +} + type PreparedGpus struct { Devices []*GpuInfo } @@ -42,11 +50,18 @@ type PreparedDevices struct { Gpu *PreparedGpus } +func (d AllocatedDevices) Type() string { + if d.Gpu != nil { + return gpucrd.GpuDeviceType + } + return gpucrd.UnknownDeviceType +} + func (d PreparedDevices) Type() string { if d.Gpu != nil { - return nascrd.GpuDeviceType + return gpucrd.GpuDeviceType } - return nascrd.UnknownDeviceType + return gpucrd.UnknownDeviceType } type AllocatableDeviceInfo struct { @@ -82,15 +97,10 @@ func NewDeviceState(config *Config) (*DeviceState, error) { prepared: make(PreparedClaims), } - err = state.syncPreparedDevicesFromCRDSpec(&config.nascr.Spec) - if err != nil { - return nil, fmt.Errorf("unable to sync prepared devices from CRD: %v", err) - } - return state, nil } -func (s *DeviceState) Prepare(claimUID string, allocation nascrd.AllocatedDevices) ([]string, error) { +func (s *DeviceState) Prepare(claimUID string, allocation AllocatedDevices) ([]string, error) { s.Lock() defer s.Unlock() @@ -106,13 +116,13 @@ func (s *DeviceState) Prepare(claimUID string, allocation nascrd.AllocatedDevice var err error switch allocation.Type() { - case nascrd.GpuDeviceType: + case gpucrd.GpuDeviceType: prepared.Gpu, err = s.prepareGpus(claimUID, allocation.Gpu) default: err = fmt.Errorf("unknown device type: %v", allocation.Type()) } if err != nil { - return nil, fmt.Errorf("allocation failed: %v", err) + return nil, fmt.Errorf("praparation failed: %v", err) } err = s.cdi.CreateClaimSpecFile(claimUID, prepared) @@ -126,6 +136,7 @@ func (s *DeviceState) Prepare(claimUID string, allocation nascrd.AllocatedDevice if err != nil { return nil, fmt.Errorf("unable to get CDI devices names: %v", err) } + return cdiDevices, nil } @@ -138,7 +149,7 @@ func (s *DeviceState) Unprepare(claimUID string) error { } switch s.prepared[claimUID].Type() { - case nascrd.GpuDeviceType: + case gpucrd.GpuDeviceType: err := s.unprepareGpus(claimUID, s.prepared[claimUID]) if err != nil { return fmt.Errorf("unprepare failed: %v", err) @@ -157,32 +168,14 @@ func (s *DeviceState) Unprepare(claimUID string) error { return nil } -func (s *DeviceState) GetUpdatedSpec(inspec *nascrd.NodeAllocationStateSpec) (*nascrd.NodeAllocationStateSpec, error) { - s.Lock() - defer s.Unlock() - - outspec := inspec.DeepCopy() - err := s.syncAllocatableDevicesToCRDSpec(outspec) - if err != nil { - return nil, fmt.Errorf("synching allocatable devices to CR spec: %v", err) - } - - err = s.syncPreparedDevicesToCRDSpec(outspec) - if err != nil { - return nil, fmt.Errorf("synching prepared devices to CR spec: %v", err) - } - - return outspec, nil -} - -func (s *DeviceState) prepareGpus(claimUID string, allocated *nascrd.AllocatedGpus) (*PreparedGpus, error) { +func (s *DeviceState) prepareGpus(claimUID string, allocated *AllocatedGpus) (*PreparedGpus, error) { prepared := &PreparedGpus{} for _, device := range allocated.Devices { - gpuInfo := s.allocatable[device.UUID].GpuInfo + gpuInfo := s.allocatable[device].GpuInfo - if _, exists := s.allocatable[device.UUID]; !exists { - return nil, fmt.Errorf("requested GPU does not exist: %v", device.UUID) + if _, exists := s.allocatable[device]; !exists { + return nil, fmt.Errorf("requested GPU is not allocatable: %v", device) } prepared.Devices = append(prepared.Devices, gpuInfo) @@ -210,69 +203,3 @@ func (s *DeviceState) getResourceModelFromAllocatableDevices() resourceapi.Resou return model } - -func (s *DeviceState) syncAllocatableDevicesToCRDSpec(spec *nascrd.NodeAllocationStateSpec) error { - gpus := make(map[string]nascrd.AllocatableDevice) - for _, device := range s.allocatable { - gpus[device.uuid] = nascrd.AllocatableDevice{ - Gpu: &nascrd.AllocatableGpu{ - UUID: device.uuid, - ProductName: device.model, - }, - } - } - - var allocatable []nascrd.AllocatableDevice - for _, device := range gpus { - allocatable = append(allocatable, device) - } - - spec.AllocatableDevices = allocatable - - return nil -} - -func (s *DeviceState) syncPreparedDevicesFromCRDSpec(spec *nascrd.NodeAllocationStateSpec) error { - gpus := s.allocatable - - prepared := make(PreparedClaims) - for claim, devices := range spec.PreparedClaims { - switch devices.Type() { - case nascrd.GpuDeviceType: - prepared[claim] = &PreparedDevices{Gpu: &PreparedGpus{}} - for _, d := range devices.Gpu.Devices { - prepared[claim].Gpu.Devices = append(prepared[claim].Gpu.Devices, gpus[d.UUID].GpuInfo) - } - default: - return fmt.Errorf("unknown device type: %v", devices.Type()) - } - } - - s.prepared = prepared - - return nil -} - -func (s *DeviceState) syncPreparedDevicesToCRDSpec(spec *nascrd.NodeAllocationStateSpec) error { - outcas := make(map[string]nascrd.PreparedDevices) - for claim, devices := range s.prepared { - var prepared nascrd.PreparedDevices - switch devices.Type() { - case nascrd.GpuDeviceType: - prepared.Gpu = &nascrd.PreparedGpus{} - for _, device := range devices.Gpu.Devices { - outdevice := nascrd.PreparedGpu{ - UUID: device.uuid, - } - prepared.Gpu.Devices = append(prepared.Gpu.Devices, outdevice) - } - default: - return fmt.Errorf("unknown device type: %v", devices.Type()) - } - outcas[claim] = prepared - } - - spec.PreparedClaims = outcas - - return nil -} diff --git a/cmd/set-nas-status/main.go b/cmd/set-nas-status/main.go deleted file mode 100644 index 69f5e45f..00000000 --- a/cmd/set-nas-status/main.go +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/urfave/cli/v2" - - "k8s.io/client-go/util/retry" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - nasclient "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1/client" - "sigs.k8s.io/dra-example-driver/pkg/flags" -) - -func main() { - if err := newApp().Run(os.Args); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func newApp() *cli.App { - var ( - status string - - kubeClientConfig flags.KubeClientConfig - loggingConfig *flags.LoggingConfig - nasConfig flags.NasConfig - ) - - loggingConfig = flags.NewLoggingConfig() - - flags := []cli.Flag{ - &cli.StringFlag{ - Name: "status", - Usage: "The status to set [Ready | NotReady].", - Required: true, - Action: func(_ *cli.Context, value string) error { - switch strings.ToLower(value) { - case strings.ToLower(nascrd.NodeAllocationStateStatusReady): - status = nascrd.NodeAllocationStateStatusReady - case strings.ToLower(nascrd.NodeAllocationStateStatusNotReady): - status = nascrd.NodeAllocationStateStatusNotReady - default: - return fmt.Errorf("unknown status: %s", value) - } - return nil - }, - EnvVars: []string{"STATUS"}, - }, - } - - flags = append(flags, kubeClientConfig.Flags()...) - flags = append(flags, loggingConfig.Flags()...) - flags = append(flags, nasConfig.Flags()...) - - app := &cli.App{ - Name: "set-nas-status", - Usage: "set-nas-status sets the status of the NodeAllocationState CRD managed by the DRA driver for GPUs.", - ArgsUsage: " ", - HideHelpCommand: true, - Flags: flags, - Before: func(ctx *cli.Context) error { - if ctx.Args().Len() > 0 { - return fmt.Errorf("arguments not supported: %v", ctx.Args().Slice()) - } - return loggingConfig.Apply() - }, - Action: func(c *cli.Context) error { - ctx := c.Context - clientSets, err := kubeClientConfig.NewClientSets() - if err != nil { - return fmt.Errorf("create client: %v", err) - } - - nascr, err := nasConfig.NewNodeAllocationState(ctx, clientSets.Core) - if err != nil { - return fmt.Errorf("create NodeAllocationState CR: %v", err) - } - - client := nasclient.New(nascr, clientSets.Example.NasV1alpha1()) - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := client.GetOrCreate(ctx) - if err != nil { - return err - } - return client.UpdateStatus(ctx, status) - }); err != nil { - return err - } - return nil - }, - } - - return app -} diff --git a/common.mk b/common.mk index bfdc36b3..50586d8c 100644 --- a/common.mk +++ b/common.mk @@ -21,7 +21,7 @@ VERSION ?= v0.1.0 vVERSION := v$(VERSION:v%=%) VENDOR := example.com -APIS := gpu/nas/v1alpha1 gpu/v1alpha1 +APIS := gpu/v1alpha1 PLURAL_EXCEPTIONS = DeviceClassParameters:DeviceClassParameters PLURAL_EXCEPTIONS += GpuClaimParameters:GpuClaimParameters diff --git a/deployments/container/Dockerfile b/deployments/container/Dockerfile index a32d36e7..bb0b597b 100644 --- a/deployments/container/Dockerfile +++ b/deployments/container/Dockerfile @@ -36,4 +36,3 @@ LABEL description="See summary" COPY --from=build /artifacts/dra-example-controller /usr/bin/dra-example-controller COPY --from=build /artifacts/dra-example-kubeletplugin /usr/bin/dra-example-kubeletplugin -COPY --from=build /artifacts/set-nas-status /usr/bin/set-nas-status diff --git a/deployments/helm/dra-example-driver/crds/nas.gpu.resource.example.com_nodeallocationstates.yaml b/deployments/helm/dra-example-driver/crds/nas.gpu.resource.example.com_nodeallocationstates.yaml deleted file mode 100644 index 2d10b456..00000000 --- a/deployments/helm/dra-example-driver/crds/nas.gpu.resource.example.com_nodeallocationstates.yaml +++ /dev/null @@ -1,113 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.14.0 - name: nodeallocationstates.nas.gpu.resource.example.com -spec: - group: nas.gpu.resource.example.com - names: - kind: NodeAllocationState - listKind: NodeAllocationStateList - plural: nodeallocationstates - singular: nas - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeAllocationState holds the state required for allocation on - a node. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeAllocationStateSpec is the spec for the NodeAllocationState - CRD. - properties: - allocatableDevices: - items: - description: AllocatableDevice represents an allocatable device - on a node. - properties: - gpu: - description: AllocatableGpu represents an allocatable GPU on - a node. - properties: - productName: - type: string - uuid: - type: string - required: - - productName - - uuid - type: object - type: object - type: array - allocatedClaims: - additionalProperties: - description: AllocatedDevices represents a set of allocated devices. - properties: - gpu: - description: AllocatedGpus represents a set of allocated GPUs. - properties: - devices: - items: - description: AllocatedGpu represents an allocated GPU. - properties: - uuid: - type: string - type: object - type: array - required: - - devices - type: object - type: object - type: object - preparedClaims: - additionalProperties: - description: PreparedDevices represents a set of prepared devices - on a node. - properties: - gpu: - description: PreparedGpus represents a set of prepared GPUs - on a node. - properties: - devices: - items: - description: PreparedGpu represents a prepared GPU on - a node. - properties: - uuid: - type: string - required: - - uuid - type: object - type: array - required: - - devices - type: object - type: object - type: object - type: object - status: - type: string - type: object - served: true - storage: true diff --git a/deployments/helm/dra-example-driver/templates/clusterrole.yaml b/deployments/helm/dra-example-driver/templates/clusterrole.yaml index 04978399..ff3fc802 100644 --- a/deployments/helm/dra-example-driver/templates/clusterrole.yaml +++ b/deployments/helm/dra-example-driver/templates/clusterrole.yaml @@ -9,6 +9,5 @@ rules: - "" - resource.k8s.io - gpu.resource.example.com - - nas.gpu.resource.example.com resources: ["*"] verbs: ["*"] diff --git a/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml b/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml index aed17a1a..670558db 100644 --- a/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml +++ b/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml @@ -33,24 +33,6 @@ spec: serviceAccountName: {{ include "dra-example-driver.serviceAccountName" . }} securityContext: {{- toYaml .Values.kubeletPlugin.podSecurityContext | nindent 8 }} - initContainers: - - name: init - securityContext: - {{- toYaml .Values.kubeletPlugin.containers.init.securityContext | nindent 10 }} - image: {{ include "dra-example-driver.fullimage" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["set-nas-status", "--status", "NotReady"] - resources: - {{- toYaml .Values.kubeletPlugin.containers.init.resources | nindent 10 }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace containers: - name: plugin securityContext: @@ -78,10 +60,6 @@ spec: mountPath: /var/lib/kubelet/plugins - name: cdi mountPath: /var/run/cdi - lifecycle: - preStop: - exec: - command: ["set-nas-status", "--status", "NotReady"] volumes: - name: plugins-registry hostPath: diff --git a/pkg/example.com/resource/clientset/versioned/clientset.go b/pkg/example.com/resource/clientset/versioned/clientset.go index 441fb1a6..764538b8 100644 --- a/pkg/example.com/resource/clientset/versioned/clientset.go +++ b/pkg/example.com/resource/clientset/versioned/clientset.go @@ -26,20 +26,17 @@ import ( rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" gpuv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/gpu/v1alpha1" - nasv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1" ) type Interface interface { Discovery() discovery.DiscoveryInterface GpuV1alpha1() gpuv1alpha1.GpuV1alpha1Interface - NasV1alpha1() nasv1alpha1.NasV1alpha1Interface } // Clientset contains the clients for groups. type Clientset struct { *discovery.DiscoveryClient gpuV1alpha1 *gpuv1alpha1.GpuV1alpha1Client - nasV1alpha1 *nasv1alpha1.NasV1alpha1Client } // GpuV1alpha1 retrieves the GpuV1alpha1Client @@ -47,11 +44,6 @@ func (c *Clientset) GpuV1alpha1() gpuv1alpha1.GpuV1alpha1Interface { return c.gpuV1alpha1 } -// NasV1alpha1 retrieves the NasV1alpha1Client -func (c *Clientset) NasV1alpha1() nasv1alpha1.NasV1alpha1Interface { - return c.nasV1alpha1 -} - // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -100,10 +92,6 @@ func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, if err != nil { return nil, err } - cs.nasV1alpha1, err = nasv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) - if err != nil { - return nil, err - } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) if err != nil { @@ -126,7 +114,6 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.gpuV1alpha1 = gpuv1alpha1.New(c) - cs.nasV1alpha1 = nasv1alpha1.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/pkg/example.com/resource/clientset/versioned/fake/clientset_generated.go b/pkg/example.com/resource/clientset/versioned/fake/clientset_generated.go index 9b8f16ad..16259ee5 100644 --- a/pkg/example.com/resource/clientset/versioned/fake/clientset_generated.go +++ b/pkg/example.com/resource/clientset/versioned/fake/clientset_generated.go @@ -27,8 +27,6 @@ import ( clientset "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned" gpuv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/gpu/v1alpha1" fakegpuv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/gpu/v1alpha1/fake" - nasv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1" - fakenasv1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. @@ -85,8 +83,3 @@ var ( func (c *Clientset) GpuV1alpha1() gpuv1alpha1.GpuV1alpha1Interface { return &fakegpuv1alpha1.FakeGpuV1alpha1{Fake: &c.Fake} } - -// NasV1alpha1 retrieves the NasV1alpha1Client -func (c *Clientset) NasV1alpha1() nasv1alpha1.NasV1alpha1Interface { - return &fakenasv1alpha1.FakeNasV1alpha1{Fake: &c.Fake} -} diff --git a/pkg/example.com/resource/clientset/versioned/fake/register.go b/pkg/example.com/resource/clientset/versioned/fake/register.go index 6a2bb0c1..c61ca6d8 100644 --- a/pkg/example.com/resource/clientset/versioned/fake/register.go +++ b/pkg/example.com/resource/clientset/versioned/fake/register.go @@ -24,7 +24,6 @@ import ( schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - nasv1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" gpuv1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) @@ -33,7 +32,6 @@ var codecs = serializer.NewCodecFactory(scheme) var localSchemeBuilder = runtime.SchemeBuilder{ gpuv1alpha1.AddToScheme, - nasv1alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/example.com/resource/clientset/versioned/scheme/register.go b/pkg/example.com/resource/clientset/versioned/scheme/register.go index f9438bc0..9e5297c2 100644 --- a/pkg/example.com/resource/clientset/versioned/scheme/register.go +++ b/pkg/example.com/resource/clientset/versioned/scheme/register.go @@ -24,7 +24,6 @@ import ( schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - nasv1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" gpuv1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) @@ -33,7 +32,6 @@ var Codecs = serializer.NewCodecFactory(Scheme) var ParameterCodec = runtime.NewParameterCodec(Scheme) var localSchemeBuilder = runtime.SchemeBuilder{ gpuv1alpha1.AddToScheme, - nasv1alpha1.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/doc.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/doc.go deleted file mode 100644 index c3d31f1d..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated typed clients. -package v1alpha1 diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/doc.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/doc.go deleted file mode 100644 index d2f13be5..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -// Package fake has the automatically generated clients. -package fake diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nas_client.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nas_client.go deleted file mode 100644 index 1115594a..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nas_client.go +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - rest "k8s.io/client-go/rest" - testing "k8s.io/client-go/testing" - v1alpha1 "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1" -) - -type FakeNasV1alpha1 struct { - *testing.Fake -} - -func (c *FakeNasV1alpha1) NodeAllocationStates(namespace string) v1alpha1.NodeAllocationStateInterface { - return &FakeNodeAllocationStates{c, namespace} -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *FakeNasV1alpha1) RESTClient() rest.Interface { - var ret *rest.RESTClient - return ret -} diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nodeallocationstate.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nodeallocationstate.go deleted file mode 100644 index 9150315d..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/fake/fake_nodeallocationstate.go +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" - v1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" -) - -// FakeNodeAllocationStates implements NodeAllocationStateInterface -type FakeNodeAllocationStates struct { - Fake *FakeNasV1alpha1 - ns string -} - -var nodeallocationstatesResource = v1alpha1.SchemeGroupVersion.WithResource("nodeallocationstates") - -var nodeallocationstatesKind = v1alpha1.SchemeGroupVersion.WithKind("NodeAllocationState") - -// Get takes name of the nodeAllocationState, and returns the corresponding nodeAllocationState object, and an error if there is any. -func (c *FakeNodeAllocationStates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NodeAllocationState, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(nodeallocationstatesResource, c.ns, name), &v1alpha1.NodeAllocationState{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.NodeAllocationState), err -} - -// List takes label and field selectors, and returns the list of NodeAllocationStates that match those selectors. -func (c *FakeNodeAllocationStates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NodeAllocationStateList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(nodeallocationstatesResource, nodeallocationstatesKind, c.ns, opts), &v1alpha1.NodeAllocationStateList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.NodeAllocationStateList{ListMeta: obj.(*v1alpha1.NodeAllocationStateList).ListMeta} - for _, item := range obj.(*v1alpha1.NodeAllocationStateList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested nodeAllocationStates. -func (c *FakeNodeAllocationStates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(nodeallocationstatesResource, c.ns, opts)) - -} - -// Create takes the representation of a nodeAllocationState and creates it. Returns the server's representation of the nodeAllocationState, and an error, if there is any. -func (c *FakeNodeAllocationStates) Create(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.CreateOptions) (result *v1alpha1.NodeAllocationState, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(nodeallocationstatesResource, c.ns, nodeAllocationState), &v1alpha1.NodeAllocationState{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.NodeAllocationState), err -} - -// Update takes the representation of a nodeAllocationState and updates it. Returns the server's representation of the nodeAllocationState, and an error, if there is any. -func (c *FakeNodeAllocationStates) Update(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.UpdateOptions) (result *v1alpha1.NodeAllocationState, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(nodeallocationstatesResource, c.ns, nodeAllocationState), &v1alpha1.NodeAllocationState{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.NodeAllocationState), err -} - -// Delete takes name of the nodeAllocationState and deletes it. Returns an error if one occurs. -func (c *FakeNodeAllocationStates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(nodeallocationstatesResource, c.ns, name, opts), &v1alpha1.NodeAllocationState{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeNodeAllocationStates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(nodeallocationstatesResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.NodeAllocationStateList{}) - return err -} - -// Patch applies the patch and returns the patched nodeAllocationState. -func (c *FakeNodeAllocationStates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeAllocationState, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(nodeallocationstatesResource, c.ns, name, pt, data, subresources...), &v1alpha1.NodeAllocationState{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.NodeAllocationState), err -} diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/generated_expansion.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/generated_expansion.go deleted file mode 100644 index 28ebb4f0..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/generated_expansion.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -type NodeAllocationStateExpansion interface{} diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nas_client.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nas_client.go deleted file mode 100644 index c149f407..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nas_client.go +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "net/http" - - rest "k8s.io/client-go/rest" - v1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/scheme" -) - -type NasV1alpha1Interface interface { - RESTClient() rest.Interface - NodeAllocationStatesGetter -} - -// NasV1alpha1Client is used to interact with features provided by the nas.gpu.resource.example.com group. -type NasV1alpha1Client struct { - restClient rest.Interface -} - -func (c *NasV1alpha1Client) NodeAllocationStates(namespace string) NodeAllocationStateInterface { - return newNodeAllocationStates(c, namespace) -} - -// NewForConfig creates a new NasV1alpha1Client for the given config. -// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), -// where httpClient was generated with rest.HTTPClientFor(c). -func NewForConfig(c *rest.Config) (*NasV1alpha1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - httpClient, err := rest.HTTPClientFor(&config) - if err != nil { - return nil, err - } - return NewForConfigAndClient(&config, httpClient) -} - -// NewForConfigAndClient creates a new NasV1alpha1Client for the given config and http client. -// Note the http client provided takes precedence over the configured transport values. -func NewForConfigAndClient(c *rest.Config, h *http.Client) (*NasV1alpha1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - client, err := rest.RESTClientForConfigAndClient(&config, h) - if err != nil { - return nil, err - } - return &NasV1alpha1Client{client}, nil -} - -// NewForConfigOrDie creates a new NasV1alpha1Client for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *NasV1alpha1Client { - client, err := NewForConfig(c) - if err != nil { - panic(err) - } - return client -} - -// New creates a new NasV1alpha1Client for the given RESTClient. -func New(c rest.Interface) *NasV1alpha1Client { - return &NasV1alpha1Client{c} -} - -func setConfigDefaults(config *rest.Config) error { - gv := v1alpha1.SchemeGroupVersion - config.GroupVersion = &gv - config.APIPath = "/apis" - config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() - - if config.UserAgent == "" { - config.UserAgent = rest.DefaultKubernetesUserAgent() - } - - return nil -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *NasV1alpha1Client) RESTClient() rest.Interface { - if c == nil { - return nil - } - return c.restClient -} diff --git a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nodeallocationstate.go b/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nodeallocationstate.go deleted file mode 100644 index f7defbbf..00000000 --- a/pkg/example.com/resource/clientset/versioned/typed/nas/v1alpha1/nodeallocationstate.go +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2024 The Kubernetes Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" - v1alpha1 "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" - scheme "sigs.k8s.io/dra-example-driver/pkg/example.com/resource/clientset/versioned/scheme" -) - -// NodeAllocationStatesGetter has a method to return a NodeAllocationStateInterface. -// A group's client should implement this interface. -type NodeAllocationStatesGetter interface { - NodeAllocationStates(namespace string) NodeAllocationStateInterface -} - -// NodeAllocationStateInterface has methods to work with NodeAllocationState resources. -type NodeAllocationStateInterface interface { - Create(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.CreateOptions) (*v1alpha1.NodeAllocationState, error) - Update(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.UpdateOptions) (*v1alpha1.NodeAllocationState, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.NodeAllocationState, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.NodeAllocationStateList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeAllocationState, err error) - NodeAllocationStateExpansion -} - -// nodeAllocationStates implements NodeAllocationStateInterface -type nodeAllocationStates struct { - client rest.Interface - ns string -} - -// newNodeAllocationStates returns a NodeAllocationStates -func newNodeAllocationStates(c *NasV1alpha1Client, namespace string) *nodeAllocationStates { - return &nodeAllocationStates{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the nodeAllocationState, and returns the corresponding nodeAllocationState object, and an error if there is any. -func (c *nodeAllocationStates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.NodeAllocationState, err error) { - result = &v1alpha1.NodeAllocationState{} - err = c.client.Get(). - Namespace(c.ns). - Resource("nodeallocationstates"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of NodeAllocationStates that match those selectors. -func (c *nodeAllocationStates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.NodeAllocationStateList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1alpha1.NodeAllocationStateList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("nodeallocationstates"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested nodeAllocationStates. -func (c *nodeAllocationStates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("nodeallocationstates"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a nodeAllocationState and creates it. Returns the server's representation of the nodeAllocationState, and an error, if there is any. -func (c *nodeAllocationStates) Create(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.CreateOptions) (result *v1alpha1.NodeAllocationState, err error) { - result = &v1alpha1.NodeAllocationState{} - err = c.client.Post(). - Namespace(c.ns). - Resource("nodeallocationstates"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(nodeAllocationState). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a nodeAllocationState and updates it. Returns the server's representation of the nodeAllocationState, and an error, if there is any. -func (c *nodeAllocationStates) Update(ctx context.Context, nodeAllocationState *v1alpha1.NodeAllocationState, opts v1.UpdateOptions) (result *v1alpha1.NodeAllocationState, err error) { - result = &v1alpha1.NodeAllocationState{} - err = c.client.Put(). - Namespace(c.ns). - Resource("nodeallocationstates"). - Name(nodeAllocationState.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(nodeAllocationState). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the nodeAllocationState and deletes it. Returns an error if one occurs. -func (c *nodeAllocationStates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("nodeallocationstates"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *nodeAllocationStates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("nodeallocationstates"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched nodeAllocationState. -func (c *nodeAllocationStates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.NodeAllocationState, err error) { - result = &v1alpha1.NodeAllocationState{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("nodeallocationstates"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/flags/nodeallocationstate.go b/pkg/flags/crds.go similarity index 58% rename from pkg/flags/nodeallocationstate.go rename to pkg/flags/crds.go index 41652613..d98c4fc7 100644 --- a/pkg/flags/nodeallocationstate.go +++ b/pkg/flags/crds.go @@ -17,25 +17,17 @@ package flags import ( - "context" - "fmt" - "github.com/urfave/cli/v2" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - coreclientset "k8s.io/client-go/kubernetes" - - nascrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/nas/v1alpha1" ) -type NasConfig struct { +type CRDConfig struct { NodeName string Namespace string HideNodeName bool } -func (n *NasConfig) Flags() []cli.Flag { +func (n *CRDConfig) Flags() []cli.Flag { flags := []cli.Flag{ &cli.StringFlag{ Name: "namespace", @@ -57,23 +49,3 @@ func (n *NasConfig) Flags() []cli.Flag { return flags } - -func (n *NasConfig) NewNodeAllocationState(ctx context.Context, core coreclientset.Interface) (*nascrd.NodeAllocationState, error) { - node, err := core.CoreV1().Nodes().Get(ctx, n.NodeName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("get node object: %v", err) - } - - crdconfig := &nascrd.NodeAllocationStateConfig{ - Name: n.NodeName, - Namespace: n.Namespace, - Owner: &metav1.OwnerReference{ - APIVersion: "v1", - Kind: "Node", - Name: n.NodeName, - UID: node.UID, - }, - } - nascr := nascrd.NewNodeAllocationState(crdconfig) - return nascr, nil -} From 8c7f2a7147cf6879aa55ec914871dbd8f0bd6e8f Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 10 Jul 2024 16:30:46 +0000 Subject: [PATCH 09/10] Add support for checkpointing prepared devices by kubelet plugin Signed-off-by: Kevin Klues --- cmd/dra-example-kubeletplugin/cdi.go | 6 +- cmd/dra-example-kubeletplugin/checkpoint.go | 53 ++++++++++++ cmd/dra-example-kubeletplugin/discovery.go | 2 +- cmd/dra-example-kubeletplugin/main.go | 7 +- cmd/dra-example-kubeletplugin/state.go | 90 +++++++++++++++------ go.mod | 8 +- go.sum | 58 ++++--------- 7 files changed, 144 insertions(+), 80 deletions(-) create mode 100644 cmd/dra-example-kubeletplugin/checkpoint.go diff --git a/cmd/dra-example-kubeletplugin/cdi.go b/cmd/dra-example-kubeletplugin/cdi.go index fe787c3b..9fbbc167 100644 --- a/cmd/dra-example-kubeletplugin/cdi.go +++ b/cmd/dra-example-kubeletplugin/cdi.go @@ -102,10 +102,10 @@ func (cdi *CDIHandler) CreateClaimSpecFile(claimUID string, devices *PreparedDev case gpucrd.GpuDeviceType: for _, device := range devices.Gpu.Devices { cdiDevice := cdispec.Device{ - Name: device.uuid, + Name: device.UUID, ContainerEdits: cdispec.ContainerEdits{ Env: []string{ - fmt.Sprintf("GPU_DEVICE_%d=%s", gpuIdx, device.uuid), + fmt.Sprintf("GPU_DEVICE_%d=%s", gpuIdx, device.UUID), }, }, } @@ -138,7 +138,7 @@ func (cdi *CDIHandler) GetClaimDevices(claimUID string, devices *PreparedDevices switch devices.Type() { case gpucrd.GpuDeviceType: for _, device := range devices.Gpu.Devices { - cdiDevice := cdiapi.QualifiedName(cdiVendor, cdiClass, device.uuid) + cdiDevice := cdiapi.QualifiedName(cdiVendor, cdiClass, device.UUID) cdiDevices = append(cdiDevices, cdiDevice) } default: diff --git a/cmd/dra-example-kubeletplugin/checkpoint.go b/cmd/dra-example-kubeletplugin/checkpoint.go new file mode 100644 index 00000000..0762fdb9 --- /dev/null +++ b/cmd/dra-example-kubeletplugin/checkpoint.go @@ -0,0 +1,53 @@ +package main + +import ( + "encoding/json" + + "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/checksum" +) + +type Checkpoint struct { + Checksum checksum.Checksum `json:"checksum"` + V1 *CheckpointV1 `json:"v1,omitempty"` +} + +type CheckpointV1 struct { + PreparedClaims PreparedClaims `json:"preparedClaims,omitempty"` +} + +func newCheckpoint() *Checkpoint { + pc := &Checkpoint{ + Checksum: 0, + V1: &CheckpointV1{ + PreparedClaims: make(map[string]*PreparedDevices), + }, + } + return pc +} + +func (cp *Checkpoint) MarshalCheckpoint() ([]byte, error) { + cp.Checksum = 0 + out, err := json.Marshal(*cp) + if err != nil { + return nil, err + } + cp.Checksum = checksum.New(out) + return json.Marshal(*cp) +} + +func (cp *Checkpoint) UnmarshalCheckpoint(data []byte) error { + return json.Unmarshal(data, cp) +} + +func (cp *Checkpoint) VerifyChecksum() error { + ck := cp.Checksum + cp.Checksum = 0 + defer func() { + cp.Checksum = ck + }() + out, err := json.Marshal(*cp) + if err != nil { + return err + } + return ck.Verify(out) +} diff --git a/cmd/dra-example-kubeletplugin/discovery.go b/cmd/dra-example-kubeletplugin/discovery.go index e5f2e5ca..9608a451 100644 --- a/cmd/dra-example-kubeletplugin/discovery.go +++ b/cmd/dra-example-kubeletplugin/discovery.go @@ -32,7 +32,7 @@ func enumerateAllPossibleDevices() (AllocatableDevices, error) { for _, uuid := range uuids { deviceInfo := &AllocatableDeviceInfo{ GpuInfo: &GpuInfo{ - uuid: uuid, + UUID: uuid, model: "LATEST-GPU-MODEL", }, } diff --git a/cmd/dra-example-kubeletplugin/main.go b/cmd/dra-example-kubeletplugin/main.go index cfd3eb9a..967ef5f6 100644 --- a/cmd/dra-example-kubeletplugin/main.go +++ b/cmd/dra-example-kubeletplugin/main.go @@ -36,9 +36,10 @@ import ( const ( DriverName = gpucrd.GroupName - PluginRegistrationPath = "/var/lib/kubelet/plugins_registry/" + DriverName + ".sock" - DriverPluginPath = "/var/lib/kubelet/plugins/" + DriverName - DriverPluginSocketPath = DriverPluginPath + "/plugin.sock" + PluginRegistrationPath = "/var/lib/kubelet/plugins_registry/" + DriverName + ".sock" + DriverPluginPath = "/var/lib/kubelet/plugins/" + DriverName + DriverPluginSocketPath = DriverPluginPath + "/plugin.sock" + DriverPluginCheckpointFile = "checkpoint.json" ) type Flags struct { diff --git a/cmd/dra-example-kubeletplugin/state.go b/cmd/dra-example-kubeletplugin/state.go index c4d96b76..6000f331 100644 --- a/cmd/dra-example-kubeletplugin/state.go +++ b/cmd/dra-example-kubeletplugin/state.go @@ -22,6 +22,7 @@ import ( "sync" resourceapi "k8s.io/api/resource/v1alpha2" + "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" gpucrd "sigs.k8s.io/dra-example-driver/api/example.com/resource/gpu/v1alpha1" ) @@ -30,24 +31,24 @@ type AllocatableDevices map[string]*AllocatableDeviceInfo type PreparedClaims map[string]*PreparedDevices type GpuInfo struct { - uuid string + UUID string `json:"uuid"` model string } type AllocatedGpus struct { - Devices []string + Devices []string `json:"devices"` } type AllocatedDevices struct { - Gpu *AllocatedGpus + Gpu *AllocatedGpus `json:"gpu"` } type PreparedGpus struct { - Devices []*GpuInfo + Devices []*GpuInfo `json:"devices"` } type PreparedDevices struct { - Gpu *PreparedGpus + Gpu *PreparedGpus `json:"gpu"` } func (d AllocatedDevices) Type() string { @@ -70,9 +71,9 @@ type AllocatableDeviceInfo struct { type DeviceState struct { sync.Mutex - cdi *CDIHandler - allocatable AllocatableDevices - prepared PreparedClaims + cdi *CDIHandler + allocatable AllocatableDevices + checkpointManager checkpointmanager.CheckpointManager } func NewDeviceState(config *Config) (*DeviceState, error) { @@ -91,10 +92,31 @@ func NewDeviceState(config *Config) (*DeviceState, error) { return nil, fmt.Errorf("unable to create CDI spec file for common edits: %v", err) } + checkpointManager, err := checkpointmanager.NewCheckpointManager(DriverPluginPath) + if err != nil { + return nil, fmt.Errorf("unable to create checkpoint manager: %v", err) + } + state := &DeviceState{ - cdi: cdi, - allocatable: allocatable, - prepared: make(PreparedClaims), + cdi: cdi, + allocatable: allocatable, + checkpointManager: checkpointManager, + } + + checkpoints, err := state.checkpointManager.ListCheckpoints() + if err != nil { + return nil, fmt.Errorf("unable to list checkpoints: %v", err) + } + + for _, c := range checkpoints { + if c == DriverPluginCheckpointFile { + return state, nil + } + } + + checkpoint := newCheckpoint() + if err := state.checkpointManager.CreateCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil { + return nil, fmt.Errorf("unable to sync to checkpoint: %v", err) } return state, nil @@ -104,20 +126,26 @@ func (s *DeviceState) Prepare(claimUID string, allocation AllocatedDevices) ([]s s.Lock() defer s.Unlock() - if s.prepared[claimUID] != nil { - cdiDevices, err := s.cdi.GetClaimDevices(claimUID, s.prepared[claimUID]) + checkpoint := newCheckpoint() + if err := s.checkpointManager.GetCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil { + return nil, fmt.Errorf("unable to sync from checkpoint: %v", err) + } + preparedClaims := checkpoint.V1.PreparedClaims + + if preparedClaims[claimUID] != nil { + cdiDevices, err := s.cdi.GetClaimDevices(claimUID, preparedClaims[claimUID]) if err != nil { return nil, fmt.Errorf("unable to get CDI devices names: %v", err) } return cdiDevices, nil } - prepared := &PreparedDevices{} + preparedDevices := &PreparedDevices{} var err error switch allocation.Type() { case gpucrd.GpuDeviceType: - prepared.Gpu, err = s.prepareGpus(claimUID, allocation.Gpu) + preparedDevices.Gpu, err = s.prepareGpus(claimUID, allocation.Gpu) default: err = fmt.Errorf("unknown device type: %v", allocation.Type()) } @@ -125,18 +153,21 @@ func (s *DeviceState) Prepare(claimUID string, allocation AllocatedDevices) ([]s return nil, fmt.Errorf("praparation failed: %v", err) } - err = s.cdi.CreateClaimSpecFile(claimUID, prepared) + err = s.cdi.CreateClaimSpecFile(claimUID, preparedDevices) if err != nil { return nil, fmt.Errorf("unable to create CDI spec file for claim: %v", err) } - s.prepared[claimUID] = prepared - - cdiDevices, err := s.cdi.GetClaimDevices(claimUID, s.prepared[claimUID]) + cdiDevices, err := s.cdi.GetClaimDevices(claimUID, preparedDevices) if err != nil { return nil, fmt.Errorf("unable to get CDI devices names: %v", err) } + preparedClaims[claimUID] = preparedDevices + if err := s.checkpointManager.CreateCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil { + return nil, fmt.Errorf("unable to sync to checkpoint: %v", err) + } + return cdiDevices, nil } @@ -144,18 +175,24 @@ func (s *DeviceState) Unprepare(claimUID string) error { s.Lock() defer s.Unlock() - if s.prepared[claimUID] == nil { + checkpoint := newCheckpoint() + if err := s.checkpointManager.GetCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil { + return fmt.Errorf("unable to sync from checkpoint: %v", err) + } + preparedClaims := checkpoint.V1.PreparedClaims + + if preparedClaims[claimUID] == nil { return nil } - switch s.prepared[claimUID].Type() { + switch preparedClaims[claimUID].Type() { case gpucrd.GpuDeviceType: - err := s.unprepareGpus(claimUID, s.prepared[claimUID]) + err := s.unprepareGpus(claimUID, preparedClaims[claimUID]) if err != nil { return fmt.Errorf("unprepare failed: %v", err) } default: - return fmt.Errorf("unknown device type: %v", s.prepared[claimUID].Type()) + return fmt.Errorf("unknown device type: %v", preparedClaims[claimUID].Type()) } err := s.cdi.DeleteClaimSpecFile(claimUID) @@ -163,7 +200,10 @@ func (s *DeviceState) Unprepare(claimUID string) error { return fmt.Errorf("unable to delete CDI spec file for claim: %v", err) } - delete(s.prepared, claimUID) + delete(preparedClaims, claimUID) + if err := s.checkpointManager.CreateCheckpoint(DriverPluginCheckpointFile, checkpoint); err != nil { + return fmt.Errorf("unable to sync to checkpoint: %v", err) + } return nil } @@ -192,7 +232,7 @@ func (s *DeviceState) getResourceModelFromAllocatableDevices() resourceapi.Resou var instances []resourceapi.NamedResourcesInstance for _, device := range s.allocatable { instance := resourceapi.NamedResourcesInstance{ - Name: strings.ToLower(device.uuid), + Name: strings.ToLower(device.UUID), } instances = append(instances, instance) } diff --git a/go.mod b/go.mod index edc3af00..00c9215e 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( k8s.io/dynamic-resource-allocation v0.30.2 k8s.io/klog/v2 v2.120.1 k8s.io/kubelet v0.30.2 + k8s.io/kubernetes v1.30.2 k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) @@ -26,14 +27,13 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -47,8 +47,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/runc v1.1.4 // indirect - github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb // indirect + github.com/opencontainers/runc v1.1.12 // indirect + github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.4.0 // indirect diff --git a/go.sum b/go.sum index 83501f1a..f98a0aea 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,25 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -39,21 +32,14 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -90,7 +76,6 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -103,16 +88,16 @@ github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= -github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg= -github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb h1:1xSVPOd7/UA+39/hXEGnBJ13p6JFB0E1EvQFlrRDOXI= +github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= +github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78 h1:R5M2qXZiK/mWPMT4VldCOiSL9HIAMuxQZWdG0CSM5+4= +github.com/opencontainers/runtime-spec v1.0.3-0.20220909204839-494a5a6aca78/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -127,13 +112,11 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -151,11 +134,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -184,7 +164,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= @@ -195,19 +174,11 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -233,8 +204,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -242,7 +211,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -265,6 +233,8 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubelet v0.30.2 h1:Ck4E/pHndI20IzDXxS57dElhDGASPO5pzXF7BcKfmCY= k8s.io/kubelet v0.30.2/go.mod h1:DSwwTbLQmdNkebAU7ypIALR4P9aXZNFwgRmedojUE94= +k8s.io/kubernetes v1.30.2 h1:11WhS78OYX/lnSy6TXxPO6Hk+E5K9ZNrEsk9JgMSX8I= +k8s.io/kubernetes v1.30.2/go.mod h1:yPbIk3MhmhGigX62FLJm+CphNtjxqCvAIFQXup6RKS0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 49c8d0a9238e9aaee1d861ace610b41dac8ba866 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 17 Apr 2024 15:05:12 +0100 Subject: [PATCH 10/10] Update README with changes for structured parameters Signed-off-by: Kevin Klues --- README.md | 158 ++++++++++++++++-------------------------------------- 1 file changed, 46 insertions(+), 112 deletions(-) diff --git a/README.md b/README.md index 505bb128..31495b62 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Example Resource Driver for Dynamic Resource Allocation (DRA) +# Example Resource Driver for Dynamic Resource Allocation (DRA) This repository contains an example resource driver for use with the [Dynamic Resource Allocation @@ -30,23 +30,22 @@ The procedure below has been tested and verified on both Linux and Mac. * [kubectl v1.18+](https://kubernetes.io/docs/reference/kubectl/) ### Demo -We start by first cloning this repository and `cd`ing into its `demo` -subdirectory. All of the scripts and example Pod specs used in this demo are -contained here, so take a moment to browse through the various files and see -what's available: +We start by first cloning this repository and `cd`ing into it. All of the +scripts and example Pod specs used in this demo are contained here, so take a +moment to browse through the various files and see what's available: ``` git clone https://github.com/kubernetes-sigs/dra-example-driver.git -cd dra-example-driver/demo +cd dra-example-driver ``` From here we will build the image for the example resource driver: ```bash -./build-driver.sh +./demo/build-driver.sh ``` And create a `kind` cluster to run it in: ```bash -./create-cluster.sh +./demo/create-cluster.sh ``` Once the cluster has been created successfully, double check everything is @@ -67,13 +66,13 @@ kube-system kube-scheduler-dra-example-driver-cluster-control-plane local-path-storage local-path-provisioner-7dbf974f64-9jmc7 1/1 Running 0 1m ``` -And then install the example resource driver via `helm`: +And then install the example resource driver via `helm`. ```bash helm upgrade -i \ --create-namespace \ --namespace dra-example-driver \ dra-example-driver \ - ../deployments/helm/dra-example-driver + deployments/helm/dra-example-driver ``` Double check the driver components have come up successfully: @@ -85,43 +84,46 @@ dra-example-driver-kubeletplugin-qwmbl 1/1 Running 0 1m ``` And show the initial state of available GPU devices on the worker node: -```console -$ kubectl describe -n dra-example-driver nas/dra-example-driver-cluster-worker -... -Spec: - Allocatable Devices: - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-9ede7e32-5825-a11b-fa3d-bab6d47e0243 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-e7b42cb1-4fd8-91b2-bc77-352a0c1f5747 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-f11773a1-5bfb-e48b-3d98-1beb5baaf08e - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-0159f35e-99ee-b2b5-74f1-9d18df3f22ac - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-657bd2e7-f5c2-a7f2-fbaa-0d1cdc32f81b - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-18db0e85-99e9-c746-8531-ffeb86328b39 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-93d37703-997c-c46f-a531-755e3e0dc2ac - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-ee3e4b55-fcda-44b8-0605-64b7a9967744 -... +``` +$ kubectl get resourceslice -o yaml +apiVersion: v1 +items: +- apiVersion: resource.k8s.io/v1alpha2 + driverName: gpu.resource.example.com + kind: ResourceSlice + metadata: + creationTimestamp: "2024-04-17T13:45:44Z" + generateName: dra-example-driver-cluster-worker-gpu.resource.example.com- + name: dra-example-driver-cluster-worker-gpu.resource.example.comxktph + ownerReferences: + - apiVersion: v1 + controller: true + kind: Node + name: dra-example-driver-cluster-worker + uid: 4dc7c3b2-d99c-492b-8ede-37d435e56b2d + resourceVersion: "1189" + uid: 61c965b5-54a9-40ee-88a1-c52a814fa624 + namedResources: + instances: + - name: gpu-0159f35e-99ee-b2b5-74f1-9d18df3f22ac + - name: gpu-657bd2e7-f5c2-a7f2-fbaa-0d1cdc32f81b + - name: gpu-18db0e85-99e9-c746-8531-ffeb86328b39 + - name: gpu-93d37703-997c-c46f-a531-755e3e0dc2ac + - name: gpu-ee3e4b55-fcda-44b8-0605-64b7a9967744 + - name: gpu-9ede7e32-5825-a11b-fa3d-bab6d47e0243 + - name: gpu-e7b42cb1-4fd8-91b2-bc77-352a0c1f5747 + - name: gpu-f11773a1-5bfb-e48b-3d98-1beb5baaf08e + nodeName: dra-example-driver-cluster-worker +kind: List +metadata: + resourceVersion: "" ``` Next, deploy four example apps that demonstrate how `ResourceClaim`s, `ResourceClaimTemplate`s, and custom `ClaimParameter` objects can be used to request access to resources in various ways: ```bash -kubectl apply --filename=gpu-test{1,2,3,4}.yaml +kubectl apply --filename=demo/gpu-test{1,2,3,4}.yaml ``` And verify that they are coming up successfully: @@ -196,47 +198,13 @@ You can use the UUIDs of the GPUs set in these environment variables to verify that they were handed out in a way consistent with the semantics shown in the figure above. -Likewise, looking at the `ClaimAllocations` section of the -`NodeAllocationState` object on the worker node will show which GPUs have been -allocated to a given `ResourceClaim` by the resource driver: -```console -$ kubectl describe -n dra-example-driver nas/dra-example-driver-cluster-worker -... -Spec: - ... - Prepared Claims: - 132ccf41-2ec6-4751-a0e5-94f3635a679a: - Gpu: - Devices: - Uuid: GPU-0159f35e-99ee-b2b5-74f1-9d18df3f22ac - 330d73e1-b5bb-40be-bc4b-2b940f1bf34f: - Gpu: - Devices: - Uuid: GPU-18db0e85-99e9-c746-8531-ffeb86328b39 - d764a8d4-4481-4bc6-959b-27695f434953: - Gpu: - Devices: - Uuid: GPU-ee3e4b55-fcda-44b8-0605-64b7a9967744 - Uuid: GPU-e7b42cb1-4fd8-91b2-bc77-352a0c1f5747 - Uuid: GPU-9ede7e32-5825-a11b-fa3d-bab6d47e0243 - Uuid: GPU-f11773a1-5bfb-e48b-3d98-1beb5baaf08e - e811664d-e487-4eb7-9ac7-678c837cbb32: - Gpu: - Devices: - Uuid: GPU-657bd2e7-f5c2-a7f2-fbaa-0d1cdc32f81b - ef07764f-8dc0-4c4b-a99f-328711702e63: - Gpu: - Devices: - Uuid: GPU-93d37703-997c-c46f-a531-755e3e0dc2ac -``` - Once you have verified everything is running correctly, delete all of the example apps: ```bash -kubectl delete --wait=false --filename=gpu-test{1,2,3,4}.yaml +kubectl delete --wait=false --filename=demo/gpu-test{1,2,3,4}.yaml ``` -Wait for them to terminate: +And wait for them to terminate: ```console $ kubectl get pod -A NAMESPACE NAME READY STATUS RESTARTS AGE @@ -250,44 +218,10 @@ gpu-test4 pod0 1/1 Terminating 0 31m ... ``` -And show that the `ClaimAllocations` section of the `NodeAllocationState` -object on the worker node is now back to its initial state: -```console -$ kubectl describe -n dra-example-driver nas/dra-example-driver-cluster-worker -... -Spec: - Allocatable Devices: - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-657bd2e7-f5c2-a7f2-fbaa-0d1cdc32f81b - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-18db0e85-99e9-c746-8531-ffeb86328b39 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-93d37703-997c-c46f-a531-755e3e0dc2ac - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-ee3e4b55-fcda-44b8-0605-64b7a9967744 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-9ede7e32-5825-a11b-fa3d-bab6d47e0243 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-e7b42cb1-4fd8-91b2-bc77-352a0c1f5747 - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-f11773a1-5bfb-e48b-3d98-1beb5baaf08e - Gpu: - Product Name: LATEST-GPU-MODEL - Uuid: GPU-0159f35e-99ee-b2b5-74f1-9d18df3f22ac -... -``` - Finally, you can run the following to cleanup your environment and delete the `kind` cluster started previously: ```bash -./delete-cluster.sh +./demo/delete-cluster.sh ``` ## Anatomy of a DRA resource driver