From 16b1888d2922965bc488bfc60739c223c146017d Mon Sep 17 00:00:00 2001 From: Jan Kaspar <2270833+jankaspar@users.noreply.github.com> Date: Mon, 1 Jul 2019 11:59:28 +0100 Subject: [PATCH 01/10] Initial setup --- README.md | 8 +++ api-requests.rest | 10 ++++ cmd/api/jobs/jobs.go | 58 +++++++++++++++++++++ cmd/api/main.go | 52 +++++++++++++++++++ go.mod | 16 ++++++ go.sum | 112 ++++++++++++++++++++++++++++++++++++++++ internal/model/model.go | 53 +++++++++++++++++++ 7 files changed, 309 insertions(+) create mode 100644 api-requests.rest create mode 100644 cmd/api/jobs/jobs.go create mode 100644 cmd/api/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/model/model.go diff --git a/README.md b/README.md index 1461f0bbf3b..0131fd46b9d 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,11 @@ To achieve fairness between users we have implemented a Condor like algorithm to Current implementation utilises Redis to store queues of jobs. Redis streams are used for job events. ![Diagram](./batch-api.png) + + + +## Developer setup +Run local redis +``` +sudo docker run --expose=6379 --network=host redis +``` diff --git a/api-requests.rest b/api-requests.rest new file mode 100644 index 00000000000..166f3c665d0 --- /dev/null +++ b/api-requests.rest @@ -0,0 +1,10 @@ +POST http://localhost:8080/jobs HTTP/1.1 +content-type: application/json + +[{ + "queue": "test", + "priority": 2, + "podSpec": { + } +}] + diff --git a/cmd/api/jobs/jobs.go b/cmd/api/jobs/jobs.go new file mode 100644 index 00000000000..d09e748df71 --- /dev/null +++ b/cmd/api/jobs/jobs.go @@ -0,0 +1,58 @@ +package jobs + +import ( + "fmt" + "time" + + "github.com/go-redis/redis" + "github.com/kjk/betterguid" + + "github.com/G-Research/k8s-batch/internal/model" +) + +const jobObjectPrefix = "job:" +const queuPerfix = "Job:Queue:" + +func AddJobs(db *redis.Client, requets []model.JobRequest) error { + + pipe := db.TxPipeline() + for _, request := range requets { + + job := createJob(&request) + + pipe.ZAdd(queuPerfix+job.Queue, redis.Z{ + Member: job.Id, + Score: job.Priority}) + + saveJobObject(pipe, job) + db.HMSet(jobObjectPrefix+job.Id, nil) + + fmt.Println(job) + } + _, e := pipe.Exec() + return e +} + +func createJob(jobRequest *model.JobRequest) *model.Job { + j := model.Job{ + Id: betterguid.New(), + Queue: jobRequest.Queue, + JobSetId: jobRequest.JobSetId, + + Status: model.Queued, + Priority: jobRequest.Priority, + + Resource: model.ComputeResource{}, // TODO + PodSpec: jobRequest.PodSpec, + + Created: time.Now(), + } + return &j +} + +func saveJobObject(db redis.Cmdable, job *model.Job) { + db.HMSet(jobObjectPrefix+job.Id, map[string]interface{}{ + "queue" : job.Queue + // ... TODO + }) +} diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 00000000000..a191d425096 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/go-redis/redis" + + j "github.com/G-Research/k8s-batch/cmd/api/jobs" + "github.com/G-Research/k8s-batch/internal/model" +) + +func main() { + r := gin.Default() + db := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", // no password set + DB: 0, // use default DB + }) + + r.POST("jobs", func(c *gin.Context) { + var jobs []model.JobRequest + + err := c.Bind(&jobs) + if err != nil { + sendError(c, err) + return + } + err = j.AddJobs(db, jobs) + if err != nil { + sendError(c, err) + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "ok", + }) + }) + + r.GET("/ping", func(c *gin.Context) { + c.JSON(200, gin.H{ + "message": "pong", + }) + }) + r.Run() // listen and serve on 0.0.0.0:8080 +} + +func sendError(c *gin.Context, e error) { + c.JSON(http.StatusBadRequest, gin.H{ + "errorMessage": e.Error(), + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000000..94585f21b7c --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/G-Research/k8s-batch + +go 1.12 + +require ( + github.com/gin-gonic/gin v1.4.0 + github.com/go-redis/redis v6.15.2+incompatible + github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a + github.com/kr/pretty v0.1.0 // indirect + github.com/mediocregopher/radix/v3 v3.3.0 + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect + golang.org/x/text v0.3.2 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + k8s.io/api v0.0.0-20190626000116-b178a738ed00 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000000..2d629811c83 --- /dev/null +++ b/go.sum @@ -0,0 +1,112 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= +github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 h1:WSBJMqJbLxsn+bTCPyPYZfqHdJmc8MK4wrBjMft6BAM= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a h1:b+Gt8sQs//Sl5Dcem5zP9Qc2FgEUAygREa2AAa2Vmcw= +github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a/go.mod h1:uxRAhHE1nl34DpWgfe0CYbNYbCnYplaB6rZH9ReWtUk= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed h1:3dQJqqDouawQgl3gBE1PNHKFkJYGEuFb1DbSlaxdosE= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0 h1:oacPXPKHJg0hcngVVrdtTnfGJiS+PtwoQwTBZGFlV4k= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20190626000116-b178a738ed00 h1:Qqj3aerxILStcStl9mGcSbVyYuLxYDr2siLyJReTyaY= +k8s.io/api v0.0.0-20190626000116-b178a738ed00/go.mod h1:O6YAz5STgv7S1/c/XtBULGhSltH7yWEHpWvnA1mmFRg= +k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0 h1:7oql7STcnJ85hz3BIbasXHH/+lLLKwOdsG8vjkZc8Pc= +k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0/go.mod h1:48PVecD7ubRgJmMRGIQfsqYu6OucVH5DzFNtACHZH8k= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 00000000000..a496b004937 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,53 @@ +package model + +import ( + "time" + + v1 "k8s.io/api/core/v1" +) + +type JobRequest struct { + Queue string `json:"queue" binding:"required"` + JobSetId string `json:"jobSetId" binding:"required"` + Priority float64 `json:"priority"` + PodSpec *v1.PodSpec `json:"podSpec" binding:"required"` +} + +type JobStatus int + +const ( + Queued = iota // queued outside cluster + Submitting = iota // send to the cluster, no confirmation available yet + Pending = iota // aknowledged by the cluster, not started yet + Running = iota + Succeeded = iota // job finished with exit code 0 + Failed = iota // failed to start or finished with non zero exit code + Cancelling = iota // Cancellation of job was requested + Cancelled = iota +) + +type Job struct { + Id string + JobSetId string + Queue string + + Status JobStatus + ClusterId string + Priority float64 + + Resource ComputeResource + PodSpec *v1.PodSpec + + Created time.Time +} + +type JobEvent struct { + SequenceId string + JobId string + JobSetId string + Status JobStatus + Created time.Time +} + +type ComputeResource struct { +} From d5f0aeb520a71400b75e585c6868937b2875095d Mon Sep 17 00:00:00 2001 From: Jan Kaspar <2270833+jankaspar@users.noreply.github.com> Date: Mon, 1 Jul 2019 14:13:59 +0100 Subject: [PATCH 02/10] Update diagram --- README.md | 2 +- batch-api.svg | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 batch-api.svg diff --git a/README.md b/README.md index 0131fd46b9d..78bacbc7c9f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To achieve fairness between users we have implemented a Condor like algorithm to Current implementation utilises Redis to store queues of jobs. Redis streams are used for job events. -![Diagram](./batch-api.png) +![Diagram](./batch-api.svg) diff --git a/batch-api.svg b/batch-api.svg new file mode 100644 index 00000000000..93e19d9da7a --- /dev/null +++ b/batch-api.svg @@ -0,0 +1,2 @@ + +
Submit
Submit
Kubernetes Api Server
Kubernetes Api Server
Kubernetes Cluster
Kubernetes Cluster
Queue 1
Queue 1
Queue 2
Queue 2
Queue 3
Queue 3
Accounting
Accounting
Watch nodes & pods
Watch nodes & pods
Create or delete pods
Create or delete pods
Get workloads to schedule
Get workloads to schedule
Cluster Executor
Cluster Executor
API
API
Report events
Report events
Events Recording
Events Recording<br>
Report Usage
Report Usage \ No newline at end of file From fb4b6c27697dad3b028267c790d7178e7c3b94e0 Mon Sep 17 00:00:00 2001 From: Jan Kaspar <2270833+jankaspar@users.noreply.github.com> Date: Mon, 1 Jul 2019 14:18:38 +0100 Subject: [PATCH 03/10] Update diabram --- batch-api.svg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/batch-api.svg b/batch-api.svg index 93e19d9da7a..390ed7664b0 100644 --- a/batch-api.svg +++ b/batch-api.svg @@ -1,2 +1,3 @@ + -
Submit
Submit
Kubernetes Api Server
Kubernetes Api Server
Kubernetes Cluster
Kubernetes Cluster
Queue 1
Queue 1
Queue 2
Queue 2
Queue 3
Queue 3
Accounting
Accounting
Watch nodes & pods
Watch nodes & pods
Create or delete pods
Create or delete pods
Get workloads to schedule
Get workloads to schedule
Cluster Executor
Cluster Executor
API
API
Report events
Report events
Events Recording
Events Recording<br>
Report Usage
Report Usage \ No newline at end of file +
Submit
Submit
Kubernetes Api Server
Kubernetes Api Server
Kubernetes Cluster
Kubernetes Cluster
Queue 1
Queue 1
Queue 2
Queue 2
Queue 3
Queue 3
Accounting
Accounting
Watch nodes & pods
Watch nodes & pods
Create or delete pods
Create or delete pods
Get workloads to schedule
Get workloads to schedule
Cluster Executor
Cluster Executor
API
API
Report events
Report events
Events Recording
Events Recording<br>
Report Usage
Report Usage \ No newline at end of file From 82434030efbcd5a23bc0242571a795ecdfa3b667 Mon Sep 17 00:00:00 2001 From: Jan Kaspar <2270833+jankaspar@users.noreply.github.com> Date: Mon, 1 Jul 2019 14:20:44 +0100 Subject: [PATCH 04/10] Delete png diagram --- batch-api.png | Bin 61118 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 batch-api.png diff --git a/batch-api.png b/batch-api.png deleted file mode 100644 index 609d882435c0f61e35dcbbbc0fa49d84f3762196..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 61118 zcmeFZWn5KT*EdWzihv*>-HoKuAl;2L(%mTyA_5{HUD747rMpzRLy$&Ny1SmaJ?c5v z`(F2XzCEArZ~Ld4z1Es@%rVA4#tKzZkiEk!{B|KdTo zwGIPA1|uydqUHg+m4@1=zI4%P;KD%_T^K4S&1o@WLdU5j_EL#~IxI&_B~$br1_6aJ z1AZe+3axGQBJAKeRt?ER3x3bwf&AsGU zaQBek3lT2BUxW+>7DX5a4$B_~;lF(p#)b2ee9}SwZ{GobX^5Jkv>@?6{Kl_eCZmDH zMSFBF`hULI&oAPl&A|Upck|aTeWCIfC8wdl{+HVZU-TCZ@cQ>l1tcOU(R~pUgZuY~ zQ=)UY`|p>+ih)53PJAu(?@I}ThPd;uYYl^7ge{zjg>w(}zwN}YRRoVDoY^Yy@0Svm zgQLSWqM(5NuY3CEm*{Y7k^lWtuccvvMdie({(Wa*g5gg8{OEt*yGI~EUWdL$`0xMw z`(Hs2k^K9m{x^vJH;Dc>i2ldJ_Ww8&(INfJo#)gaC-l8mYETZIg7nm70T&=lmOZVw zM7|*Dzt%u{|3u{PM7&H0+k7etd5nja$$y6UuldmbE-VwScg9C_I4r5RFoe^~wb9^X zmo8l2!oZBRWbYDf>!Em!pdW&U^RVK$Z#js_rBH|5(+2{=r>Pjz}KZi2i+V z|0T;WD8d8>3gMspU%o}|FWLqlB>(S)V?R*OAaaP#TtUVpwN+=1h9xm_cluaMR!O`B$NGG)!$z-ckmI^%uV zZ}$CDFuhQVR)two5u|x)>suM+iYdNZ0IAPW^or~C@kGkJ-|eghd*B8eekj4XO~r>?f3>51Xuc%7SWztWu0=LUt#uFP3T*5 zF2te@`;Wf9NBNHAOh%lD4jx30u=2eqicGWqWV6H7{?%UAmN(zt#4&0jMKT*S^VbqL z?{C-4Yqx)SvX*LmN8o5^@3Z5z@9C_2Km1*OEKP0fKVcu>0WE(Cxa)tatuPW+BHQ!8 z^_jYg;jM8Z`^I65){UBZUycCbR+i-T50m?@o8w%ALbr`|$KyI23&m(Hp%H{!<&6@N zB(^7Di9gxHaDkvCkj!w~o~rKejb}czy>y@VG0#zoD%T9ZSOb4M!?PWBpK1_g|!dmE0R|D zUSC92hQT|0dmfUM#9=-qJ{|gnAgUq9{Jm#Ks{5ShYDUl-XP3Jz{TU)bHp!=RUW41*$(cp9_rgfO8_5)uIu8m=@PnXUoxi z9(G?IY^6<0Zu2+BjTSsseH(Y_Or?AUV%2IJti5O0Ij|cd65vIv?8DiJ{#@U=*Djh@FlWQ5adj84(lUe9@L z0MksT(ZeyrX0kj0L^RUajl*Q6rRu_r^I%nYHFbmg?i~zLL4KujlkV5YF1vFAdz+9J zr0^6Wzh?RA5}le>`<837Q~UX5fgx~t;j=nw)tg$!C7W|_TNssgvr{#CJoDbpQYmA1 zsw8d31R+#TO^xSSwhh);%$S|v*=}hwnq9i0$WQ##ZkO~YK@csB$g*N`+I(aC=zINQ zTCu3bZl>1JaH%6=E(i8nX!CYZM}op9cjQC5)9q)wlIE%IcxD|qupa#}9l6BZHi_GO zadXE$Uq7g$A8k)h)UbX%G+H0YEj=r+*3{3HiG#D9s@%5?de0#n&xA@Z>Em*HbL}%) zG~1udgYkE4D!@o4`V>G!JQfCh{>AR@!^cDQrxtUKb;DVYF%QV~ z>zqvO%bKS_maQ0vai4Y7loD{l94Xb~<5SfP$W2Rio3fPH$K)KtUAcHQMp!}g)k-a^~S6H3OA-Im)qUB~wOtKGEe7haeh_OB;50FZ=e88pqFDjz(njWa1dc2?zFFy!CyK)w55?|3Xj!HXx0X z6O~e|{jT`J12{t&!D2y5=l9z+hy0t6R6#K?y(4(GU>FUes%$4T$K>_08(re#j#q)v zJylUNPrL39lZQOS?#nz>nsq`g_dGPC#3xYDuC#bQeR^}b+o0E0rE4yg^DW9k$sC+w zHys&ldO~p0@pPBb#LB3A&sXp(OndQCScZkp=?!`?9LF=>TPIZKb`95A`w#FvUY)4^ zRIl&rT5%T+3xzr(5gC(+n^T2Trfx$lXrEAc!mt*u$-2;>rCFdD)*3?R8~C=+FN$hf z^F{BGo$TZz&jXX9RG*9W{1WTjrqj7;3*}qN8pjRyl^sc+0?nLM^?C1BSmb9lf*VD< zwM%zW#MWrrg()Zy{dH-&uiwugc=nDu_aFON4O&B*%r3<-PYfK|5c53t7u}Rej+0bu z3i|amB~+euiNA}5d4crf31z$D`N2v*jw1haUY8$H`E@FL1-t5=#vN}Sff9{*GT2G% zo_ddCB}LGu-Z6|ae=Na^iat&)#N&hQi)QH@Iy86txO}JI36tF)lw6EjDb9O;no}xK zvYAC;&$K}+VswK~OT={J{zzX29xq|&*m`(!S&vu3)5z3-WFeM}_pTc3o|{==pr}_I z>YUg>PiuoBylpZfuiuULt{q>mCWlpSMikd4!OJaU{C(-5Ev&J&0_v|QCLtIQ>L1q_Dlg!^+zUAF0*!x@!i4_mJ;6BdQSDqmV z#m!I{3+eeF6E-O(K1djP`edo+FrGRbk-lEN1?lXwco~h!*F0|Tv+p5op9gvtKcTa7 zmiM6J)5m@m?;-u_I`Cxe<(H3pd83cC5bHNLVui|iy+G^Ge^=-VOKuJ%duwuv+>lr@ zl)t(KvQgBlTOHxNbkyyTe_#O~SWGiZw>%_i&Y}DAoc>g@!JqF6@oSgYMp|-1HsTr4 zFyHmMM(e(ma_Osc;$4YVeS>6S5|NSAKxh+Vp`;Kbd{^K|>9u;Wz>o86UQQ{92p=Z@ z6ZD{G@Tc!7vg~M7nvpU&RVNiWD_j@9gvOW6J8m=fY~C-F-wF{f#fchGulUBbP<$iZ zgNUhO&35(8I1+PF+22EpNy_oZ1^Y2+a?-9kKLW>^_8JAeA+_66^8Zs_X^Z+gF` zo|k)9XmSrBwFiwo2=@AM)Td(D^td@pN-cEONJ5T-Oj;>*WG6lOFW&MWNVPDHqlt|h z>N{#6J8kDvCl`2!;?B-CxYw}6nqZO-SAp0m!Q9w#H88zm2Ki$~PU_9Y zWzd6v-ac{*JG$anI=G2j2^n+;yZE zKVa;R_d6Pvlq{a2C#<%cC47VlD5!uVN@)MHncPV)_LbE-N;|0MJDxJ(mWDyzGX`vW zanL~$XL$_Wm|XTebNR%KWt7YMVJVc3q0#nD+-@uUU5t+D{Z&~r9hTVTx83x2aW$X+ z7|PTr%rVy~cYN{AQdW62T8Bx%dS!$yng8Z&X}mv9SMM6Mj+MK#!y}Tst4wU4OPDEh z2lK99oJMU*Ky0OhGB? zaf}9unaA??7lPFykya+lEx1_9l}(sRL{Pzf+0f`HlDcwgV7Hp{Qi*gWO*uWid88$H zD7w7auCeQRei;*Wb5 z{ABe$>+%OI={NqdD3lU!q?lC6^WzZ*nQLZ@jH_R2mOSRr#+}jKX)Uc=P2QstKhfzC zrsva5O`)(xxu1VOKurrbXKEy~fb4mulG<`1K<`q?{v>chI01E0vc0CRlrwp*^4?-dzQUKQL6bvh zMSmHz!aUS6u_)Cd8V?9thcnt^=~asvY5T((0Va!B#Fz4nChfLAPj=m+C44z%n8U=9 zCH-6C34uCtK(T{#YY zIRh28BFJ`q&J5Za=DZS4VA0!Z^#uLrk_rQzg6J7pI1`b{yHb%9fbzfAJU=)tX~V9g z8EJJdU%c|*u^Q{DLFwO`Y4ohveY8D>{6P#27tvz;jUhd5W7dt1sRfy0C-s~Ny3d}> za8&Vf&DH6=-#mlpr_F?_V5F=uGc6Udcb^*dqo-PZ!CkY$(j(&Xh=>Gm5lGw|=`HBn zTW!aLr`k84cN7Rk(UnZxSN6S>vGQuSlEChj*!+VR@s0Z(L??UKr}OrH zLf~k5zU!?@qj^f+F(A>hT|HgQ?gzM!6A~QBU1tJ?NfEJK=bhu{4f%AA5wyt|`z-jLSabn>e((|LhtVu{6 z6`_C3jLWFJTa?vmUlP5L59Uqi>5t*;r}`MG`u7k4Vn)3;bektQgNq|v@ySH6G{6mF z0pmSYX}KDR&hiltlTaz4#U5g?3uu8!K{eBL<3Z~}HuHgpL>?;VEOpE8_)FtB_j20` zJicflHy-pbj+;jJQkuS+VzCak$eO$)JbE{1RVT>1SzO@_FU`mugmgU98p)`-!?bKh z?(O!0k7O4lm?FD=qc~YyTC0Q#wix?mm0X#+r_GyJx~%hXlYqHePVtY0dx8HIrXsec zydZ2jOko^jgNI8;=h`C}?Y`*7puIsIfNxO;xNo$4(wP*vIKJ^&l-_6ttt88{GO}oi zaYDopGJeJ0V&Ey#)qC+5HJg+cD?fY3`L#j#B{7D^H)YFyiI7 ztWQAFSN6bnzQLMxel0nEn{~;e!jE8HSjs80Nt=ucaep`qpDdU081H)wZR5GxTPFOK zQ3Fqt?NU@i-(93qAE)dB_7ynsM`Sr)-of<{YhIdu&$3scUyYTJT^mR%$r{#aan~{Z z(8MzCKQ*7Pn9d(2q*cG!BtP$|UAf)KGJpOHhIqVi%!hbosXwE z8z*6l?+O)FNDIX$JOPxRqiVBV%M|u0-HF-wS~ShC*Wec6L5j>#0UO{#jU^1Fqn2a* zAJJswel$7W_!>aSWra}hx`}rNh%E{YK0q=EcfX^CubFX@{ld9V@~zp&E5q|})os|# z*91V^_=$zKphML+_H&>WMg&*Dw;0Y6H{4%rchhO%v7K5@d7lPYv>Oy$w_ALBYxBYP z#05GvrQ6oJMAzd`xGR^)M!wfc>bFe);^JsM0H6p7rfk34n*am9Yc8tv3sX(S$#PS8 z0Btd!klYvpmMZ|jS>(goMs3Yf-6wmX22J0{nHKJz#tdgmA|DpbpT_JjbxyAZ)Ogp- zHhG=47AR-i5X`N$U!5Or3+`5Q2%sNf98}XL9n4l_fS(NjC=YS4tx9(D`{xj305ggC zsRa%n8dnW6cI&d*`?lCl;I$ZU1p#di$6~(OXV1@TW2`Ws<>sQ7C+0%t*!dz|H64$o z`y{TE)cLk|PS9?1yad+%QkSr1XR6BDu=6c(fPmKt0>!aUY1+=L?rzV1Bkfu~zkru6 z(dUT`*@v!Xsv=*7i?TUP?@_9<8HAr^(-WqSc zwI;r?dne^-C7g4M6Q@`2vfc^+Q+$4Zv|9mW*uA!GXrw;6XX&?zCO=x!L??4*!Pc=fU=-!@`#RW}QvqJ)zt4NNd4O zPU?l7U^TAY_5AcZ7TqpwqU3s90+mprp9i*X=`~)L6JVU(kP1=)GLz8QbH}&XAYb_4 z#b}n2R~C9kwe#LbUl{{pfI@>XL6XCmQ7evsS_H#wTCvTto9S4{V%@?607=UPHy{h zbaIZ$*tw5M(nX3nqTptW@6c4vTi2H^ZNR(ENT1oT)XbOMcTuZ&%9X8J;YqDsWP@0t z#@3f7pE5;I_&px46r-*pYV*#qdl=<++T?4RO@w|67pGxy{K=#sG=y*Ern1?s1?5H# z6`FeW9^(Knm$HrRT1$L%M)z0``t1b>$*K-5^Vn%}_l9hC>q0yRA~?aRIzZ5!FLW<= z;E|Y9Ic|Y)CVyuMFLRKthKyZi6ie&fZn!Gb_A=C^Ml)jD&llKk$HYM{yqhTVVrIN{ zo2vcBDkQdJx_IT|tv1%K7YBY2kQc%BN4Yb_8mu3eZqhR71aNSDp3zAtE$kVVv(ifQ zSONLB#&)#WbRylFx#+o8=nv17aHm0m)5M0|=4nB5(k@mSeg(fDnIWQ!?CV6^Z~1mU;_=qI0Rp>evcf!4?+9c3Hd0sWLaU)yJ>KkgQ*RSc zA=RFdCr+CWPefFOj%=T{fVm+R^Qc(MQpGqmw+ z%>H_JjR)k&-oCZ=t}SloRFSp4We2{84bFFJIvj$}zBYlTMR66nOBXpVLmSEk zFXn5Ge3!&#)T$#eB{-|f?2AQSh}ol;;bt;YAVV6NS6jG+r@&?<(dqkDfA_rhwSM)| z+oGpOl%tF)St~QiYzne8EG8&#YMNR3HgswK3y$Kqq@LjeXN*5g&rj0aAH|}h$>>ox z+TJT7JGSH`h&{CyfH6NBAY!4|o?-T1Kg7+hRk@8-7YCYrr=`~@ekLV&UY8ITXp~{m z@uKzzA7gTtlgeV%fpRf?i+4GV=8N_#l)>!za^EQG+!65#-B z*`3-ujN^(a7C`H%|5DXJHeSv|r<(T$^9Rc&g?%b(dwz6#DW17HZVBdMw!K%ooN|(H zmT?$I-X)DL=gk6B(Wl6LBDf?8GC`sZIX=cIpgArhz~WO$vs!)+_j(lDZFCkMsfzo9 zS~IWbg{Mn6oM;sw(XT*hc5K<|VbW#A3_`*z-=UX@XClC8q~_2{Qt7qf))mP}kZv05 zs?=j)uB2m&kz+K?`_OhRCblE8{NbtDyj74GYsrA#YxMDlg>p?|()+`J>Bb^q?;k_< z=ctK@O~3w&^86!Ri3#J>EVIZ2bgyVj%s)d*Wv+RJNPe*zs+LiUA`-!D7 zDTZfzpYQ7M$f>Z?8CEu@itoBsJL7U{fz+)Mrbx^Wey5=R0>B=AbILqcmlRaD zJSZQw08obnuEA})T=3rKQLDT@pat>wHyD=-FNKxo|tT}_*a^#EIXzxp$ zrq(W@BK>Dvz3q(2`%XaBeBnO1eX!CyU8V1`biWB>FokDOo3_inUely31K*;F&C}e( zW7I+?9wR@^P5aD_PgR0Lcez59LluC-|B%CnQ7Y}{=el~dg08fYFdi712QbdH1FDv?5E^~GUbnBXmaWGUm{JVLAAqa+J*4iRPn=B6lg%_QfS!tvM+BJXrt z>2|d_u8sqTiZod{ywCOu5IDP7iVzXgNBb}kCEpe6)Ua3y5|7#3Pugbq5sM3}Zz@{c zSD2bfb~pi*sI9(!m~`{lbX%=w9~=u;qf*9#?7bJr};DR^pBnSvs$}A7OYK{+6PlPkTo>1=kfsPVJLK|pQ8ls8*fT4d;60%|a-oSRBD>Tn;@ZBw`q9d?j|tF(o#71MHGOPP+tEL*vs4jKp2}c>_mIX zs9E_vOIjfs>B1c7wUNPV!}dOh-Sd967BDm05Q7%qJqMsGZ@E#O$`Z~bFlsR)J}H!$ zF9vmQ%fRROtKnrMQCNGvf3wphk+rZ;(gCi!2p$#7$@>~R+g|2UrteGl>2a5NrGrkw z2p-6rh(JVq=sj4Pjz(VF73mU$&XI=WF_-US^0{s-hoNZovQy|6YFB+IRLKpCW-Qn{ zdtLg5Z?|z&VJBwV7y)il4w&({4fF;uL38|6IPc>@0^OlR6AI{%fvU<=Gxwq>f?TOo zw~pU4YukZCn3FQEz*}q}^~oqwaJ*J7oVVpZN0WvqT0zW)Woz@#EmvvGO~L7+YYck%<}=B4~1l; z_?ATA#WO}1#MgpLZ#eq?k{?!94A`h`_iCD5Pw{OMevglsGbOw7b($(O%@*(auE}ef4i22~jj0`r6bk%yJrYO z$92ZVd~Wh}iM9_^M697AHMWbBw{1OCQmxFe|bO=NCphKugYW zecYve0jo;9)ERlomkROW#cxWDW7gq*U?RiX?VBa6^A74On2>V@i1$RCiyspBq<1#j zELrMa&_7}n`cW$0zgd>kpDEg@jYv@>=sQ5@98ODBe5l@#~fCu&tRkuMRk z!BpX^frP=+`kfP^kg}Udbs0A5&UbCU+fX62*DE(lFs-@Ta$g^5ht$XJj2651FHbP} z(2L=YeM<{sqFTNq1%HUI=c4>kpQn~ZTxz!7HG-z&2~SIRtUI8_bnsLTqyiQA7u&t-1DOY= zp%Xs0p$w7n#ksVU4g{BU>E}2D8uJt7vBQr8cSz|4sAQ&ri0-x3csPLmK4xTdE=GrH zHe`4PwbS{1PJnPd4k!vcny;ZYGU z9ev6n8&^Qhc6#r3<9R8Xb2#cr@z0t7a^ivPLj7hb`hdb|hw6}fh%WA|Q!jIye`_)( zi{BMmF_Ap4+m;1w$P?0=oN$?7{a2K4ganD+q-(zbAYGcl$7 zH#AmOp%$*kCC}hgyrnY9hfP>~dV83oP5}0~Ay+?IAMGS@=WD9i#bH*Qxu2le*jSqR z_nxU1wSYsabav zm?6k?)XAs=I}kWwmB!B$mRB`f{&kjno60Islk9UUAw2AtT@&wd}>}E^v<+ zjuj}Md;@Gh_B)79ja?hRX>U9xlU_Zihc{rN0!DIWkt4mPRm+UpTfq~;d!1~Rt#?Kc znU7+U@IOwRZ^*pvfw+|A0FvlL7r2~=84Cq&fL1wG4j|@CD?rp+f|>^4Lh!)$(Cd7M zmo+VYYIorI&DA+~S|XI@cKj+&fnq`ld)o3oVy)wei2r5EKONv?38*<2K4*XP41C98 zsrM0_lKst4E3LlvKxs6$j!?5l9^!)RTsR^eim4WEfx5_+Zh_vGJxA&q7*6!dR8tsO zh+XeO3(W*-lkl}SYs{WLc9zoAU>Fbc%pkyGe;UD)7;6^zI^@cjMYonMhoIaLizCS) zEEBIEW5jd)D%H|LipKXJ=RAiR_FQD5p+MK|hk%0bm(R>DqvhWp5mKMB@{-nBoCEsV zIA@jZFP#$xu8`(q0w+ZwOY8yIof(P@{1>RR(q{VZNy$CwIWru3K(I;j#I7WS6BqNp zfxK>2wavtmI=!5(f+!g(p(0RVNlly@)YWc{6~?$m5X`xpXSH0POn-Ja@3W`Jdz7n?}IAW$ho-AW+2&#P!QI> zxZ3ZO2acT*MeFm8osrydt!8geHugw75-yl;*(&et16?IyAzFh_(wgiT+>_s4K4B67Cl1J-16}{O z7sQ`3qb&s--XtKS4?*#nTMbeX6``RO$mwP-)kU*)6RS;f+WM>wpQE&qOrhDb#^+M3y zJ_7U9=R3fGaR?IEWR^=axavpsg2zauZ%xi>8;haQupHvggu|dwYyrgeuDp6&WmF#7fzEqV$Q`h_@263;KI%APX$+!p7+&?8@A z*1uJTOybd+LTxY|J>U+4Zgr<&w>s!Kw1WdywwP_c6}z=IWKvF{o3m`H_mv>0Uc9B!H1o}@OkWm90a}b7@R58s8b#uR}wD|gr7OK*w zC|IU?1IK6eTkxw75Fe=XPP_tY1R*`qGNt!;v6h6TDtYvPCf+^g)xI(oK9^=_h0&lz z44M)GUZ#wqe3LT-bd#UPT|v~BI(EeFGmd6JrE^=DKhEyf7;f@%FEUgYDiG8)l_g*_ zExQo07h<74`k-F4RUs-f4$kD1)!iq{SlP|0gIHQ*bm=j5&TCUQ8aKJFx|ZsHY(ORB zy4p1gM__5n{w$3nbZN{b?)<$p90&^h`(n;x%96pUyP!d`TpdieNB(-hx7FTZ<0X#5 zM-4Qmin15SRw)KS3fP$Z-RjTFjZDUoXzZ;+G^<^m%)4}l zuQ6!*eDfeID1X#60FK`bcuxCOzh*}YS~24-zopa?^h7g-9{B(&l&)l1pJ=3|`w$7c zVBAZIp#2SV>T>L%;#)CsG}yY5nA~K|o>+Qg;I(;J0e&h&_VMw)6D6(qHfVl-TozjZ z*N8esQNK!HL5to;!vJP&&#Mcax+(Mov2@mdKq}!Ygs}GX*w}xh@;V7&4J65>UrhAuHRgNt903pUU%8TCf^Zm}~4}}^f$;v|Ow7?Hu z)o^SR^-0vH1y02Jqro_$uM_YL6VOw}D%eHkWN7%6-e|D-AYXljZVv zMtvRGdh1u#p737n9}kZu73Pl=vRwaT&TF@-O;o)L(89JDuK?dL$3V2aG~ErPtrc#q zpTvXkLjzG2b>WTL-FNqQ3}RCNGa$Jx)$bq$h{6GZ zGe~Xc4JQuQRF;2_0&ciSTk-_Nz#h2?*FjYS?H$NKy$txDeEb?3ld2V7Ddwe7ZU@64 zi~u*t14fi`v(0p$lgaAmDs&dgQLDhhIJnbrbFmqL7pmJ+SBvB(iFPAaE+Hi{tQ_8z zyC$oK_3+nh1}+J$@|Wv(QQry#{y!?87l4XNCV1?Sp#_}z%(7vK zh|g|-+1w_H#ZI)+zt?GnT-ggvrg7~Oa0OqAJ5mINyp1&e8r?(1QrZdvIGH1HX)&CY z8|sR^d29Z392wYlj^sB29~BA3V?WhEaEhclb&WnK;6IL0L4q3GJ6xl5@+=1Nb0_vK zus6@KnOT9o_b;a4wF1D}E`kri!~g!LJ#r;rg>P^Nx~HII1Tp^zh8FX@?_~*QySiQms)#%3)su!6aJC?FF?!?SI(vs){ z;QacH*Y<$s@p)*VoHdNRodgw&Powu$j|nOC7>l7mEkSg6{Qz_IPusU_r>Nr@hiUBr zAMY7SWR)uXx$kubE)q!bdD|7hCS@BQ?WFFLd12JTFT)9lkm-b?TUPZCv3d$Z*Dfiy zK&v>lRXFo)uY-oN4jbR^p6qovAX{qLqEY^40a|4BADCnsXRRyNv^dCK-@wt1C=rn>{9&wJSf`daWjzznN+Q zs@`CgqFN|z&e?6o03HTr77X3&<5 zgK5RuTYisH0-;yqcK5>jcoLwMJ0z+Pk&in+hur0P`U>L_Ku49Gl^O(7kY?3uVE%01 zcIqH7|JH19?G9Alh(LbX%5i)9mK_F}|IG8KaIj!-#a~zu^?@;*_D{OBPJ%4uvgQ16 zZMwM$^v?kxCi+iDSOD-*g6IpR43zDaPS@Czeyex2UI5e_8Vj)IMJ6_bS{VS!%d4^6 zD~3*;Qupbv_ukQCpP*1d1Rj6qM$t!fI?7w!Q_VK^{@A)WzP z(P|;S+p9x(p{u>8W7d_R0!`l*-Ua<#zP-?o3uAks0#RD|WL%}&okY(GAZmcPMlJLM z13Jy%q>!>1`ZMlpB?VvKgWTdixjP|fS8~!YiQaS1&0`XBvh6mV zzOMtVkUU5Qw%q;2O<;m4qL`i6eaB{SG%f*!G$DiwdbqBzYk|k7KZMlR;*#cA8T(jk z!`yT>cc~o4Ydtriub*e$=9EvR&U+3YBFZK@HT`4NPBB zj|0aqm`~++SrQ!eqt%0S=?Yt1J*TQ|t(Wx`G_dq1{cf)#;z8M*WB>L$8I$?Kc}X8) z`8laThhVy7TYX@L(w0p>lYny8fI4(*dV1V{Tp@dU$u~Qluz9Ljo$gmU>j$7zTpeiL zZLbjPRCteuo>p9AJNy1seF8E*@u@HAIi)ky?bL6oi1K`0X`iNJ!d;{m=l#b9{Qo`z zeGKp`pYhXG|35_VZ`|-#3j87qGGb$n)ZY^VKNBOFI0BTpL%Q?O0ypL9j&mjGNqcuj z9|=vC8HYrWI&Bo@6V_$W+az&W$Jc?>5HS_-58${=6EI{UUDxej@q1l<@$pJ@2+*0D zB2J~KJ%@8Z1H6gPd>WEW{`A#1!@BR5VH-6+J%)52HlRj9f8oq~4v$pON<&n%6`S)7k%h0J25zqqsce+77Oakc$E2<95LlyC_%0QNeYM2={pczvAXWZ*& z3DJ7GP}WJp0orpeo*9Rr=<*Dc?sw#CZU6x$E2QxzExqL%&~@(TS}n|fU(OOe0N9u) zm#d$57!XL?im@MPE{ExfYDgZF0HZkY;kOq6)oUaY256h^@)r00lqTq3MVTVRv*=lM zM3UMkb6Ohh05Xlg2Vr;pz3aGE9w1d#fl6PYfdR6{3CTc3i4zwLIgj*T*i|12I$R+x zBhs{!OPvp$-vC4a#lLktNzA7oKi}ctFdwK4ufAS~xn5rEZQzfqql{ zWW8%qG<63c1m#1qsXxU4dG*hVkjbNfaH7EvH2cRf0Usx&p5hRS&4@ISD(kw+Up*^JT zi}hSHuts?x4#_*AQ+vM=T*%hiKL`Wi7uc<$#tT z0R6Vj*C#sAk;Cka0pM?OACiyh8FTt_Px4tfGCj`noo>ea3~FqkYNWg4PybnM7&uF) zs6uG)Fd6Bu#_w0Wnh3%o5(omXI8^q6&K*SI(YlOlnI{#h<|p4F=0%w1Fzvw|1%phg z^)8k~Jmen!nE%YISCKG}a2IRurw*!8$E`z5!2GCNNhExzPVe2q>1yON`2V=tGCwRL zNjx3fJ_s#>m=HKp_l>dcJR&Zu*tBa>4%I+Z+|lp33Ht1j{0FbA*r3C?{eVKX-FgYl zi9vUGyMXVDr&Q&Bu_s_;z0upn!^QqQBAGLE_S9iJM+{IC5nS_t&^MsWVV78^vMb@o zrpH^pi@Oro-bVWS?!8gArmA`$f|DgE`3+12m=FbwDhkc26{@sDZO^NK=HOZBO^AFE zhL+54KgUlg8yJY!u4XY=o~x&pU_=MaY1KEN{dqtfNp&$t51aqI&t_{angrlZ9dC_ju|&KKuc?RYAbzys*y2nUlKIOlA!!v+_0p5bkeRYl(q)0O`Cf_F{>Q+`8^ySBA?p-2SPymF|-& zRjMDJWP)I7vhPH7_3C$x?VXjcmVfAVF6gGIH6KwSRk0fODrm_?|JC7+conK;MgY@-Mw*W!EK_wo5>Igm$UbaADRFc*79$y$0`#_!ApmI5tY6|~`p zV2&^u&?LNx`J%yS8uaAtlHdk|w2mU2`yO>ZlYI%+W=dKEjG%`Ku$XoLrokS{OG>=H&l9SBu^Jc;UkZ#Af0AKt| zFhh(4HD$Ab`)zo58&Ei#J$U$HX{*~L=1O3w}L@--6OB+5abiHPjq9l)c0T;%T z@c{8~ZLp>%D4EL|iE+4^2~DF^*V6ow)MFu_&A5!)uI1Y85SW$|od=$D(x}Vv(4&L# zl${iH8GkS=ILZLGF##c!%f1>Eye!82$b{Gh6e1k{aWEM@T5UVMV2>}Uzz4<=`40iP zkO+FJ2kr{c@r`w0%kyaf&Fbmh)&5*eA@RTg1GEPq{?-*1bbld5;BI1%q^Lp%7 zI@$pUI$|!Kwa#k_xMMPRxY*oo_-uvpU~#F zB_v83!AT>NEIa^^W0TgPi!#^p%Ruz`zU-{#Dy>#CAy%8QndkjGZKAyu<_|;AC^%3(`}3QYKYvqG@*N-@lY8PA@uyr>^**RBPAOT$ss3r`5JfXU z-*|79$dmC4Z<}n8E%YjrB4qc#jy%|WxmCN9`n&toM8)lG$GHW>M;|c6Z5K{}0yT6G z%pwofYpW0~Ulg$2Ck*y2SNo+4bcw>TNQh?T(-=$v^lE|)yT->IGbN`(uJjBcN9ipt zCAkU_FrM>$zE!8<{o@QFn?+sD2@W)RAcZ@c0noCzQ-m%*;+k8b6ro7&@5uk%uS9VM z2$My*=(GPl-7iNK<&1`?MSBGX67xX53dZIkACb@WK(h*Z3DX}e5c16nM&Irhrs6x*t}WMH(ettQVoQ?=7Z308~5-HQUfRePWj3)KbJtq+sm6;54qb4U3$Dz0QTAYle9ZmzSjLzxd z@0fO>j#33HOeZzc#@jZFH6YVQ&o|@yGZ3MP*@Pv)A`Gz@m;JPU#0mO1p;)hTpS8-6 z-%HNvRwp)ALWw$<@Laph=mi*$Z<4rc5FOvkXy2i+qF4m33a%mE{G(7(1+AbV#7 z%NWer;=p+qj-G1otENE`&|ngQ4I~kGHwU~yT6`o6n3RSj@Ro~y&?hxeI7jmOUU^i2 zajPVda$Z(?f)lz5ayU!5TQDaLyswxF^dVvKBWeCDPKfXT9qv&%4ZTja4GD^P)90%0 zsSK#Lv zRO&*=dG-txTk;7RS8xZ)*if72@WMVoe%5hl(Nh2)XdeRoI~imONtsVKOymo==uqEB zeCoY?*x$971q(d3D6c^uAg=SB4@dHBlP7l<4RhqP-`SvOph*W(5<#;l6Qu(HU*?kp zj=bPnoNz$xqJP4gZO#)3&611Iqj7*psFPjUR z$TfPM_@S^R2S5^Nn|1&S*BDxroQFh^Cv(9Tt%kEUcg3*szX~*>|5^LNM3mcbbq|JL zO&sHdpwC4jR2{9Bx4u4c-oF6$vPsSl)?ff-6>5jG9U;2}Mg5?Xp2&TVsjH)?=gF_G zvphj~nP+TEzsXw9;%Z<=inO}!D!LX766cN z5s9pG@b-=SuPnh!Hwd0MnJ5jD1qbFO{!d(N6-4%X{slW0C1r8nHdo97_gI1$Q+3uE? z7QTgH6Ch;op-)0zJa9O`S3(W78uo)>JA1%c8z&?f_)e-7KO2U=_(d*_mrId=!Ea}# z4$4fh3tb<@=z)`JtX*wmnG*$GS|j?Ne~vgTmn=7A=8@%a^-PU=sqQ2V)L!534-4N1 zoZ%Koz(+k1)|N4USMV%D4_s?n(~&&*Dey1#gYlhy;G^&V1%4a9R{TD1pf*P#Avj57 z?yaT!Y0uQ+%LFyBgwmWP8PY1tX`6~|W3-GKb!<-X zN1oUZjG576Ye0W*cDDCD&-?sf+;m9aYumK6`NE3B<%d#=e8Bv*SkG&_+4>})_i0av zI<-;BB8SGJqoJi0OQC)Q9j^1VZOtURddPCr(>sr#n(QoVdwZqj=z&_M+syx=>pj4+ z-v9q`blW9+?=3e%wz%E)2%(Ub6=m<471`fF>YkIbyd$!NHwxl zu!z16mfB4kyc^YC2SxPc|G)D!C9;^Rb=)&Zmb`_`@0&Bdf9j(8mG(f-XM`BWcH!^y zMWrEr!m?5eT$1)thUEWRh@UBwt1u^yJQZh8$u@b;X>nyLdHT(=tnB4>fpE#yg6g{d z5g_De;O}X`9&Pz4=(c8nJP?}VwOD5|M{H77EABQ9CGp*w1r1hWM2q2yS%Gkd{Cs* zrp3{GuKV=g+XV+D7LNgv!C8MEAbCP21%R)!*6vUXDqTh`@k4Lx_qkSxC18m6pQ6s5 z{1jPq$xrF{N52RumQhMQRJ`wh&UO4#?j5 zEL-4FBiHfp{|Vv>Z{vY5BliG9GdvWyp_~kzc$-NVGcdae8qd7$)_L{EsJ++t&tDHJ zD~jZr8vs=&PV;wN*#Tzfg;Tv;9B-8)fVmdW9K5};2Z1iUh z=BP7!M5cVU4B^^6XwDdyF^h&oJTyQkumgxk zX|&*Qs@H}2B;%qfLc_~kPFAG8ujFFZ==(fd(&e7prMrB3lF)-y_1pm~$-$%lsxY8j z1pXg(h5&LZYaKShRs#b zO+JFGCJ$3;$Q5cMaAgH2DCktwV&|164s5 z*!=E8`;nC+=l6N$N)lB6HRF~=u-^<*i&)BYizxo*;DwcR3+X@Z$js9RsXmav$XevW zu={ELYQe_05>EX1MqROw1HK+4_&L>v!uCI9{sH2V2RKTsQJ@d925~@0s(qF(dqH5y z+o&QqkmeCyxF-gP>EQ+j19JkQ)h{olfX2^Z0J!HmkZk62Johy2=KpZv`7@O=-h5LR zD+=hE6a=ExtR)bfVze8*kvYz^175!}(7|MF1>uDSJe8p+yt{P&Qef2Gldw{hj;2EiO!MF~W#!wg&F4xqKBC zNJ-;6d|qcQE6j7^pFtlvMkFfi+J=#)T$Pl96&NtJzU(_2%4~hJq?_mve&uFW5r6 z4P9v2(e|JLn}1Ux|0!^P9cUVScrwNMChO`sfMsr=n}m`SuRJuNvTFd=VVQ0zZ2WkZQ>)1L^rJ+buIYgM&*iKeNm1ezLuL zhU>nVW;XkBBv`wyng0qivLrMW&=@KTQr0oAm^T_xGDUt$g9|MOE;_a2%49O_*Peq$ zJ<%)8S!NczvH#jLfBrT9D)H=ts0epi!6*ohdU?S5!LD zfX$mEx<6V9JF0Nlj*LR2wCJ-2x=pZQ$Gig{kpB1EV(w5So6P|1_0c@^_s%>l$0Q9~ zK-wR{Xs#B1Ritr64Dk@C=rAp}fWXg11}NUx^m8S5obc8qM$K&jx_MYCE22w)x1_?p zfTl)ST_zd8Vs6pF(Te`yR47X$$1&qAIZgEsMODR%0e^68or)PQ}9U)lZY_YE)UaXw+>_tdVBDW8*G{PPc&ApwA5^>NC-_pwG~3*a+4NB|}bLH!63y!L9((y&k^ zRwa)8kE;^PlUz-Le=*}y1RkyvyT&Fp;$6UFn*9c}XU_Gfff7HPD$txHJFZ%VVyCj? zcsK0^dG1OGP_(O)l0?00R1V>hxD^1M@h*g=FgDo}sPprM(8%NNtRh_7Af*5$0zZP) zoNC9^6Pbr@=LBzctHocN)vjbOuBNuTQe>3u$&sn5aBc64o8liJ4_M3bh6pOFe-&~- zjj$M2G%^6E6aW6XCw1jl8gX1S_h%(r%$Xn8 zfn#%N5_8D|fDsV|KT?gS4H#~K%!R(Q2sxU4KB)Gq&*$rp$7*k^pv}#H{>XXl@ z?m+C_yYJf24FYnmELBk|(5n2HbbHA%z&)`QfzM#ue9%5v`Q%bb210%JpOPA%ywE9$ z^1O}FC9vPMv^cfq#9OB#oc!P?i`%_lt|fg421@&|fk^(BRiW_Nnk?x|x&!+hmxHgb zmQ3uw9PoH4+^dm?NQ@GlCq_?G|Hz6#3V)H8`@~)O1I~OC`R`xs?Rm7bfY=_xxe?C?6WtOX5^S>rQ3Rt|;AU2Na|9#+h&2q%1K+-9Bg5c!FCEnS+ z;L+!14v72!SXtd|*a-|BXYY&I$H}ZWZR?aX4Pk+kZ8VXZ{7MmZ9VEP6h$dqx z9c~eMILu$(*_x(4&9Z(`=b``ud|L&%0m*TX!w1n~sY?7uMXeCGYaalPTAKhyP*4 zP0GE|eX4D8GW3ja8Nq#{M+6U$w&&WtjooHmB>CP9~-9 zMkZaLnklzDQWY)KuJYa}#j% zIa#uSzu`WlRI-y2X=J|x0oBjf<^IHsE5~mYk1n!1>;j<4c;WUiipBK6X{h$vo+2FK zY9siT0RTFzhFr)R-h0`*4IA^5rannvsgkcEgRJ+H8p&iSav=nCi(vxD;K$b&!3Un( zP%gf7baXucBx;9!x+lH(4G2T6pQpq5;LDX+2?Ii9JnvNr2&=i`4dd^}YeQ}w)pu#4 zRYoWeSJ9G`LVEUHslO2Z4QSy?+n}<3#)r_T6$ijBTJrHG%X5!M$Mq1aEHzm&pI=%YWWn|)M-=dr6 z>~DmeD*bLPJaiHyvS;kRKYh1$l8@a!;yw$PvDwqLFG&iAj$W$L zsXe`#&{qN!1G;;oF?#+Vq#2&=tSs&Y=F~52d%4YUI^(Hw(kU4*JDNX(m`&fIU5Iad zZgw$XS>IeL)~0dkB|-1o1P0sZQ5Ly~G669|j9{&y)oxw)a+}$4;qG;+ckzths~cy6 zq+8`jfLB}tolR|ggnsYu)Xu>sQA$KmBo3N0WPWB4IDZt;HmuKQUJjU!ZO-FT}nXP8n*H2q$oV)xm_8Jk&GZv%!R-dT(T@}0jTfPg=b)FP? zcvml$Tq?QdpwZB}4(-Y372T=)8d{&%!)>w`-#)){r={XfOr`j0PNAxK$|7Lc?@nm- z6_sjrpcfY<9(}vm|L9%IGnM%iPjFWOA@L+H^JBJ(YaGcL);qbP4Wr-uXHEsJFTbQd zRewgE#m8vlgM>J7lnpuIUjdnaHFFFqR{>R0iytQkkZTZx6>`2;o1ixyxv8u4AP$`m z!CYeq1mBke;CRY`J33g(()uqFVUe33a<*_tG%tidHDWLZlvSE5g zCY3wiZvrNRb-hL*(i6q$%RI5vOqMAI*j3{o5FxnC3QOGMHq4EmdPMx@#zWo^8DStE zYiP-bzR%L5s?srwp|-WT_IL}$N@hMe!j=)8vh zi*`Rg>Eeh_bA?!?`2@=I6&HEi{RWetT_vjX=l%YkR4aS(N?DMF$TG<YiBh+{IL7kci(wLGbp8h0t<8@x^sVd>(Dh?4sv~iK!Qt1^b(0)DF{$t# zKP_S@qT;JLjN};^mwIlSQ`M7DcoB~fZ(7}atWPc{ie3!w+c&$sApn9r7qi%F3d}(( zmH1X15*T(o+-%-uQw~)}5GQaIX|1jy>fGIILF2M>VuF`flFezdO|%R06)M;jUU;Sp zF8m9k()UN5t1BNn2D@41^B0h(GY{-|jcXWnBj($oD?#wHD>VJXV}G*k?mhcnRrY`{ z*T1dn<0``03Q7Ms&q=3q1(6&fF{4}^C-B0}IT3E8e`_%ws|MOnf3y+w@>|%y+e9!RY(w~~5vINT(-N|bYdTP0^WZ{QOlG~ym~ zd?e!e^$@HdLF597cr=f-fD4^BOB=_5N!Eem#8eeYhV-cVYX(y51qeX3vFqpHiKpf~ zxWr%dOui%L9Vbu3=a1D6dg4CcjhyTwum$9Nd`Zpvd`a1hU%)GRVVSjiwfA-n!R?$z zGAT^%sfA1L7T7O0WOfKQ-#dtKb$Zt0Cn8UxCD?r}D#8*c`ZSyCPrQDkp~B3Z`);qt z&|tc|q*2}Qz^Sah^N)SbFeJCEJay|O3^f8GiW7MEK^%9t;W*KM+ul$X|Jhw|);}ug zssGp59TWEY80nPw$iG52rL@;9`>6Y=YknyBpp~x1i9VX_hQ^O{jr5y7yJPip_|K&qU%^*_Cay1mK|CU0Hgz8nNrPud3{ir#5tlTimEQsV+SAQ=Dvo8+awq_uT<6Hd-6t_}Y70VcpxQzb>|c;D zP?K7UIWORamm2arPOA5y1ZcRcaNb>gg@^Mvo|0oYu~<^J1iv^%kUJ-ysV^=~K-hm*P_ zM_(RYV7gYUj(Y?_o1Zdu==9NA!$o9EmT8ecX)Zjg2)6U&eX+TZ=~tJyp0fkSuiDk0 zd(Fo=BdbLo|7WJU-I6FZ;Yh8he_{^UL8}U)Q$L*W(p6^ETsiNRlAfD9@AxkN>gSC_ zkPdyaAYotGs)6W<5ZfF1<|kI6EGi3U7RBf{5oW6IoMe^%6!LHnYO!<)vD<`^YcEY2 zwczw-k6^jUI%}B_WCfi=&_~(Rej#P93$>aggvFuYPg*Uo>ycZ1@xF9gb;iq}u9fG@?78@;(h^Zs#RJrk+Goiojc!Z2Cf7( z51s?X*UTHYNa%O>l{0h>K{pxSmu^l0`l_m#?iYl9COHs7ks>)2(}t%P3%t9^;Rq8& z7ag7%^}d{8#-Q|Kx;q#JuC=%6ok^~8RI$X5RLK`ItJP7ho|{l}x8&F3t|XqsMq@ed zmOC|1<~{pD9P8lmwVjdO*n)7qVOcnu=Y-X}imVONj-fKtYki24$hqu0+m&)B60#D( z2w7JD#WPezsk`uaF4M{tZMgIlBmE3Xx;$jxyAVLS-FYednDj_->tdI9q+L%n*T=o) zVEM=Qyyhvpek{wqf16qFaG6URAJ-!EY5o&Q6?Spr#bY_~LG9NGTBBCia+Zwpl8laX zFZcw-4i6Ha#xK{wIK>%bc}-GyK?NlE*4*TrwX1@Ziic0tl=Svi_mgfqj#ynBNq(xU z>Dr?{KgNz~VEAvYIbGl2{@SjavKrlb7#V4{nMNG85f}Y9H?YUt>X>uR9F$J|tDZph zwSL{r4I;>3LMs5A^Y&uWX@I{{8*$h9$5lOCPHB6%3@QA6mv>r*&iU5a^KjKh^*FhF zvJ;3lN6w-@Wg1neaO|Dz{_0V$x|AWj>D8Tftaz+Ab*w%t;Z`Gv$I1RbzJ`eGUSY$d z5Dvp61T~#)ADA2I$j6a#6BzI~ewwCTft0{(^V|4JpwCk1p0&}txF<)ytflwK zHkcJm_c>|TVS+oh39_fC2L}H#nbUz05ebfs!t|r=Hu)+bqND;1mr-x>NX(B?*P-!hz+jw6W^(wGzk#W09AL zEMk~rNK{R54`Jw|@oyGlTH^c6#vW?aft=r&tIPpdtwM>5c5Hjt+Tb(@{u_OiGZ>q_ zz1ie6@IRhCwCYtO%HgUKY-hOG(uj{DHsc-pSO~_1En~wBIIQj_sL4fFbUCa?CNfDe zW`J8$m@&6>BDxBt7Y0!3_$s7t(w6b35U2czRXqmj<5*J_OO|k`1BPm zS*w@FBe!A8`Z;f39-fkkR^+ppVk^5#oJX}&5k`o2KX_ZWD{?z+K-77nY3w)PeVRV4 z$Fj7?#WYyqO3Xx&CY94On~t`mh&N<%eK`=S^N7f#!Taf)oJOtQnjtW;O>#>`cT0)$ z-gUh+2AA#jhkRzslMq|E9sT3X|5)qDOGvgxRShd!18{e{64cmtxC;+>`?Bk~xQhmS zHxF=MAAT1M5Ri5_c{Q9laIs)4Bv9grnsS3ULi;^8E2V$4dm@F6W6AT{IDfv1PwOwC z0DNl7*LuGIgYP)7BICezuUNjC%P>>>y!U$rgzsxea<6>^<*Cg1zGr3sYdwb%73E<@ z1gJG;4`3ExU?{cRz7*jVtyvpye8#``mvf%p_0DyM7Vd9NZt_y=K8E)?oWEUu^h)d{ z!0>}$#_D=*(I`1JMG@!#2SUoP6U4?wtp*eumWGzy#?Lwp^aH>^KSB~mAN#-m@-zvg zL)q^b0kv(WL`lN(pm>(3tr{5Zi%x8j;M1;pDk&KZAN<_v`zm7IBiCx!{mK|i!U1fB zj=$&e(O$C<`V+*x%xYXUzrqoHVk&`zgk;(^QsK;VgrZI3*gOmby&{X(-T%Ft!w~6H zsR2Lg71fK4SrVbwb8Vc#!U7tpzcWvt>D&4!hwBR^<}n9w>^@ce;tj0bU8e8Y00~vT zNKT8zvG5*JO)(c`evHPFrMtZRDrwF$H3%)lU9MJfR!U@*x9y|~#7K3zHm=S?Y~EVc zj(MOPL*4@!2C}E6h+Nib;5ezM%B6JXE|%{F3V^j3>7XHo59VYO8W(YTlPQ%Y9cnHEnMGTS)dPtlK41? zHm*2UAkNmo55%P1ncY_88C*&7!@d_FYc;q_p?W_Q!EM#ZP$H3IvBfb+r@E3Q(ks|Q zddQjb1CC(^dFlW#+uMMGPH41Hl~g?>T8Ap{@!KG>^u7Jp`_rUM5!57eH+era|BqKb z%NJgHk)K@q>w+$ku>8Wp^`A~7O0@OlZVH*y+@U$e?Gu{TyL5cbCJhi}*z7z1sS*0U zI)s0{kSjkX0k=#gvYwYsO=|)T#~pGEv?)G_>fOcv9MYAldh#bH;erJx0jMm?$YIb< z&$u=$J?dJRybm0QtYjKBow<9s(EiVvA3^;Rqjd-r49Ffor6sZU&E`4lhssNX(+4}LpU}%pA_3`TG zg|98!%AEiAWhR@Uko!4acF>v?^-sq6aGm@JH<-jwM&F{5b#p9?i|0R zRT!KVR4Q|5R|hy>>?Q8yJw-4kmBA%8k2DapSm*U@j?PEQmR`M z5SK-lTHD4t2scJhMvF7E= znH=f;|*G z`ePBzBF1nAKx8?;*^9kBytI>uR4YJw-*d{iOoI9ZkncVd;xS?-B|0$9el7*iO^JR1 zw&+Yxcq|oP1EMF22s;scy}g%{IZnW!TV@pMiv{Aq)e*&D$jF~$o$|tDZ~R~ zqR*M$&ooiA3w2omt!cWG_dTS7V17!o&ohlAL{#MCZ#%7^zLe$rGC|slIt*q3cWmKJ zz3jNw8Mb$!(8{;RtFL=EQK}7inyY4|l0=F>AV=TKkxQN=qOI*1nE}n*71!CI{B4wV zIjpfjKV9RJlGD*NbtLA$arH}^>VXZTb{1C;RjpAkG2er$R@V%_x>lUedZ?cTcdT$d z^))3W)o%W^74Oqa7w}iv!R%7*LjsVH9ZKzlb?flmQ>6N|Zo`}7C-<@!9nu)+r8NFp zeg=AB!JhQN6yIqFH{gh@2N{Ws{4QiZB!BQTiJ-L#T;dVzU&l1{z3~KrsT_85dTzk& zvL#^ei|(Q~X>ln4=r^^V-iL93am(DwM7ZB)?hR|e<>KptRNoK1a!y;4Z0W|_>^O)r zL$h)ZY@JLlt#4}9pP#AU80FN8+UhcXGfhdkN*b(?d%E8gd5cx)IQ8y`CUhIqzMQ-e z_-@z>>MZRvxiX17&BoQVzslOILO9}P!xFOqhMDngvgod^m|G1#Lp1g0~EK`Jv3!3#@kk#xSXqo+1gfte5)i+;Apne2zE!WOigB5sW^qs$50)7mI4%0=!bt#K!g1GO}1hML1RWpHf>gk^1LevYsyF~sSE~ek^A{j8fm3X9ik#cwI9MONZ zwXsy}1Lq3x%Il^{_7LR}9P|yq;D`?6kfK45UT^O?Se!yE6{-Q9Nc5A>>SqbdkC920VV2{ZvX(xP zSR^bv#g~DQBoY~;rk=jN9T6_t+4~B^@ZBaxlO@uDZ~0cA=)D#X=u96OZ8lHwtJ7Q7 z6;|vKn5ihYY~!7%f`ga$idRN&geWq3xB}9A`SwDM3Ti7GbD5vIN#Y5uxvkZ++(ZYO zM!Q{R;wvbJ!6(|125fh~2HB;R`i7+OCgDV2ah>pDiPGr85rEoL(LGN{5atlEquw$- z^ThN7l|SbI*cSlydRNNaEe96|NY#U_;Ozs{L1Njf-79AFHYu>kFNpSfwwQSa3Vw9A z-VGA$ofK`lPh1`u#rsy&pzY8L*o{`3Yrh+Mg&U>O_GA8`h;PoM=cd9Faa_Mur!*=AyihT3*>xA|PNiTxc zE55iks-_ay=3w}7kSC3s7tCnSK7wANF7u(R=HNFu5JTHzWn)WRkY%MIf{SLVWVTo8 zU^>yUyqT&kT+KJw}afW1)OD=+svugZwk6r7wqjegY z^dwHNoW>efao1Zulk3DVU@drf4CVS8w0p6^O9OZl*I5c3(gdxO!f1XNbn&PXd!_hc zr;o|nDHMPbe2(WDQ|~Rrt*gW%9*#pRYm&%0yURatg?~)cEAgjDwFzy&+>NT$r@4Q+ zM$7@#M*OZxJ}+k^*2Z>-mxW{wVM=IiK|Hn^0$^QCF>m}C@PNrviq5jjqvBj?9X|OZ zq7b>)4>_uSojZBRg>+$>6fO61qe7_%lv>TgB`BIsRjefOpv!{hls5vDbWflUDnB9S z7@JFaj*10`Am6!05(W}F6BhZW2zESE26>x8zsgOkAL$20p0VM`lcgtBqq)6JgjAeOjs=FGfUoaP}B1}B7r&TUjeiF)VVGxKhS+493`_^4_4ZF4y;TQeMFmE zB?Jx#)@iS+vab`{HDOytm;;x8HYwVA;Ox`8=9)ezH1+9R^i+k;%^SOB_>_wCcU%k+}gleED}JgCFrj+ zBNT=g>8^xa^!moOHTqh|iSP$Rmf^ag6U$-l*aVT!zDTV7yTbEc`q)Vy3R)}{q102P z13<5n@`hi=pB1_IlR^YV&0#r&acqXhxT;G!i)kC1J?SN}ujcgO35CzeqdH4fcD6r{ zEr@@k!;CTyQ^s>`w|a*p%6zM7!)x>MUPxJBqTVDZjU`L)w15-6ph+?_Gdy&TaZQF1AXD{PmE?!;PpuU4NW|UJ{Sn1n)o{INoT#+d!JjuKD)2 zH>4~HTS^&u7xNu6{~ULi7Cu{EVMS|yqGBp@@4b4Du1fOWCwGDZ(PMD^;i~@i^VEek zNt`p_?eIeYk;LclNorg4Ec2?!<(c#|Pxi5^@tAzI1Q3=i$EjP?MoQ!wLdy>u8jGiSBV%M4|yb*)(MCni0f>_BQo9#q! z_)gqLS2${7a#{{t*438Jjcn!IX8+LwJWaKjsaw?gyUS*`Kn; zyS+_1@-YZXa2rbOO2Opw^E7KuHZ5Wx_q^6*|n_5vJg|Qd{9|y6-Y!=^co~NwlS~0H|F$B>X+`sr>l-s9qm5fZiW0jG~J1*I~tQKQT5fgF^*#(+x_`9 z5S}(aet2B*ll!~Rx=jKe-PHe{KTfpqFggmz+wu;@cnr+O~O^-zz-jv9W6}8DXR_woZjC+6& z!?sdIJa&Dv@Tzs*>~M$@8y5!4RZzj}kQ7QwAUFa2NLlFd(TF-I$s?lUZt7X_n_Gf2 zMx%_Anna0%`vZ~=7w|n7v~5ppF`I4skPxs0kyxwKQ7I&~&!L|zM~Qt_Zq1{n9MYiV znlwG2{~h}g6y;MW)iKfUix2yX>s9is$5uOr!$c%aVx=IC7M}T_4!!jGg&Xg-|^069c(5nDWRgcO8R6CQTvpd+n1Pn5E z*iu;^Vm2Zo_CGt7*ChGp9^i+B$`2W*jTu`86w|E3WzUWC_nb(0l;&Gt#Gcnz@LQkw zid`!3-XKUz$Qg|Eclq}EReQf{O0+RI{VG)iH;QyngnF9ZP0g_7 z4GxcN9BC(GFxkw_R%QmS9y$T4X^K%%ihYsrgk61hZ`Aag{ly|0y%sYR#a37JB>(Hn zyJ=ncBKY&{nx}DT?S2=cX^ANRY#HS3_i8k@h zkn<+jWn*x56*NLGyLy)P?hvhdgb$eWjepV$t>8C5t!Tdsg5QFi1OonB={g$diPIZ>v-nm9=~oU_?t){uXGN9f*F>Daes{z7pb zhluzQNj#E(JBu_f(H?dK-(#;z(mvKI9ushvp3(YM)B|s8B@gkdEjWH1G{$vciu+RQ zp8XGE;kR53lQ1=&=~Mx^{rKnf$0&bHROp5s6-lS?dZe?gEn~?)XctD7sA9jh7V7Jj zt{m?lDX}g#LUDedZ=F_HG?kkfRF z_Z=YV(FWlTeQ-O|npnRrZW12$2m929YUL+DrF)z1UBu=mu%}eewLSJ+=ktCCu5Vi= zj%|&z!5=Rf*CQ-jp=PrUPMBCAa9p&*G3Iha<&i8l$B_N>6(Tw=ax;k8cQL(v0D?~! zpC|DS1o6N4HM^pcOZ`t{u$9 z6HP?fP@gIP3Ip?#pCAcXeS6z4^GtcAoniC%6N)y+*}}=2Q_6DoUw%~utZbUNRd&$& zO5bwh+q_+F3;W9y_H3?4!?FEP$HRh*EKEbWTj|=(HLi*3eM8EmDDR~5_7=+0u;EEi zo47;1c9Pi%rwgS#pxa>~jxJN3|^lk?tagb!_`oJ^W zu9jn-zvp^mHFpj)cGky?US=)^^gEu$Pa}>=BaY1-_{E?3yazR)E17g&u}rw%;2-uQ zzmzB+vw=0q4c;_i&m^ws)KN7TqJ~vDzw=zjV^Yb&PCZ)h?7*ivt zg|g8)&3KOghJ`$%KG9VctzXwgOnOr%Q=`scH3@{wVv`bi9%ttJMWGeX)4YlUR2(mS zQk>v)l+Lb67s`_H7P7X}A~9+~{ArmP#oRdJm8Es|)27|4qQzHi?|*r5P7dDiFnQMt zhX-q+#WMGw-CsiWOdpa33ZMeGrz`boXGrg8Lc^NsvM{p$mYO!)8Q-$8y2R;RxyBTrO=ecucGLGY%$=WCz(`14L{thhiVB*G2O&oYP=yz}!1K+21|<6e5& z{@MDrXY5e5I%b z7jif1OHlArgzFq0^AELLGC`^26XBMdY!L42yvF|N^WSIXlPHJttXYd(_&8}uF!3cL4hjvbaPQ;L zqU)b-Tt;o(#>sV|U*|pp0foL`Q*5Nk^urc@Fw#Y-uzw@cC`fj5FyE#LXJ+33*0uro z>UtjqQdGK3qVP!38cnL%<8mzbQsCD-P!ka9@e+cO1byLN>`&9r2bm1aEm-Kn zjVN0_Uk8Ej2zTyTz+M6kIN~=-k$bmYkk}|?>)U(%h-53S%IJ;^-DfPvEn~uZgCdo_ zDDh`9vvRmiQ~;ncP~D;?79Cpr^BQ9?qNx4X3cf3!>WU-cBJ1#d^#8)Au*kMlplxVN zR~fj}=H~nDrk|j;^8vWCdCm3kRpI*PzTER)(De)p$G^WF*TW@*c8AFlubFwmq*_5e ztCj-}!(h0U@lPYG!%g1nX%rW!BX7od z7g=X^bm1n0^w&64vA#LliI?HcRTDX9o4y%xM5ZNn*>Et5s;3ddgnw6zLzZ_{=megU zJzhaASWPyRzrpGM74hY>(Ft9dt=r`ZrARhzvqJySMOEcs&aI=Ui6LUg!z4YG@2^(| zOSqdnW-cd;6W7f+?Nm=n@=4Q=f7N7-`z4~mm>)(IQoLW1REcy%9S2|m`|{xd_Ca=Igv z(;ZZlWY4qrk6Mv}NfDlKq340>NeLr(KIbUdi@E{Bd+x*Y=mM3U#Qn43NT_;G>bI-Q z&tK}+Szag21+Zfc5{Ep|`JrGGxZ|#IfY$2F9dxkr8l=GkLCdRcR;#gdk2~V&8Ue^} zKu~Q^tTcZ0{qp?XhTlYhgCF?)%yJ*Y4^$ued)cZCM3b(NO?Addm+h}He^?uHKr!5F z{QOV-kq2uSvf9zO=(ZFHMi8)l`y_mh5Vv_l^$8bkLeCZbGwk3?5L_-3uT?Z3<3Y}2 zO?I=pR}q`qP4K5aIvb;Y1c;kA;+Zmo_&?|aEm8{Mz11{=WQ=95D`f+eeP02P_34F;3$$hs^*s*5Gz;2b_IyB)e%!t+gtwXYdC+9e zaAY@;&Yo-|ZLj7~R|Wsm!#k~edW!OFNaDD0u@=oJ$m49S>^wh1!AO=uRqe>!mEzN& zLI8_Eye<8%L&GP;F&I}$Ffl}NdE=z6ne5bS>Q`&=gu{1ezgGgE`ON(bbwX{du%CM? ziE2O}MEW?zeU2Gj2SPWq!W*=?{(m5$@G8!*%&bt}_V0I@rNOV~`QZ(Cue{O2gkB=< zR?YB-zasQ9Sn!r1PRZ&CTzC_*j0AF#*36!6g5SQg3QNfsh(V!g{2?eqoFs(QC!N8T z=!M+GO}bFTq4NHcvBlitU@t_0Fx&3Nxil-Xln;Gsi9MxCWXS>I3jX08B%I?U1cR{2 zTxRV#)>sJq18c&|fiD?d3@eFDNQA_B3dlP`)DSL<3IeMcX7g*SB-+bhL8$*TvyoL^ zZWk+uj0snuBgO(r-pj55@hxuSLtO4pn^4Tsg+yyyn9R|y_naG+%=G?^oTA?kj}wcS zdcxz{`P3_p(*INtQ;L>lB=z@>1Nqp=9E;{G&2FeM6fH#`lWC>b^o%2*ewL^ z@VGSyH!zF?&zCna_3RHGkL*@h8BTZ%jac;uj&@1sce|FU()wo4L+6$)X{3(oxsRh* zb&>NIgcVn)?ryQ_H5@j>T?Z@b?x4^dhw8%l^{S6Q44r0MR;8O2mT7R1!6Eq-kiu&j z_a@>}I~ZuGO(uywb@0W@R3nqA=L86u-NSs6E`_}Ah4?IIqPS|x0YBY)AcnwvCmx3u zg_bgAVF1vxMY8ih^nd0KS6C~*-HyfoWfI|s(?f*aCm-RcBfs7gd3Wmp0*Zt&N_+ASrIo8pHxIGk|w^7YOn=7qfIC3J?=Rp|8)_tF5B`+jf(OE!! zzd3(WsSf0L_TXX1@G2;rAqBgOEEXxV9X;PlCvdIhO`{DpC8B-)^DP_338qruV7?K) z)ls9mNO)MF*mFDT?DSR8uT4xZP5jSK*v+jHCMY$Z|UgNiOLVqt~NMbFq#xt5N|Fe;m8nc-KXJ2YwdM zXdIGxFs)qSoI@IdNAyV&a#&gmj}98LVnrDmB2l9eG4P-;xtZ>n-8&+BsginKKtxNq zGWfgAV%RXw`j?7ix`b12D7O;G&!7#Xj<=!q$QedI_K5v^iFcAGM`Jr5keq^^u!U;F z`lja&;T$-iU7kg%Y8XKz5(F1EXJ( zjWTFOLv4E$pmM|VAJ~QcD{8hN*|qJ46|9(cl%(spqYuc@8!1Qoe~`4&_n^bn;tiQy z$k14F-)Q_pkTa_Yrf$4OoKBSIZF$r}2-=fwMVXfN4k#W6tG|u_aU0LL>x&4)EIlVq zuNV?cp{?0O2?`6sQMj~43c_QfYXYvL)=#BTLjcIj3XS6Hy`wulOnHqh>!VKBD6Vp> zz{yu*mlI<5E40#IY1{<~hTFYTfW@~``Lhf4Ig!jQKWIeCk)&9G17Zt$?JmMFX3s8; z6ix*lrokpq7|B$9YYqUByg3+5`s02=fd5vb_75#r!p;uy2M!0;(JYsX6kPTo`6)Mh zLOGdS0(!Tq1&=zi0~-Hab*i;yf99><*8t^9q@EI=TBM|4Yg+>YiZ{})eGs)pwFVPq z$MEs^aAzw`!npm?E76#&Cjv<}WTYqIl6Sf2o6>wMa?ads=!lTy(g(tSfSK?F9`@5w z3Fjpkv*2tMrfeb2uaoYmoXJB>;fjGCp9;zw+Z-@nxPQ8$!hQ#T_3quw_ zs>BQp-Rpi3tC`?w_e^ypm!2H~TJ2-0`o&a89^)B%WCUqR;Mn0R)WL+e;4v_f25{5c zE2psAV`+2EPnn{`VsNy4Q-uce6u;$w%bk|>onY?ov&7qe5cF6Nt+weA*YU3z>44k% z z)|TU*!f#%rqmg#wuy@K@E9*>|6ZIu)d#pQ(e-@E+XcOF_djr$L%315%b@Rj&w+ZL% zGcB%to6&D6peH366}Kin(CzE=GhN?gMAHQi`t^arGD|FOy}%ik?YR!M3g+9A^jwJ3 zS#m1u1pLFf|P;bs@Sxs@=*(y?!?0KqkeJsUYMA!1{Giptq|{T~!82SrlO zSzOImBlHKJ(?5f!pyYm?vDGFyvI7aq#j@fSFL^hPmIzge`=d7`z}f4*vR{d}GD?Z>zy zb0*2v`GfQLswcGYv2+CZGbr?+rcDscfM4d59tJajwNdEk@aCrF<}-S@*`Si+_RSa4 zyOFkbQ%0#WsEt4#Lb_u@|6y9b*>#)vYx}KxElB2chIoRnei$U-Sf=oTZu7>dkzKUc zqqEB0?;sDl_X6z^YxgnxakcfjWomIXMbC0LAuEw2!`{tjTM-Ak8(Pz$q_-_8yT$)_ zC2h#(9%ULaupaFGM^nB@)TPY9%ECeglnEW)5vj*@+Uj2WNn-CFr1x#6e5>DymrDPl zsz>l-w#r8n2~Iuv&B!;0Lg~|?%n!r5QJW*Bt*ZGt^u_^+;cS!b;_A|SCX|1AJ$7iX z@;dRj_gGVV!GKzsvPM)r0?em6kwUJf$$E5@z6j7?)g4|Xj&F-eszHYi%+^y1aW9o#x_))mNhpE1QrGGB*nwy#lF?SCh+oIpD;g>I9zHYM8wMSwlRFQdQd?;i2&<#+e>Gp{d7C+#ds4iDo*^qf|*q)bbn>jOTluO66_&_cr+FgE+Q1 z%*Wtqy3KrG`};`ZCx!WvpR@IwR~g@daOzi9hb>n6X&2O@Zu;88yB^rI>*ONMnAMYTB<@!epFe=C@ z!u|iX_uk=D|Ns9uLdPhZ2H6}OkrpM6m01ocM`UZ7NgaC~nOR9Rtf-W9NcP^5j1(bz zCNtUNdw+UWuX?{f{eJ)bzSs4;K3!d|lQ`$`d_EuJK5pascG~WA;=#+nB@}t4);r-x zGDLuKvKmk;iFAIuSSV<$V-Lv|t4_i>LZ9CxlFJTyma_0MsXG};b zKB3M_#9dXc-+xoJ8MK%lMqRMub3DEC=WPr1l4T9zPac4n(CVaDI9V;!Md}NuGDoMQ zPMyr)HmTGf_E*NExUjY4ig|=Y%vC5g+^stj#w~)kMEC`1rEV)@1gmB*Zud%)li%|y zUl*%}?TJoc5^K69!cxdAz{!7tN^7NH z;0|$0Q~(ybj%lj^%+X15`J}r;^D>5nZ=C{{=ZLE7{dtjTq1@tfA2GC4Y4jEBUUDJi zPUSw|5AE$Xn=zvUCu2rA%XouTc_1ZPSBSo)GrMjs!bdtZh{xfKn>hCN6O@!$x{4fB zguVBIGF=2Bg56);4FHtk^~s$Ax;e>KZj=66G|qz?!pS)Mh;ec2H~1FEwAo)#`jEC) z)whd@$L@lIy)XZ27{eV_k1EEkG{bzT#(=3Tf_Qk;gUWY^%r{%vbc*zhbJ?PC!;Okf zYz}@gE<}g1jdXI zGln*7FYp=*5!aZ@UX&DCL0WZ)m{8#99F{7(tePZd$k}bqH^x2u^0>b67`r2`rT=rkg;&jPW%c@+D`H$VQtobJF|RrR z&W$#qa67CeB!M>wy+`Fy9y}XjIa3T3&}i&Ew{2D8%TF913^}qT5$^jAjU{5-G{IwCl@7&?KA$XvtMj0DUFP zr(newJb-4wiG~~U5z?}r#0}n&TRnTJQ`h~UOaCg&_?9lZ)>`e_+i$@yl6#&{rwIzH z7H$(PxC?EuE?Luk(S0l8$1!duf(A0wdm*`=S7Gt^{^s7mE#8<~nZBE+|7kn`x;g}w zs=@X014amKR8GiaFTT9`=mh~T0i))B2myDk$m3$3;Ri~R9DvZBscZ+<5&@h_GNp=Ea2#S7 zwroX2_>I~7a9R->@AANvBo}Di_hyOQ*HC6b0o&@Ob)H#6$)-$Hr>8>X?7P6Hey4I- z&eVb?U}X6j&2HdGGbx_$yBy!xujF866JuAtfdwWx2D6K9w?JZL-Bl>#MohfO>6Y@N zpb-WK1asAk`+oATlJAv*=Jn{JhuYJ32^^QxJZg7KLWctH|7dcdaXE6w&cs|S&AWvL z1P#^Kd>C~1G_Qx;xObR6u){3sVd^`&;{*YE<^G|a`~oeo#oG4Nyx-ZqzwX*=%^$f^ zkRUFjoq5QpH_?BO=m^tOr1w9|LjpKMr}=*OSK+~I?&-AMD8%%}CSzD_F%gb6TuTwI8=OhiB zX;s~cOm_B&q8h&qvvVIUYNOUcMs@@d;;s<6q*$dDtCTrVsuYMjFYrVlWXDW0Seoc1 z%divLQHQJDy~8gPMSs%6u%a|bel+>O@m4O4l;pTVBd^gl&spoF6JK&3MsR#WgUfk?!W*z0PGx zx5$T%4Ls70E9jFxjLk(>#2}2clz}6*cX}Ei&E|MUMwnah1Ph zGQTwr_S)v03v>!-u|-b$`r-QJHo)|h6^5Ub?j!Q4OI<1e_RUA?IE|=nw0qEq_2LJH zJxYm+0S6urU zgguH>Ycg>21}Lw2U)=+S=hUnMgdTNK^i5Z*;FrZxOG1>_z9Y z9a}8ag(P;!PER7rXU^juZ~}g0dr1&{#T{lfTnr$m6|>RPKPEiLfO>B>$u!ixUPo-C zhec&3;=Ut&c3EhDP(f6M!ROkWjXsqN!hT|o0co^z!H`9ZE<#9Em-dZ{jvZeDA-Xd8 z2Fuc3L{V=KasVAqP-|P0kj(>Aq0|IuB2a-z-QJjkSI#Wjw>BC{fW(AYd)B2>5d%;h z7l-G=^0j+nq}OE3VzenS!ewqGjLJkKhb2o3ZO1btsNS<`#mjx33eDo!+b9czz9hzNIneWddyVf#LE*?o9#khH9F}I+ z@9ax6cqP;#Z@2i?%`LX~3PiVbh?bZI>jMTyQ~I*Z^)=vg>V17ZSkJyojoQ6-ZpMmH zberHd%4=(DRNq0kRrDg9O|7oCMk>dlijJR#gK_=zKKqjGJtn?>P!zuJ1GBx#IzC@k zxX?a+=rCoYzbO#-pdoyzQAX~@_($>CB3p3Pr2@u>b8@KX`KG-##XyqVH_gqfmn?k!^!}>Y+-P$J^#4U8J-rKsh*xpiU!Gp>iQlBGIFO*{k03(Mo{O&+#D^{bq0$Lazvnl3)1F~LH*}Zg?)P{ZVd7Q( zFe*2KAKOcTjX}04vf;5I=vroG4wlA(jjuBY&i|HWB5}l2e$OQZ#8i*#TUlmwW!GE( zTTQ!K=^P&;JKRSYib=Y&^h|)Hm)k&A6{J22z}|sZ}vNn8-scM*@kXatlEwq~2Y^J7pabXtpWu2)6XY zZ8s|DBQdXTkQjSXG&$-gutqksl{`iZ=w>|G?2=;c0?SDsoHo&`9njMJO8KDHJ#j4P zCVpT7V7r1@VR}Kc)Wsx~*lB3M<|1`Fjnt$9HFND9p*)%zOx!E+8>>%N0J|%m#ibpONq#;;`eP8H-^)TdlUxnU%E<2y`FMz~Y}IU<|V{ zNyOPc6+A`>UGI43x<1gK%ntfEecw!LB6+dxW@V5iA5< zlb7FUzxFyPims2FY|u`M&dISkm-%eFDnH^}dgcLv%F|)!n$eHFvgHJHEXRb>=Al{y z6i>~E(06L>C~HCI*6Mz#{s0M+7n!_^jy*4?y5Zr`9Lm=$5_JdPQ3#Y%m?yE11ulfI z39gl2oNp9x+Hb0}y9%2n9y8@dWy5NYKg(yvKrPFP&%%0p3%X@y!toSyN0gQ-^0bRH z?A}~^(Fw=dz^n@1Y@kT+1jiE_F6IcwmmgN8` z@)Qj|#RpfatNU3p3L+#mtF<<|#N3Q2Svd&&rruu~Pm|wx+3g_3fheCwrT9jzBsY;6HfjcWX+617flQtMPQ}JnR%{NGpTZ#S@6>< z+nWjSZ%tDe!jQ8X8K-*PIkB8SsxQ(5^@olu-MLb^QNSL{`(aXd7|ru~Y)Gt2CjJV?er zgJ4!aPAa@Lin8YysQ8~m2Iwg_AqaniJH-Y2=T%<&xn|Y-r^Oz!h~4oMhAng}c@!;P zpC9v%xvfniAo{#4;h5=(knd9LZARainY_s8-eW>vEplqw<_)nITz8Z8Fm~_u#XGeW z#j99XPdK7_Q@v7-(Nf7WP8atXPdW}VPNJr$z96#l5&MyoL#r0FF4O6MD@J->iXR@;GL|IJDfG@rKo% z{gi?f-8%$CgxV;0y{bcYp1LQ(VaQR%TSqyIFWvf*2uOhUk^cSes(eFww7d`EK*HeL zDnM0)ji8c=NffE0x~dJv_xtXAP8I);E9DeUNPhbRji5q74C6^01Ueh%8(D(~X8eOx zX54OsV2A>S`sWTlq382bk!<7p*k+B1&g8`eQL{YBv>)@66hj49bW7cDY?b-)^pV@X zq980U#UL#Yjbz;u%Od?;`KvDl(%9Sf&~=QlmvcnCebA&TP&#GjWvkEK_3s-u}HHpr@}zlmjhJXr$l;{m~HZgOb*GRca{54pm>e+4k#F$dT{mO z1>rZ@pTgJIR_`O4XqAE(SeXAOK!V1%*=^ZVC+>se2&1B4jF<68wLpkM!Nbzw$|p3 zz8w}yGD^%&kp})$Lu7l@(7r&pR9%40C8i}}!}+Bwy(kL~D;ZBJB_`JsC6bB&;+~6R zn!nvxCALR8VD@H{H9=S9TRu;TO1hd{H^bGTu1kB6(o^^w$~m_;gRdTCKwU0M*t_Iy zVx1%F7enNbh!pUu5~MT#hpQ18l1ZrYI24Go+(qtblNnQcu>N%?;3Y&@q$ZTDdI{6{ z`3U>KYzcaCuRz>Rem&QqXsdq#gf@t*j6MLAzFq7+OXtFOz^0=WiTB?{&iBjoQfYf5 zg(Afbd%IiFsMxBFNL28M(E#zg+04TX0t4~be0v4^qyfDr_#$(5)0KbQEcjonb1??=&R127gSu=S96Kwp7Jv zI&-hemc3aA#h5M>&1n7vR711Lxai6Kb;qH!eFI|q9F>Xb@;FmV@W5fPx6g`Z{54kunT2Vg+0>pVyWYj7}Vm?-!kGQjwU2r8*8ab zY)6Tbp_S0+!jm!bb^ZxP5L68hA*I;k*DD@oqh7GvShEWFQ}l&qo~~Wt-GoWPD$-+V z`X=~6`2$czvFS|ElTLrA)1dkz#8BpWLc{pY6I{2V^_+W4z0&~(6|t1*<&MZmzm`vQ zzy~mhvbZU*=VlNu@D_WlEqmnn4VOuA+mU@a{S?Q0H4UKI+Z$2KSW}T6iBY4-ye2HV)2o?Gv=iys13v8?p z2YK)UoQj%V=ha9d{!|)NQ#6~w=!|JCB|#okrPBcq9IfwNE#ptP7{V>z&AzCE370Ti znjP|6KK`a`X=a2oosbH7z$2W@hIjF5JMJSvA+U4@?)spik7FbD55NVitv5{VN!Vv? zgSDyAaC3-WCg|n~0Uu>MSXnK~NJ%C$I)kr~vUyE{<(*G;_+ER;1xFi zT5j>y^W;AHgOwl?;jfsKi~yQ{ZD>DV%tn<0HKbu6h;qKl*^bR0^%BE=sdh&9EH7u5 z3dgy@9VW$UqDw<``x{Pe{x**joP}y!ny&>mJA=Q)Ldd3oyGe=SP&MZLS*QxYeqsz) zOthAM&hf{z|9uJk^Sbsjgi@BkuIT;IgZw$b{~U6HH9)d)haP+g{`YGCy=+S zlpH|-354=FqvrtvH?+|DYuVVEHITg>==dkZa@FjY2@QFN}*i zZsvoSkn1@RH916o&f+aU1=J7kH)I3$$rkJsuOXzjN1v`d`&Ial)iQ4F zsr8h09)hGo3c?ZYeTP9Q)d@@zM6z~MZ2`-ci7p8K1|Zu%GfK!}m~(w2c$Za!gqZvw z+G?FQw{v{~2c<6;Pd}Q}baK5m3)G$hDEAZqU{!UQ$Y2Qd{Vq@>Qj26&5i6CF3e8|p7Lu=9zAr{?-ywot)KG6ovoUBzFa4PDsFB**|ltK7s14q^4<=m}#U7?D)U5P7-22S3vBUuHL z`SnpWNT<<|Wa`}s4$?MICXW>n#;b8l+ncuLmp#N3B3Zk*abf#+$lI4wJhKr^R75!} z9k$;2wLllQVOI_d6)3@5#Y5VDC0?^5?zBcPLl9-*wwN}^g>CL1ae8)ib-Ln~Qo9lb z)fTIrKnir^cQbW@I!$MWB=?R*=R~a~;1XCHf)p7DUq_4Rmn*r$B-4s*D!5^X9L_uF z7N^g#fZ>E{7X-8wALcfNMBpxIXu*LYc>FOf+`(8yFK%|gd<4^90PO9)1!KFlEI3N@ zp=x~Os*#;CFoZ$wkn5CXvz+ae8UiWX*1eDiNmY9%n3**jFF7EJf{W5m`zbv&-$H)(7at`0lVe0PL5tXMt3({zX|ge=~<7 zVDxPtcb*DVPxJ&RrY-Q#CSf`{A-zb0MsBFc`hD3dj|gRK*IWjj-u&Hmh0WWSrcOgC z2jh!^Kx{+o@BCc1p8;E!Vh=1Bg0(`4bDX{l2Lv>9^I=*zwH<`FQv28`PL~!)3+>KQRvb zLg(-{loqL<|AI#fs3c6Y97z?@Q%8=0y3kT+o!O z@$$#?OKIJVX_UU|-^;DO8&GO!klF2GcLo0Q-N-WyF|(M=X?et=JlC|mX?6!sqX4@z zm_!&|eq7o4SiIbAQ?m%*i{VdtA@B|8{EqPcu7m7JuGMUxX`$Z|2-1CIBry2#ya#Tl z0C5DsW|(=SuZ0I_QTjiz0N>uJbd+QLR*e8<7RJdh%gTEP{I$P8HXU8-vB~^~pc9yV za1tMw)6<7+<}Hdu`XZiSpM$hmW52%$RpwFiQh5uaYwdewQDA-+)!V0A@{1QU(YwAk zMKLwv8Tp9yj5|!tXldP_VqZKTD3KnwI)xTr??ppCx(7JuzhAr~Up1AXT0kI8Ju48cj2E5pII5W5K5?OlBs^-o^*Wuf>YmW~78#|PuOlMAd zJR$TPU6kuDx$SB#V0^r=FRaI;ML&KEkMV}!rTrf!s8)nztuTm(;*i|*9!fboMOnG- zfzWVWRNiaUL*?9LIC51dg1YEuh)cK8u3$G9+M|o!$rxge+A8=nkp2C>YM~gl_grdy z8o}RPs7zTG?9~`s8m1JlCUefB@2qejFF<%-CO_Z(yOQehUNBI}YJ*KsR5aq^%FIWB zCuE6Ec&#uYL@L_P$rXToNnD{{kIvTePaNys-b}1=V&;DJ0-wQmHbQYl_j8=xDxj1c z1>CmN^+|%A&E>$bg^nu{M>z+Tt`3t(c7WH9zRwqPp$|yjuL{p{($OukK}ldqvEVK! zrQ``O7AfxuB5?wn`|&7hs_nv$K<)R6jub!Ed>>JwUCzuP)+rq(3l{h}a^$A@vMdyz zKXoE1l4(>+(=D13ehk2|7Et!uv4IFQKW{iM$$ecbXZh`*{ZD)z5sL#Cf^#mgYwG z2!gHa2>yDctIzzHfAU=*8rin)4XU57znB&`b!!!d=sEY5R6>E{xY zSRelTYk+-XDVYWBThxU=9diOBnn_5X!!TN}uniaH{r#p!5m`YCvj^>K3j8ScVprhi zBSFzSxfSaXHukKfZQ&Y^*n2W?J{PX ze2**#j)%~9xE`L&74A0%B)CPLFn=ESXu}ig0uc!}wgt%F3IO%pvBr55&0Wa2J{w~Z z>Od4Lglg=X?YFZDb{o6jaPz3|H%q`>pMLO7hEqC}?>ux-Cm}^ANy*j={0O1|okz7< zkV6ZSj~I~~J{v+BEIVYbCfh(|UBl_yWo=3gczd!JEt*H zR?K7h8!1;kkbC|4(Hh>_26{!_CDGt#qYq%W0zNA@=TIFXojjPl7TF&V5=;ZMe;y}5 zhPiLN3%a-wM?zvVpv%}t1?RoH0!ZWrw;l!Z7%lr=qaCROJ%IOkdo*RGOB-={if89_M!w#u#Kh0)o?1V$ zuFAngoTyWXC?I8$ZHHLFkFFNWnJ-^@iOo_PA)9h7m z4WQb9Dc0cAYElmeemtV$XCNNQZwY~Mwm6@!f> zOn0v}OBj=xHPWUTApssDA9DZBFcQ$KFOJol&*Xt^keWOmocB6yCZSJq&3$<;gYxd% z3!q4V$V}$swnmg+B2?hPX6F*Gvdi44*&1j-Q&cj)hM3tF_#&4^&z*pqSwsi#t=+4uhGgH!-vj|l<`1$@A0RPYC13dETjFaNlVaKahlr&uJ4qd( zPZC~iz-f**Rti{^UmcO(5XsqRsaUgaWLb{ip>$-@b1de5P6&hH9Yh0*3V1Li8~mZ0 ze(w0_Or30LL?>+D&ZF0@Ba--!he(F`Bn&;B;uc$l5S=yEw~Z_Wxy$T#y0>__yRXb2 z6l+Ql9EJIs<_5%Q+G2>|)m#y(!UQ(}(#t@o(q#ZS|qW)aIYtzPe@+aaFw_H!rOOkQA^ zOP$CQ<^EeNJq6I~-u~QJU5COxZ?Qj^i~VCItMHi>C>I%LnbOW9&O0Az17kq%;eikP z?JNC|)xHi|_vFtzqo+wp%d!Z@yyEGOkk@gw5%8qz))1Fh-I(8?>ys*L#M|&yUar2! zCZ;1nP+bMI)%iF2Ol+CHTNQzlQ0pP(*8sM|)gvwqAgri=| z4tZ_aPEsZc+_@8geTITL*maNZk|pL768knmG!6nrUsdRSV z#V1O*O2+cQ%6NcHiDd^Xv&eFEhSUD4y!^~)_EBDK8_bCb94Emv7-r^?JD3S5q>d}? zkp~=AjWZ*dEZClRBBj}55YHfB`-t|wN=8pWx*0(`f3r-0gyygK_8Ktc>EDz7g4V}J z37njQF2MOtL{JBI1q+9EZbGkj)X4`_$B5tby8AbT3nL#D);wB&74Kzo?SfEhKJ!sCyQr@i7_qEAv6T@L$bE45O4n$eQw#Vt_9ESmtO?dI*5 z;(z95Ka;dU3{?KF&bMAR=L;{Q1maJO@6d&GQ5I4rx13w9V0uMnoU*Neb%j`k9hjhG zRHj0`;E{y_!X@x4_qUwddD3I>W#Md)!Pm%uJIdFA!0_4zy`$In;0(O9QrEjZHT3cx z@sVOK``GisDnljL6hr4Q)wN)-Tkon@V}HBBu&|eT*FcIF;xqo(Xxhv#>4SEg9BUSw z=3xRBLR7~I{E5Q=w}?vTjmkYYd5O56Hf;ZDEM>!gHR%BTtc#AKvic6ZTVR$D?_*IH zB7%{oB1nwvP_@3AMkxB=oR-%OnBm6+0ywr~22YR>Yk?DEQn2gv+PZ3^=-EGqN%x}` zkebsmIqP4@DFf;zxNLma|w%Jy~X z7d8^Ek;0EJMv)kgx#rmxuc?S>r?X9NBkG?8x=++hx!9b<=X5H=cf*PB-Xm@ORC8FzKQ%|S2Uw`!Ot_*O5B@Az4(^AiJ-hVv$|M9= zLyBI?_XR#$%^-qB64&W5n7%TvLdXHDqP3IB=!lgvB_Q>L5KPl(XUYN4Yy&{{ZK=%V ztjH;y{Z!2uMO;_!&?t0b&UvheM_cyYGTH)9#k&&KntrY`>i!^iVD$8TcTkqvC-IQr z%S)e)q$IF2M~FI-QBK7dspy6xou z+vM=rx0SBhh?PB^k()vwLdIOg6-18;7(jrvwa*Q8umwz#YNP|MOe$$Ho&N$59(&?D z9wQp{)0$$K-aa#hcr+s^Z6WbU9DsKmPvD(UJ-0Dy9Am>Z8fw%0r&>P>b+q_5Id&>uqB(!ql}6}ntdbqHX=yC zq!^B%3vdos#YhRBFjtSuvbb#!3>+k;7gR;BZtyr9GrK+QA=prD1;o6B0MDU?7CtF zrCd#gE3qFa>zHN9@$G!d+DS5eD0hW+F7~KH*6p}w>Jb)R%8jt!I}KjIu1e!a`Ex2j z$a9K>uMSQ;FbiQ;r+F_J-*VaZE0i(EBtM4WS&D8HN-tCl!gFyl;o9V7bUuS01$A0k z+XA&czB+QS>KoUn@SLI{H8nVe_{q|BqtQ_JbD(?f+tLfwA^X+%JvX`>R30OgmQ!+P zK;^eKV|x4g%Qtit?nJrr@*?^w1<6w+4KjbZLF z)>No3g?$y`pfnrHWgw1Ah9MSBig_xct5e(R=nCH;+76wo4qw9Mw`#+2C3ynk%7P(x zq#ey^i0}2ux&Jfs|8-v01p(clE+}jHA4v_&r#TpQ1Z!{$euVm{z#ubSuz>q<(KDDd zLm>=}F)bkNLIt;M`zSuQz zwAfkV<%(DQ_i+9zI6x0;nJL9W)OEj&L!&bi32#q`ko)rFOy{M7+PM0?)0;xFzqd%# z?qT~*Cnw+6QQ1t+y|1xs+DR~gJpsi2ev@jI{0Rr#_Znuu?RR+;GtG-Nzr_`{t(^Cc zWpiYbUPe?IM0DQ=%@B3<@f7duThONm5blAZ{&Zw+Yq00i<=>x z#Buj!%>$aDYM~!>kzajI5H*9U>a5(BF9`1vMQ#^pavht+lR9OA8MdA3pu5b ztf-=GczA2g{K;JNJ!}GvKyhN4-5+)60P6(fdD@wb_Y+Do>oXZszJ2C(4$@zMq5aQl z`~53VBO=#xLD=X$BFE@SL1b0bhzy7!rb6uJem`jNK7kSgNoFL^Ct(ecxm~Qa( zI#cal3dCh%8v){@PY0(dRb3(C#p|9okA8$k(ebJysuHTkO+8969lDL)f9M{P$R8MGH7xs-k*=JddruSnh2u#n);G9aq1G9^B6d;i^J+aq-# zXj^yh`v(8mDF5xwIo>cgqjjx4&#N~#oF7GjpN+XZH7KVU$v(e#8)Yfu4YE7!^esib z=Zs@Wl~a%$iGP81$E#ZN707l*jiD;1dEN~gL(+#>X$AV|h3Dqzo^Lu#jJ%Cy>7GFB zK06Pal@s}D+ROSk5dDNsuq@%TLcL!Nd9fIO$>PBjtObUecg57VlSdHS;*!F zP*d?#G|dri=Ru}Ul+HDX1UcR$V*h2LFzsjr4GV#ItmVi?_a@39F*&!_ht9gJ=PvJD z6WEInbc1bO8%TVp5d<;c*ZP;kd6j5C>;!p~0yqWSS^SKln`F_w18G?xMG)R08uwB% ziEWV?LYZoA;5~TXzY2QH$gasXMrTDCUjXKFH4xMrK|3X-cBZB96C@NbmJ#Y<%YoaG z`Epa#aokI20ajJ!yfd>AVZ)gJq2D?t1QI^^G|!P*X)X%@na3eLmtOC!Mj&x< zVv|F1gjK}QI>XOqP@Mo+2)_5F%iKfdFiNgZHM~w^aN#t5XrTq^5tgfP3+m|LW))}>{4uT$YM|(3w&7QW-!jv!gJpyZ zK>^Wyf;CZp5Mno@gI`AA#FAN7!r>OPr(1jXzx3V8aA+#@={A9k_a?zZFK{6Lz+gQG z+>*dlOD8sF-3=&{9oh=oVas}#r~hMe!#@qQV18rdoOk7;2Bo~A(b_SG7)v0O)&~#2 zz-j*#rGEgN^}gvnHOk9u00ptM84!j3Ph*I!y8jwOsNd3&Dx9g_y`8~dJSW8CJ}B&` zi%=t_K+uS>YiK@fb_Z&hmNsXepJ@QFLD`KV+;)IdHA6kDft4}}oGgl*4TwtnR77P) zz&p7goER_Fv9PkE=r#~yq(hS4C%}jLp6)M}hoei&yT3-ktix|;ZtT6d8*|W2J(@Ze z%JD_u@g_g}#Qgdjlojr^(cY16%y@Sta92ZSIi$u}O#3(V;(9O@34Q-|lYV9_Kt5n@ zPfCD4Myd(#AUes6^D^nu9GW0z2NY~TqTS(>V}faP!Kfd)r5W#;|!hrCqZz600O`4C0Kw?7D{dA$ThC|3t@|>W|y|gk|29D!u zrzAeT$1IVdEQq=ab;L}0JH*1O4`KB&yoyC%Lz4o34^T)YGPm7c8%< zR4p-IUXEtkIdXIY^JS`5%S`de<3@boXu^34N)#hhw#mphUr1gd_byL6+Ph4NN)weV z`k-~h%-Pxb!?#thQWq2_l;w)FI$n_CYGD4}gGc`Q?gKfZ>#>(6<*!>NL#`$J-#+^L zyTRLff1*LLp1&|vM~PbX!@7iGkLV_lGBam6_#Y^m?(j}YNtyq|aLK>i#gBGOq3Qi+ zn)h4vwR4QNpR#g(^>}Nj=i2h?b9k3|e?z)EZTw<d3HA&AlL8NSH}-o*UJ< z0%bnG`GNj^^Oa`L{~Y{x4n}lZNuC%p59zq~9CjEnc7Fub;g5x$8y>gzOFzWQQ-Chh z8k2p>1=p^E8!zJnJ?{&yg#z28Xw5dip|LS%V|k+Wa*0-sOTOdCNyDDL3jh0a8{RO_ z^$cH@|M_{PvM3bmaH8f%H7~3q=p79YzPbHE*+p^&p#4u7V*;@*7Av4xG4UFTTwN;@ zVW1T8398nQme$u?Z^w1ogAZP8tIYG@#}9Wx1=7audz7tRRNU@}>o=1PkEs-VexK&j zoBzk-#+n`^U)Y8}QN2LJh(6g1ZP^BZtKWld`cRHtk8LQYOpBJCuWpyQ_|wbYtfJO{ z7tM1TKXcpGGs+l@F>K)zZ=b%=@m;d$P77$w-3RD{Rl{KkUwO~#+lceah8>(Ll6fI2vKXZ@^fQQ0^TDUM?6 zcFv3GMtLpk2i@~+J7jJb-`&Ic!1h*F^Z>Ny8{pjOejb{w~qL=8c9(kT^M z7&f{InXnfcj~v|j&%FLMZ^DInaJAp|-y`$hiAp|vR`roi!Kpsb(LS;Kd~&X%#M2Xz zsyz=yp^0bNc0F>wAMz{(yjeMud#Z+phUULROIN6k5h~-NENLQtJx(zku-yngr}p61E}p)Dg1FODN(Hw zJh-PNHKGEz*Vv91l0^sK)`W52hic60?*KZ;UR#8D_iS9hYn4XJ(_~?yvrW#?%vcHf zdpYKRZU9&weQZLw*hN`#iYmB4=bzZUi-RI+jFSQC&*39`hW9SY_0&Yv!QVEJUpIR5 zMFlO2_2&13NB{d*H^1{#5u1Qh+0FK!SM&4zZn80=>Bp~J|BoU5ymp+Q7dA2Ikf_g( z&Fjz4rbN+;@!)oRZ2S7>l>hnL*aTW~Z$&m6J<7j~ZpRsHf@IpuPc%P=^XF>CvcdKv z+VK?im(fY4d11})Ca+lj-CchlptmT@|I=AIOn(`jDITT=srbC`@4xihNKeCF@$BLw zzCQ=`>rRxCV032g4JUtH?w>E7F*2g{nvbje&(}ugZ3a#n70dS?{(8edU+D3|d`r3& z^gmx4ImQ1658fg;C76tC`#yDL1-<{rNkEeRItRCrC$hA>Trv6T2aP}P1abp;qP)0h z6FXku|MD!GPLyu505h&J;W7VT zp6NkYE|Sy!!ylze)*3>_O}^#j1I->LVb|yw{QJ^olkXOFK5kllB;?A3-1^s1C zunO|v(9gKomHr*YA3s?K3nJR0$yoOc(SNV#j~~EA!K8Y69Z&z;CI9{d=u@zw{`d0#{`5hQw zzmMGe8CCUw!-wx8|88;E2AFoOUFTl^ZRN&Me#MQ_v>f>D#;5I-+;!{BhM9l7){P@` z$L#!5cKkcfc|u^Co1J}u`^!UZItLrg=XYv<`ESPK*aXJMA7};tas+&aDff9IMBp#~ zZ3aQZ|9<#y`~J_B|9|}z z;{WYzwuB@kljg*KFNA--^92fatvwcOe+xpby+Pbcug_wpfzyQy{-~>*R8Chk_4z+$ Cnq$2H From 5bc973411179c8633e0d442174a21d77eee3b0dd Mon Sep 17 00:00:00 2001 From: Jan Kaspar <2270833+jankaspar@users.noreply.github.com> Date: Mon, 1 Jul 2019 16:49:18 +0100 Subject: [PATCH 05/10] Move files around. --- cmd/api/main.go | 4 ++-- {cmd/api/jobs => internal/repository}/jobs.go | 16 +++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) rename {cmd/api/jobs => internal/repository}/jobs.go (71%) diff --git a/cmd/api/main.go b/cmd/api/main.go index a191d425096..3356d465ae4 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -6,8 +6,8 @@ import ( "github.com/gin-gonic/gin" "github.com/go-redis/redis" - j "github.com/G-Research/k8s-batch/cmd/api/jobs" "github.com/G-Research/k8s-batch/internal/model" + "github.com/G-Research/k8s-batch/internal/repository" ) func main() { @@ -26,7 +26,7 @@ func main() { sendError(c, err) return } - err = j.AddJobs(db, jobs) + err = repository.AddJobs(db, jobs) if err != nil { sendError(c, err) return diff --git a/cmd/api/jobs/jobs.go b/internal/repository/jobs.go similarity index 71% rename from cmd/api/jobs/jobs.go rename to internal/repository/jobs.go index d09e748df71..6aab473a50a 100644 --- a/cmd/api/jobs/jobs.go +++ b/internal/repository/jobs.go @@ -1,4 +1,4 @@ -package jobs +package repository import ( "fmt" @@ -11,7 +11,7 @@ import ( ) const jobObjectPrefix = "job:" -const queuPerfix = "Job:Queue:" +const queuePrefix = "Job:Queue:" func AddJobs(db *redis.Client, requets []model.JobRequest) error { @@ -20,13 +20,11 @@ func AddJobs(db *redis.Client, requets []model.JobRequest) error { job := createJob(&request) - pipe.ZAdd(queuPerfix+job.Queue, redis.Z{ + pipe.ZAdd(queuePrefix+job.Queue, redis.Z{ Member: job.Id, Score: job.Priority}) saveJobObject(pipe, job) - db.HMSet(jobObjectPrefix+job.Id, nil) - fmt.Println(job) } _, e := pipe.Exec() @@ -42,7 +40,7 @@ func createJob(jobRequest *model.JobRequest) *model.Job { Status: model.Queued, Priority: jobRequest.Priority, - Resource: model.ComputeResource{}, // TODO + Resource: model.ComputeResource{}, // todo PodSpec: jobRequest.PodSpec, Created: time.Now(), @@ -50,9 +48,9 @@ func createJob(jobRequest *model.JobRequest) *model.Job { return &j } -func saveJobObject(db redis.Cmdable, job *model.Job) { - db.HMSet(jobObjectPrefix+job.Id, map[string]interface{}{ - "queue" : job.Queue +func saveJobObject(db redis.Cmdable, job *model.Job) { + db.HMSet(jobObjectPrefix+job.Id, map[string]interface{}{ + "queue": job.Queue, // ... TODO }) } From 856b4f4f862399b140be3efe60c1772971399420 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Thu, 4 Jul 2019 10:28:31 +0100 Subject: [PATCH 06/10] Initial commit of executor code --- .circleci/config.yml | 17 +++++ api-requests.rest | 2 + cmd/executor/main.go | 171 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 13 ++-- go.sum | 82 +++++++++++++-------- 5 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 cmd/executor/main.go diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..52398c4e1a9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,17 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/golang:1.9 + + #### TEMPLATE_NOTE: go expects specific checkout path representing url + #### expecting it in the form of + #### /go/src/github.com/circleci/go-tool + #### /go/src/bitbucket.org/circleci/go-tool + working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} + steps: + - checkout + + # specify any bash command here prefixed with `run: ` + - run: go get -v -t -d ./... + - run: go cmd/api/main.go \ No newline at end of file diff --git a/api-requests.rest b/api-requests.rest index 166f3c665d0..9f45cdd7463 100644 --- a/api-requests.rest +++ b/api-requests.rest @@ -8,3 +8,5 @@ content-type: application/json } }] +### + diff --git a/cmd/executor/main.go b/cmd/executor/main.go new file mode 100644 index 00000000000..e06a131d5f2 --- /dev/null +++ b/cmd/executor/main.go @@ -0,0 +1,171 @@ +package main + +import ( + "fmt" + "github.com/oklog/ulid" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "log" + "math/rand" + "strings" + "time" +) + +func main() { + //config, err := rest.InClusterConfig() + + kubeconfig := "/home/jamesmu/.kube/kind-config-kind" + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + + //rules := clientcmd.NewDefaultClientConfigLoadingRules() + //overrides := &clientcmd.ConfigOverrides{} + //config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() + + if err != nil { + fmt.Println("Error loading config") + } + + clientset, err := kubernetes.NewForConfig(config) + + if err != nil { + fmt.Println("Error loading kubernetes client") + } + + //tweakOptionsFunc := func(options *metav1.ListOptions) { + // options.LabelSelector = "node-role.kubernetes.io/master" + //} + //tweakOptions := informers.WithTweakListOptions(tweakOptionsFunc) + //factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0, tweakOptions) + + factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0) + + terminationGracePeriod := int64(0) + + t := time.Now() + entropy := ulid.Monotonic(rand.New(rand.NewSource(t.UnixNano())), 0) + + pod := v1.Pod { + ObjectMeta: metav1.ObjectMeta { + Name: "test-" + strings.ToLower(ulid.MustNew(ulid.Timestamp(t), entropy).String()), + }, + Spec: v1.PodSpec { + TerminationGracePeriodSeconds: &terminationGracePeriod, + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container { + { + Name: "sleeper", + Image: "mcr.microsoft.com/dotnet/core/runtime:2.2", + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string { + "sleep", "20s", + }, + }, + }, + }, + } + + result, err := clientset.CoreV1().Pods("default").Create(&pod) + + if err != nil { + fmt.Printf("Failed creating pod %s\n", pod.ObjectMeta.Name) + } else { + fmt.Printf("Created pod %s\n", result.ObjectMeta.Name) + } + + + //createdPod, err := clientset.CoreV1().Pods("default").Create() + + + podWatcher := factory.Core().V1().Pods() + + stopper := make(chan struct{}) + defer close(stopper) + + + defer runtime.HandleCrash() + podWatcher.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + pod, ok := obj.(*v1.Pod) + if !ok { + log.Println("this is not a pod") + return + } + fmt.Println("Added " + pod.ObjectMeta.Name + " status " + string(pod.Status.Phase)) + }, + + UpdateFunc: func(oldObj, newObj interface{}) { + oldPod, ok := oldObj.(*v1.Pod) + newPod, ok := newObj.(*v1.Pod) + if !ok { + log.Println("this is not a pod") + return + } + if oldPod.Status.Phase != newPod.Status.Phase { + fmt.Println("Updated " + newPod.ObjectMeta.Name + " to status " + string(newPod.Status.Phase)) + if strings.HasPrefix(pod.Name, "test") && (newPod.Status.Phase == v1.PodSucceeded || newPod.Status.Phase == v1.PodFailed) { + err := clientset.CoreV1().Pods(newPod.Namespace).Delete(newPod.Name, nil) + + if err != nil { + log.Println("Failed deleting " + pod.Name) + return + } + } + } else { + fmt.Println("Updated " + newPod.ObjectMeta.Name + " with no change to status " + string(newPod.Status.Phase)) + } + }, + + DeleteFunc:func(obj interface{}) { + pod, ok := obj.(*v1.Pod) + if !ok { + log.Println("this is not a pod") + return + } + + fmt.Println("Deleted " + pod.ObjectMeta.Name) + }, + }) + + startingPosition := podWatcher.Informer().LastSyncResourceVersion() + fmt.Println("Starting position " + startingPosition) + + podCache:= podWatcher.Lister() + + nodeInformer := factory.Core().V1().Nodes().Lister() + factory.Start(stopper) + + for { + time.Sleep(5 * time.Second) + _, err := nodeInformer.List(labels.Everything()) + if err != nil { + fmt.Println("Error getting node information") + } + + allPods, err := podCache.List(labels.Everything()) + if err != nil { + fmt.Println("Error getting pod information") + } + //totalCpu := resource.Quantity{} + //for _, pod := range allPods { + // fmt.Println(pod.Spec.Containers[0].Resources.Requests.Cpu().AsDec()) + // podCpu := pod.Spec.Containers[0].Resources.Requests.Cpu() + // totalCpu.Add(*podCpu) + //} + // + //fmt.Println(totalCpu.AsDec()) + + + fmt.Printf("Total count of pods is %d\n", len(allPods)) + //for _, node := range nodes { + // fmt.Println("Node " + node.Name) + // fmt.Printf("Node %d\n", node.Status.Allocatable.Cpu().AsDec()) + //} + } +} diff --git a/go.mod b/go.mod index 94585f21b7c..bee0dd19e53 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,15 @@ go 1.12 require ( github.com/gin-gonic/gin v1.4.0 github.com/go-redis/redis v6.15.2+incompatible + github.com/imdario/mergo v0.3.7 // indirect github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a - github.com/kr/pretty v0.1.0 // indirect - github.com/mediocregopher/radix/v3 v3.3.0 + github.com/oklog/ulid v1.3.1 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect - golang.org/x/text v0.3.2 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - k8s.io/api v0.0.0-20190626000116-b178a738ed00 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + k8s.io/api v0.0.0-20190620084959-7cf5895f2711 + k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719 + k8s.io/client-go v0.0.0-20190620085101-78d2af792bab + k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a // indirect ) diff --git a/go.sum b/go.sum index 2d629811c83..ee0fa5dcb88 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,13 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -13,81 +17,93 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 h1:WSBJMqJbLxsn+bTCPyPYZfqHdJmc8MK4wrBjMft6BAM= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a h1:b+Gt8sQs//Sl5Dcem5zP9Qc2FgEUAygREa2AAa2Vmcw= github.com/kjk/betterguid v0.0.0-20170621091430-c442874ba63a/go.mod h1:uxRAhHE1nl34DpWgfe0CYbNYbCnYplaB6rZH9ReWtUk= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed h1:3dQJqqDouawQgl3gBE1PNHKFkJYGEuFb1DbSlaxdosE= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0 h1:oacPXPKHJg0hcngVVrdtTnfGJiS+PtwoQwTBZGFlV4k= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= @@ -101,12 +117,18 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -k8s.io/api v0.0.0-20190626000116-b178a738ed00 h1:Qqj3aerxILStcStl9mGcSbVyYuLxYDr2siLyJReTyaY= -k8s.io/api v0.0.0-20190626000116-b178a738ed00/go.mod h1:O6YAz5STgv7S1/c/XtBULGhSltH7yWEHpWvnA1mmFRg= -k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0 h1:7oql7STcnJ85hz3BIbasXHH/+lLLKwOdsG8vjkZc8Pc= -k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0/go.mod h1:48PVecD7ubRgJmMRGIQfsqYu6OucVH5DzFNtACHZH8k= +k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50= +k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= +k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719 h1:uV4S5IB5g4Nvi+TBVNf3e9L4wrirlwYJ6w88jUQxTUw= +k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA= +k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g= +k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a h1:2jUDc9gJja832Ftp+QbDV0tVhQHMISFn01els+2ZAcw= +k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 048dc259389bf836233a3a2ef923fd9e88dc9024 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Fri, 5 Jul 2019 10:50:03 +0100 Subject: [PATCH 07/10] Adding monitoring of nodes/pod utilitsation for use in executor --- cmd/executor/main.go | 136 ++++------- internal/reporter/pod_event_reporter.go | 42 ++++ internal/reporter/pod_event_reporter_test.go | 48 ++++ .../service/kubernetes_allocation_service.go | 142 +++++++++++ .../kubernetes_allocation_service_test.go | 222 ++++++++++++++++++ internal/startup/kubernetes_client.go | 22 ++ internal/submitter/job_submitter.go | 13 + 7 files changed, 533 insertions(+), 92 deletions(-) create mode 100644 internal/reporter/pod_event_reporter.go create mode 100644 internal/reporter/pod_event_reporter_test.go create mode 100644 internal/service/kubernetes_allocation_service.go create mode 100644 internal/service/kubernetes_allocation_service_test.go create mode 100644 internal/startup/kubernetes_client.go create mode 100644 internal/submitter/job_submitter.go diff --git a/cmd/executor/main.go b/cmd/executor/main.go index e06a131d5f2..0006c132d54 100644 --- a/cmd/executor/main.go +++ b/cmd/executor/main.go @@ -2,50 +2,66 @@ package main import ( "fmt" + "github.com/G-Research/k8s-batch/internal/reporter" + "github.com/G-Research/k8s-batch/internal/service" + "github.com/G-Research/k8s-batch/internal/startup" + "github.com/G-Research/k8s-batch/internal/submitter" "github.com/oklog/ulid" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" + v12 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" - "log" "math/rand" + "os" "strings" "time" ) func main() { - //config, err := rest.InClusterConfig() + kubernetesClient, err := startup.LoadDefaultKubernetesClient() + if err != nil { + fmt.Println(err) + os.Exit(-1) + } + + //tweakOptionsFunc := func(options *metav1.ListOptions) { + // options.LabelSelector = "node-role.startup.io/master" + //} + //tweakOptions := informers.WithTweakListOptions(tweakOptionsFunc) + //factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0, tweakOptions) - kubeconfig := "/home/jamesmu/.kube/kind-config-kind" + podEventReporter := reporter.PodEventReporter{ KubernetesClient: kubernetesClient } - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + factory := informers.NewSharedInformerFactoryWithOptions(kubernetesClient, 0) + podWatcher := initializePodWatcher(factory, podEventReporter) - //rules := clientcmd.NewDefaultClientConfigLoadingRules() - //overrides := &clientcmd.ConfigOverrides{} - //config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() + nodeWatcher := factory.Core().V1().Nodes() + nodeLister := nodeWatcher.Lister() - if err != nil { - fmt.Println("Error loading config") - } + defer runtime.HandleCrash() + stopper := make(chan struct{}) + defer close(stopper) + factory.Start(stopper) - clientset, err := kubernetes.NewForConfig(config) + jobSubmitter := submitter.JobSubmitter{KubernetesClient:kubernetesClient} - if err != nil { - fmt.Println("Error loading kubernetes client") + kubernetesAllocationService := service.KubernetesAllocationService{ + PodLister: podWatcher.Lister(), + NodeLister: nodeLister, + JobSubmitter: jobSubmitter, } - //tweakOptionsFunc := func(options *metav1.ListOptions) { - // options.LabelSelector = "node-role.kubernetes.io/master" - //} - //tweakOptions := informers.WithTweakListOptions(tweakOptionsFunc) - //factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0, tweakOptions) + for { + time.Sleep(5 * time.Second) - factory := informers.NewSharedInformerFactoryWithOptions(clientset, 0) + kubernetesAllocationService.FillInSpareClusterCapacity() + } +} +func createPod(kubernetesClient kubernetes.Interface) { terminationGracePeriod := int64(0) t := time.Now() @@ -71,101 +87,37 @@ func main() { }, } - result, err := clientset.CoreV1().Pods("default").Create(&pod) + result, err := kubernetesClient.CoreV1().Pods("default").Create(&pod) if err != nil { fmt.Printf("Failed creating pod %s\n", pod.ObjectMeta.Name) } else { fmt.Printf("Created pod %s\n", result.ObjectMeta.Name) } +} - - //createdPod, err := clientset.CoreV1().Pods("default").Create() - - +func initializePodWatcher(factory informers.SharedInformerFactory, eventReporter reporter.PodEventReporter) v12.PodInformer { podWatcher := factory.Core().V1().Pods() - - stopper := make(chan struct{}) - defer close(stopper) - - - defer runtime.HandleCrash() podWatcher.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { pod, ok := obj.(*v1.Pod) if !ok { - log.Println("this is not a pod") return } - fmt.Println("Added " + pod.ObjectMeta.Name + " status " + string(pod.Status.Phase)) + eventReporter.ReportAddEvent(pod) }, UpdateFunc: func(oldObj, newObj interface{}) { oldPod, ok := oldObj.(*v1.Pod) - newPod, ok := newObj.(*v1.Pod) if !ok { - log.Println("this is not a pod") return } - if oldPod.Status.Phase != newPod.Status.Phase { - fmt.Println("Updated " + newPod.ObjectMeta.Name + " to status " + string(newPod.Status.Phase)) - if strings.HasPrefix(pod.Name, "test") && (newPod.Status.Phase == v1.PodSucceeded || newPod.Status.Phase == v1.PodFailed) { - err := clientset.CoreV1().Pods(newPod.Namespace).Delete(newPod.Name, nil) - - if err != nil { - log.Println("Failed deleting " + pod.Name) - return - } - } - } else { - fmt.Println("Updated " + newPod.ObjectMeta.Name + " with no change to status " + string(newPod.Status.Phase)) - } - }, - - DeleteFunc:func(obj interface{}) { - pod, ok := obj.(*v1.Pod) + newPod, ok := newObj.(*v1.Pod) if !ok { - log.Println("this is not a pod") return } - - fmt.Println("Deleted " + pod.ObjectMeta.Name) + eventReporter.ReportUpdateEvent(oldPod, newPod) }, }) - - startingPosition := podWatcher.Informer().LastSyncResourceVersion() - fmt.Println("Starting position " + startingPosition) - - podCache:= podWatcher.Lister() - - nodeInformer := factory.Core().V1().Nodes().Lister() - factory.Start(stopper) - - for { - time.Sleep(5 * time.Second) - _, err := nodeInformer.List(labels.Everything()) - if err != nil { - fmt.Println("Error getting node information") - } - - allPods, err := podCache.List(labels.Everything()) - if err != nil { - fmt.Println("Error getting pod information") - } - //totalCpu := resource.Quantity{} - //for _, pod := range allPods { - // fmt.Println(pod.Spec.Containers[0].Resources.Requests.Cpu().AsDec()) - // podCpu := pod.Spec.Containers[0].Resources.Requests.Cpu() - // totalCpu.Add(*podCpu) - //} - // - //fmt.Println(totalCpu.AsDec()) - - - fmt.Printf("Total count of pods is %d\n", len(allPods)) - //for _, node := range nodes { - // fmt.Println("Node " + node.Name) - // fmt.Printf("Node %d\n", node.Status.Allocatable.Cpu().AsDec()) - //} - } + return podWatcher } diff --git a/internal/reporter/pod_event_reporter.go b/internal/reporter/pod_event_reporter.go new file mode 100644 index 00000000000..0be4d1de6bd --- /dev/null +++ b/internal/reporter/pod_event_reporter.go @@ -0,0 +1,42 @@ +package reporter + +import ( + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "log" + "strings" +) + +type PodEventReporter struct { + KubernetesClient kubernetes.Interface +} + +func (eventReporter PodEventReporter) ReportAddEvent(pod *v1.Pod) { + fmt.Println("Added " + pod.ObjectMeta.Name + " status " + string(pod.Status.Phase)) +} + +func (eventReporter PodEventReporter) ReportUpdateEvent(oldPod *v1.Pod, newPod *v1.Pod) { + kubernetesClient := eventReporter.KubernetesClient + + if oldPod.Status.Phase != newPod.Status.Phase { + fmt.Println("Updated " + newPod.ObjectMeta.Name + " to status " + string(newPod.Status.Phase)) + if strings.HasPrefix(newPod.Name, "test") && IsInTerminalState(newPod) { + err := kubernetesClient.CoreV1().Pods(newPod.Namespace).Delete(newPod.Name, nil) + if err != nil { + log.Println("Failed deleting " + newPod.Name) + return + } + } + } else { + fmt.Println("Updated " + newPod.ObjectMeta.Name + " with no change to status " + string(newPod.Status.Phase)) + } +} + +func IsInTerminalState(pod *v1.Pod) bool { + podPhase := pod.Status.Phase + if podPhase == v1.PodSucceeded || podPhase == v1.PodFailed { + return true + } + return false +} diff --git a/internal/reporter/pod_event_reporter_test.go b/internal/reporter/pod_event_reporter_test.go new file mode 100644 index 00000000000..9a0926a8930 --- /dev/null +++ b/internal/reporter/pod_event_reporter_test.go @@ -0,0 +1,48 @@ +package reporter + +import ( + v1 "k8s.io/api/core/v1" + "testing" +) + +func TestIsInTerminalState_ShouldReturnTrueWhenPodInSucceededPhase(t *testing.T) { + pod := v1.Pod{ + Status: v1.PodStatus { + Phase: v1.PodSucceeded, + }, + } + + inTerminatedState := IsInTerminalState(&pod) + + if !inTerminatedState { + t.Errorf("InTerminatedState was incorrect, got: %t want: %t", inTerminatedState, true) + } +} + +func TestIsInTerminalState_ShouldReturnTrueWhenPodInFailedPhase(t *testing.T) { + pod := v1.Pod{ + Status: v1.PodStatus { + Phase: v1.PodFailed, + }, + } + + inTerminatedState := IsInTerminalState(&pod) + + if !inTerminatedState { + t.Errorf("InTerminatedState was incorrect, got: %t want: %t", inTerminatedState, true) + } +} + +func TestIsInTerminalState_ShouldReturnFalseWhenPodInNonTerminalState(t *testing.T) { + pod := v1.Pod{ + Status: v1.PodStatus { + Phase: v1.PodPending, + }, + } + + inTerminatedState := IsInTerminalState(&pod) + + if inTerminatedState { + t.Errorf("InTerminatedState was incorrect, got: %t want: %t", inTerminatedState, false) + } +} diff --git a/internal/service/kubernetes_allocation_service.go b/internal/service/kubernetes_allocation_service.go new file mode 100644 index 00000000000..8c5d233b05f --- /dev/null +++ b/internal/service/kubernetes_allocation_service.go @@ -0,0 +1,142 @@ +package service + +import ( + "fmt" + "github.com/G-Research/k8s-batch/internal/submitter" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers/core/v1" +) + +type KubernetesAllocationService struct { + PodLister listers.PodLister + NodeLister listers.NodeLister + JobSubmitter submitter.JobSubmitter +} + +func (allocationService KubernetesAllocationService) FillInSpareClusterCapacity() { + allNodes, err := allocationService.NodeLister.List(labels.Everything()) + if err != nil { + fmt.Println("Error getting node information") + } + + allPods, err := allocationService.PodLister.List(labels.Everything()) + if err != nil { + fmt.Println("Error getting pod information") + } + // Todo Inefficient? We could monitor changes on nodes + pods and keep an internal map of where they are. However then we would be maintaining 2 internal maps (ours + informer) + + processingNodes := getAllAvailableProcessingNodes(allNodes) + podsOnProcessingNodes := getAllPodsOnNodes(allPods, processingNodes); + + totalNodeCpu := calculateTotalCpu(processingNodes) + totalNodeMemory := calculateTotalMemory(processingNodes) + + totalPodCpuLimit := calculateTotalCpuLimit(podsOnProcessingNodes) + totalPodMemoryLimit := calculateTotalMemoryLimit(podsOnProcessingNodes) + + freeCpu := totalNodeCpu.DeepCopy() + freeCpu.Sub(totalPodCpuLimit) + + freeMemory := totalNodeMemory.DeepCopy() + freeMemory.Sub(totalPodMemoryLimit) + + //newJobs := jobRequest.RequestJobs(freeCpu, freeMemory) + //for _, job := range newJobs { + // jobSubmitter.SubmitJob(job, "default") + //} + +} + +func getAllAvailableProcessingNodes(nodes []*v1.Node) []*v1.Node { + processingNodes := make([]*v1.Node, 0, len(nodes)) + + for _, node := range nodes { + if isAvailableProcessingNode(node) { + processingNodes = append(processingNodes, node) + } + } + + return processingNodes +} + +func isAvailableProcessingNode(node *v1.Node) bool { + if node.Spec.Unschedulable { + return false + } + + noSchedule := false + + for _, taint := range node.Spec.Taints { + if taint.Effect == v1.TaintEffectNoSchedule { + noSchedule = true + break + } + } + + if noSchedule { + return false + } + + return true +} + +func getAllPodsOnNodes(pods []*v1.Pod, nodes []*v1.Node) []*v1.Pod { + podsBelongingToNodes := make([]*v1.Pod, 0, len(pods)) + + nodeMap := make(map[string]*v1.Node) + for _, node := range nodes { + nodeMap[node.Name] = node + } + + for _, pod := range pods { + if _, present := nodeMap[pod.Spec.NodeName]; present { + podsBelongingToNodes = append(podsBelongingToNodes, pod) + } + } + + return podsBelongingToNodes +} + +func calculateTotalCpu(nodes []*v1.Node) resource.Quantity { + totalCpu := resource.Quantity{} + for _, node := range nodes { + nodeAllocatableCpu := node.Status.Allocatable.Cpu() + totalCpu.Add(*nodeAllocatableCpu) + } + return totalCpu +} + +func calculateTotalMemory(nodes []*v1.Node) resource.Quantity { + totalMemory := resource.Quantity{} + for _, node := range nodes { + nodeAllocatableMemory := node.Status.Allocatable.Memory() + totalMemory.Add(*nodeAllocatableMemory) + } + return totalMemory +} + +func calculateTotalCpuLimit(pods []*v1.Pod) resource.Quantity { + totalCpu := resource.Quantity{} + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + containerCpuLimit := container.Resources.Limits.Cpu() + totalCpu.Add(*containerCpuLimit) + } + // Todo determine what to do about init contianers? How does Kubernetes scheduler handle these + } + return totalCpu +} + +func calculateTotalMemoryLimit(pods []*v1.Pod) resource.Quantity { + totalMemory := resource.Quantity{} + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + containerMemoryLimit := container.Resources.Limits.Memory() + totalMemory.Add(*containerMemoryLimit) + } + // Todo determine what to do about init contianers? How does Kubernetes scheduler handle these + } + return totalMemory +} \ No newline at end of file diff --git a/internal/service/kubernetes_allocation_service_test.go b/internal/service/kubernetes_allocation_service_test.go new file mode 100644 index 00000000000..ab1b8554685 --- /dev/null +++ b/internal/service/kubernetes_allocation_service_test.go @@ -0,0 +1,222 @@ +package service + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestGetAllAvailableProcessingNodes_ShouldReturnAvailableProcessingNodes(t *testing.T) { + node := v1.Node{ + Spec: v1.NodeSpec { + Unschedulable: false, + Taints:nil, + }, + } + + nodes := []*v1.Node{&node} + result := getAllAvailableProcessingNodes(nodes) + + if len(result) != 1 { + t.Errorf("GetAllAvailableProcessingNodes was incorrect, got: %d want: %d", len(result), 1) + } +} + +func TestGetAllAvailableProcessingNodes_ShouldFilterUnschedulableNodes(t *testing.T) { + node := v1.Node{ + Spec: v1.NodeSpec { + Unschedulable: true, + Taints:nil, + }, + } + + nodes := []*v1.Node{&node} + result := getAllAvailableProcessingNodes(nodes) + + if len(result) != 0 { + t.Errorf("GetAllAvailableProcessingNodes was incorrect, got: %d want: %d", len(result), 0) + } +} + +func TestGetAllAvailableProcessingNodes_ShouldFilterNodesWithNoScheduleTaint(t *testing.T) { + taint := v1.Taint { + Effect:v1.TaintEffectNoSchedule, + } + node := v1.Node{ + Spec: v1.NodeSpec { + Unschedulable: false, + Taints: []v1.Taint{taint}, + }, + } + + nodes := []*v1.Node{&node} + result := getAllAvailableProcessingNodes(nodes) + + if len(result) != 0 { + t.Errorf("GetAllAvailableProcessingNodes was incorrect, got: %d want: %d", len(result), 0) + } +} + +func TestGetAllPodsOnNodes_ShouldExcludePodsNoOnGivenNodes(t *testing.T) { + presentNodeName := "Node1" + podOnNode := v1.Pod{ + Spec: v1.PodSpec{ + NodeName: presentNodeName, + }, + } + podNotOnNode := v1.Pod{ + Spec: v1.PodSpec{ + NodeName: "Node2", + }, + } + + node := v1.Node{ + ObjectMeta : metav1.ObjectMeta { + Name: presentNodeName, + }, + } + pods := []*v1.Pod {&podOnNode, &podNotOnNode} + nodes := []*v1.Node {&node} + + result := getAllPodsOnNodes(pods, nodes) + + if len(result) != 1 || result[0].Spec.NodeName != presentNodeName { + t.Errorf("GetAllPodsOnNodes was incorrect, got: %d want: %d (%s)", len(result), 1, presentNodeName) + } +} + +func TestGetAllPodsOnNodes_ShouldHandleNoNodesProvided(t *testing.T) { + podOnNode := v1.Pod{ + Spec: v1.PodSpec{ + NodeName: "Node1", + }, + } + + pods := []*v1.Pod {&podOnNode} + var nodes []*v1.Node + + result := getAllPodsOnNodes(pods, nodes) + + if len(result) != 0 { + t.Errorf("GetAllPodsOnNodes was incorrect, got: %d want: %d", len(result), 0) + } +} + +func TestCalculateTotalCpu(t *testing.T) { + resources := resource.NewMilliQuantity(100, resource.DecimalSI) + resourceMap := map[v1.ResourceName]resource.Quantity { v1.ResourceCPU: *resources } + node1 := makeNodeWithResource(resourceMap) + node2 := makeNodeWithResource(resourceMap) + + result := calculateTotalCpu([]*v1.Node{&node1, &node2}) + + //Expected is resources *2 + resources.Add(*resources) + if !result.Equal(*resources) { + t.Errorf("CalculateTotalCpu was incorrect, got: %#v want %#v", result, resources) + } +} + +func TestCalculateTotalMemory(t *testing.T) { + resources := resource.NewMilliQuantity(50*1024*1024, resource.DecimalSI) + resourceMap := map[v1.ResourceName]resource.Quantity { v1.ResourceMemory: *resources } + node1 := makeNodeWithResource(resourceMap) + node2 := makeNodeWithResource(resourceMap) + + result := calculateTotalMemory([]*v1.Node{&node1, &node2}) + + //Expected is resources *2 + resources.Add(*resources) + + if !result.Equal(*resources) { + t.Errorf("CalculateTotalMemory was incorrect, got: %#v want %#v", result, resources) + } +} + +func TestCalculateTotalCpuLimit_ShouldSumAllContainers(t *testing.T) { + resources := resource.NewMilliQuantity(100, resource.DecimalSI) + pod := makePodWthResource(v1.ResourceCPU, []*resource.Quantity{resources, resources}) + + result := calculateTotalCpuLimit([]*v1.Pod{&pod}) + + //Expected is resources * 2 containers + resources.Add(*resources) + + if !result.Equal(*resources) { + t.Errorf("CalculateTotalCpuLimit was incorrect, got: %#v want %#v", result, resources) + } +} + +func TestCalculateTotalCpuLimit_ShouldSumAllPods(t *testing.T) { + resources := resource.NewMilliQuantity(100, resource.DecimalSI) + pod1 := makePodWthResource(v1.ResourceCPU, []*resource.Quantity{resources}) + pod2 := makePodWthResource(v1.ResourceCPU, []*resource.Quantity{resources}) + + result := calculateTotalCpuLimit([]*v1.Pod{&pod1, &pod2}) + + //Expected is resources * 2 pods + resources.Add(*resources) + + if !result.Equal(*resources) { + t.Errorf("CalculateTotalCpuLimit was incorrect, got: %#v want %#v", result, resources) + } +} + +func TestCalculateTotalMemoryLimit_ShouldSumAllContainers(t *testing.T) { + resources := resource.NewMilliQuantity(50*1024*1024, resource.DecimalSI) + pod := makePodWthResource(v1.ResourceMemory, []*resource.Quantity{resources, resources}) + + result := calculateTotalMemoryLimit([]*v1.Pod{&pod}) + + //Expected is resources * 2 containers + resources.Add(*resources) + + if !result.Equal(*resources) { + t.Errorf("CalculateTotalMemoryLimit was incorrect, got: %#v want %#v", result, resources) + } +} + +func TestCalculateTotalMemoryLimit__ShouldSumAllPods(t *testing.T) { + resources := resource.NewMilliQuantity(50*1024*1024, resource.DecimalSI) + pod := makePodWthResource(v1.ResourceMemory, []*resource.Quantity{resources, resources}) + + result := calculateTotalMemoryLimit([]*v1.Pod{&pod}) + + //Expected is resources * 2 containers + resources.Add(*resources) + + if !result.Equal(*resources) { + t.Errorf("CalculateTotalMemoryLimit was incorrect, got: %#v want %#v", result, resources) + } +} + + +func makePodWthResource(resourceName v1.ResourceName, resources []*resource.Quantity) v1.Pod { + containers := make([]v1.Container, len(resources)) + for i, res := range resources { + containers[i] = v1.Container{ + Resources: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity { resourceName: *res }, + }, + } + } + pod := v1.Pod{ + Spec: v1.PodSpec { + Containers: containers, + }, + } + + return pod +} + + +func makeNodeWithResource(resources map[v1.ResourceName]resource.Quantity) v1.Node { + node := v1.Node{ + Status: v1.NodeStatus{ + Allocatable: resources, + }, + } + return node +} + diff --git a/internal/startup/kubernetes_client.go b/internal/startup/kubernetes_client.go new file mode 100644 index 00000000000..d3f9c2741f5 --- /dev/null +++ b/internal/startup/kubernetes_client.go @@ -0,0 +1,22 @@ +package startup + +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func LoadDefaultKubernetesClient() (kubernetes.Interface, error) { + //TODO Add in a way to have in cluster vs normal kubernetes client + //config, err := rest.InClusterConfig() + rules := clientcmd.NewDefaultClientConfigLoadingRules() + overrides := &clientcmd.ConfigOverrides{} + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() + + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(config) +} + + diff --git a/internal/submitter/job_submitter.go b/internal/submitter/job_submitter.go new file mode 100644 index 00000000000..414ab3a0332 --- /dev/null +++ b/internal/submitter/job_submitter.go @@ -0,0 +1,13 @@ +package submitter + +import ( + "github.com/G-Research/k8s-batch/internal/model" + "k8s.io/client-go/kubernetes" +) + +type JobSubmitter struct { + KubernetesClient kubernetes.Interface +} + +func (submitter JobSubmitter) SubmitJob(job *model.Job, namespace string) { +} \ No newline at end of file From 47ff068a6df0eb0c4e7cb3d63717cab1518418c8 Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Fri, 5 Jul 2019 10:53:55 +0100 Subject: [PATCH 08/10] Removing pre-mature addition of circleci config --- .circleci/config.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 52398c4e1a9..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/golang:1.9 - - #### TEMPLATE_NOTE: go expects specific checkout path representing url - #### expecting it in the form of - #### /go/src/github.com/circleci/go-tool - #### /go/src/bitbucket.org/circleci/go-tool - working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} - steps: - - checkout - - # specify any bash command here prefixed with `run: ` - - run: go get -v -t -d ./... - - run: go cmd/api/main.go \ No newline at end of file From 3eda200e7d8ef37013049e1d08a81a2f93aa4bdf Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Fri, 5 Jul 2019 10:56:57 +0100 Subject: [PATCH 09/10] Removing unused createPod from executor/main This is just cluttering the file and is never used (And if it was used, should never be placed here) --- cmd/executor/main.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/cmd/executor/main.go b/cmd/executor/main.go index 0006c132d54..b4478dc16ca 100644 --- a/cmd/executor/main.go +++ b/cmd/executor/main.go @@ -6,17 +6,12 @@ import ( "github.com/G-Research/k8s-batch/internal/service" "github.com/G-Research/k8s-batch/internal/startup" "github.com/G-Research/k8s-batch/internal/submitter" - "github.com/oklog/ulid" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" v12 "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "math/rand" "os" - "strings" "time" ) @@ -61,41 +56,6 @@ func main() { } } -func createPod(kubernetesClient kubernetes.Interface) { - terminationGracePeriod := int64(0) - - t := time.Now() - entropy := ulid.Monotonic(rand.New(rand.NewSource(t.UnixNano())), 0) - - pod := v1.Pod { - ObjectMeta: metav1.ObjectMeta { - Name: "test-" + strings.ToLower(ulid.MustNew(ulid.Timestamp(t), entropy).String()), - }, - Spec: v1.PodSpec { - TerminationGracePeriodSeconds: &terminationGracePeriod, - RestartPolicy: v1.RestartPolicyNever, - Containers: []v1.Container { - { - Name: "sleeper", - Image: "mcr.microsoft.com/dotnet/core/runtime:2.2", - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string { - "sleep", "20s", - }, - }, - }, - }, - } - - result, err := kubernetesClient.CoreV1().Pods("default").Create(&pod) - - if err != nil { - fmt.Printf("Failed creating pod %s\n", pod.ObjectMeta.Name) - } else { - fmt.Printf("Created pod %s\n", result.ObjectMeta.Name) - } -} - func initializePodWatcher(factory informers.SharedInformerFactory, eventReporter reporter.PodEventReporter) v12.PodInformer { podWatcher := factory.Core().V1().Pods() podWatcher.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ From bc54746b523ba7f339a5448b53d1a002f9c170fe Mon Sep 17 00:00:00 2001 From: JamesMurkin Date: Fri, 5 Jul 2019 11:08:15 +0100 Subject: [PATCH 10/10] Tidying file line endings --- internal/service/kubernetes_allocation_service.go | 2 +- internal/service/kubernetes_allocation_service_test.go | 1 - internal/startup/kubernetes_client.go | 2 -- internal/submitter/job_submitter.go | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/service/kubernetes_allocation_service.go b/internal/service/kubernetes_allocation_service.go index 8c5d233b05f..2e95c323a12 100644 --- a/internal/service/kubernetes_allocation_service.go +++ b/internal/service/kubernetes_allocation_service.go @@ -139,4 +139,4 @@ func calculateTotalMemoryLimit(pods []*v1.Pod) resource.Quantity { // Todo determine what to do about init contianers? How does Kubernetes scheduler handle these } return totalMemory -} \ No newline at end of file +} diff --git a/internal/service/kubernetes_allocation_service_test.go b/internal/service/kubernetes_allocation_service_test.go index ab1b8554685..30f3b8cf7cf 100644 --- a/internal/service/kubernetes_allocation_service_test.go +++ b/internal/service/kubernetes_allocation_service_test.go @@ -219,4 +219,3 @@ func makeNodeWithResource(resources map[v1.ResourceName]resource.Quantity) v1.No } return node } - diff --git a/internal/startup/kubernetes_client.go b/internal/startup/kubernetes_client.go index d3f9c2741f5..eaadbadb444 100644 --- a/internal/startup/kubernetes_client.go +++ b/internal/startup/kubernetes_client.go @@ -18,5 +18,3 @@ func LoadDefaultKubernetesClient() (kubernetes.Interface, error) { return kubernetes.NewForConfig(config) } - - diff --git a/internal/submitter/job_submitter.go b/internal/submitter/job_submitter.go index 414ab3a0332..5a8f8706b99 100644 --- a/internal/submitter/job_submitter.go +++ b/internal/submitter/job_submitter.go @@ -10,4 +10,4 @@ type JobSubmitter struct { } func (submitter JobSubmitter) SubmitJob(job *model.Job, namespace string) { -} \ No newline at end of file +}