Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cgroup limiting to Linux Executor #52

Closed
wants to merge 49 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
2c4d21b
Merge pull request #36 from hashicorp/f-executor
cbednarski Sep 16, 2015
cfb6f37
api: add region to jobs
ryanuber Sep 16, 2015
39789e4
api: fix tests after struct validations
ryanuber Sep 16, 2015
24cd105
api: composing jobs takes region
ryanuber Sep 16, 2015
da35ecc
command: fix tests after job validation
ryanuber Sep 16, 2015
4c8eb1d
jobspec: default job ID also comes from key
ryanuber Sep 17, 2015
8e6bc8b
Merge pull request #55 from hashicorp/f-hcl-job
ryanuber Sep 17, 2015
06aa011
api: sort all list responses
ryanuber Sep 17, 2015
0f0bbbd
api: test job sort
ryanuber Sep 17, 2015
c2364f1
website: Working on internal architecture
armon Sep 17, 2015
6bf4beb
website: Replace Vault -> Nomad
armon Sep 17, 2015
b0eb417
website: fixing repo link
armon Sep 17, 2015
5770e8f
Merge pull request #57 from hashicorp/f-api-sort
ryanuber Sep 18, 2015
e56635c
website: working on internals documentation
armon Sep 18, 2015
b7cd4a4
merging doc updates
armon Sep 18, 2015
14009c5
scheduler: ignore allocations in terminal state
armon Sep 18, 2015
126147a
Fixing build script
armon Sep 18, 2015
4dfb938
website: add docs on scheduling
armon Sep 19, 2015
557aa14
website: update scheduling links
armon Sep 19, 2015
f3a2929
website: cleanup install docs
armon Sep 19, 2015
b8269fd
website: cleanup dead docs
armon Sep 19, 2015
5dc89c0
website: adding drivers section
armon Sep 19, 2015
c3d93b2
website: imageoptim
armon Sep 19, 2015
1126e3d
website: use new hero
pearkes Sep 19, 2015
8495e94
website: Add DO logo
armon Sep 19, 2015
d0258f3
website: reduce whitespace on home
armon Sep 19, 2015
ebb9f3b
website: homepage
armon Sep 19, 2015
7f06829
website: fixing vagrantfile
armon Sep 19, 2015
0af2584
website: homepage copy
armon Sep 19, 2015
60dada0
website: imageoptim
armon Sep 19, 2015
385b1f9
website: intro page
armon Sep 19, 2015
b32dbea
website: use cases page
armon Sep 19, 2015
1acd810
website: starting vs section
armon Sep 19, 2015
74ae886
website: finish ECS and Swarm
armon Sep 20, 2015
2e86401
website: HTCondor
armon Sep 20, 2015
6f0ce8b
website: Kubernetes
armon Sep 20, 2015
bfdfa6c
website: Mesos
armon Sep 20, 2015
9ce7b4a
website: Terraform
armon Sep 20, 2015
1a92aa4
website: YARN
armon Sep 20, 2015
b0707ad
website: custom
armon Sep 20, 2015
45afdff
Do not use prefix splitting to deploy
sethvargo Sep 20, 2015
a11671e
Force Ruby 2.2.2 on Heroku
sethvargo Sep 20, 2015
bba002f
Merge pull request #66 from hashicorp/sethvargo/faster_deploy
armon Sep 20, 2015
3308912
Merge pull request #67 from hashicorp/sethvargo/specify_ruby
armon Sep 20, 2015
7e491a4
spike on adding cgroups
catsby Sep 15, 2015
f03c157
exec/linux: move Limit to after Start
catsby Sep 16, 2015
27d994d
executor/linux: Add UUID for the name, return error on failure to apply
catsby Sep 16, 2015
f73bb99
Linux executor with cgroup isolation support
dadgar Sep 20, 2015
a8dbcab
Update UniversalExecutor to have string identifiers
dadgar Sep 20, 2015
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions api/allocations.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package api

import (
"sort"
"time"
)

Expand All @@ -21,6 +22,7 @@ func (a *Allocations) List(q *QueryOptions) ([]*AllocationListStub, *QueryMeta,
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}

Expand Down Expand Up @@ -84,3 +86,18 @@ type AllocationListStub struct {
CreateIndex uint64
ModifyIndex uint64
}

// AllocIndexSort reverse sorts allocs by CreateIndex.
type AllocIndexSort []*AllocationListStub

func (a AllocIndexSort) Len() int {
return len(a)
}

func (a AllocIndexSort) Less(i, j int) bool {
return a[i].CreateIndex > a[j].CreateIndex
}

func (a AllocIndexSort) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
20 changes: 20 additions & 0 deletions api/allocations_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package api

import (
"reflect"
"sort"
"testing"
)

Expand Down Expand Up @@ -49,3 +51,21 @@ func TestAllocations_List(t *testing.T) {
t.Fatalf("bad: %#v", allocs)
}
}

func TestAllocations_CreateIndexSort(t *testing.T) {
allocs := []*AllocationListStub{
&AllocationListStub{CreateIndex: 2},
&AllocationListStub{CreateIndex: 1},
&AllocationListStub{CreateIndex: 5},
}
sort.Sort(AllocIndexSort(allocs))

expect := []*AllocationListStub{
&AllocationListStub{CreateIndex: 5},
&AllocationListStub{CreateIndex: 2},
&AllocationListStub{CreateIndex: 1},
}
if !reflect.DeepEqual(allocs, expect) {
t.Fatalf("\n\n%#v\n\n%#v", allocs, expect)
}
}
3 changes: 2 additions & 1 deletion api/compose_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,15 @@ func TestCompose(t *testing.T) {
AddTask(task)

// Compose a job
job := NewServiceJob("job1", "myjob", 2).
job := NewServiceJob("job1", "myjob", "region1", 2).
SetMeta("foo", "bar").
AddDatacenter("dc1").
Constrain(HardConstraint("kernel.name", "=", "linux")).
AddTaskGroup(grp)

// Check that the composed result looks correct
expect := &Job{
Region: "region1",
ID: "job1",
Name: "myjob",
Type: JobTypeService,
Expand Down
21 changes: 21 additions & 0 deletions api/evaluations.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package api

import (
"sort"
"time"
)

Expand All @@ -21,6 +22,7 @@ func (e *Evaluations) List(q *QueryOptions) ([]*Evaluation, *QueryMeta, error) {
if err != nil {
return nil, nil, err
}
sort.Sort(EvalIndexSort(resp))
return resp, qm, nil
}

Expand All @@ -42,6 +44,7 @@ func (e *Evaluations) Allocations(evalID string, q *QueryOptions) ([]*Allocation
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}

Expand All @@ -60,4 +63,22 @@ type Evaluation struct {
Wait time.Duration
NextEval string
PreviousEval string
CreateIndex uint64
ModifyIndex uint64
}

// EvalIndexSort is a wrapper to sort evaluations by CreateIndex.
// We reverse the test so that we get the highest index first.
type EvalIndexSort []*Evaluation

func (e EvalIndexSort) Len() int {
return len(e)
}

func (e EvalIndexSort) Less(i, j int) bool {
return e[i].CreateIndex > e[j].CreateIndex
}

func (e EvalIndexSort) Swap(i, j int) {
e[i], e[j] = e[j], e[i]
}
20 changes: 20 additions & 0 deletions api/evaluations_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package api

import (
"reflect"
"sort"
"strings"
"testing"
)
Expand Down Expand Up @@ -94,3 +96,21 @@ func TestEvaluations_Allocations(t *testing.T) {
t.Fatalf("expected 0 allocs, got: %d", n)
}
}

func TestEvaluations_Sort(t *testing.T) {
evals := []*Evaluation{
&Evaluation{CreateIndex: 2},
&Evaluation{CreateIndex: 1},
&Evaluation{CreateIndex: 5},
}
sort.Sort(EvalIndexSort(evals))

expect := []*Evaluation{
&Evaluation{CreateIndex: 5},
&Evaluation{CreateIndex: 2},
&Evaluation{CreateIndex: 1},
}
if !reflect.DeepEqual(evals, expect) {
t.Fatalf("\n\n%#v\n\n%#v", evals, expect)
}
}
40 changes: 32 additions & 8 deletions api/jobs.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package api

import (
"sort"
)

const (
// JobTypeService indicates a long-running processes
JobTypeService = "service"
Expand Down Expand Up @@ -38,6 +42,7 @@ func (j *Jobs) List(q *QueryOptions) ([]*JobListStub, *QueryMeta, error) {
if err != nil {
return nil, qm, err
}
sort.Sort(JobIDSort(resp))
return resp, qm, nil
}

Expand All @@ -59,6 +64,7 @@ func (j *Jobs) Allocations(jobID string, q *QueryOptions) ([]*AllocationListStub
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}

Expand All @@ -70,6 +76,7 @@ func (j *Jobs) Evaluations(jobID string, q *QueryOptions) ([]*Evaluation, *Query
if err != nil {
return nil, nil, err
}
sort.Sort(EvalIndexSort(resp))
return resp, qm, nil
}

Expand All @@ -94,6 +101,7 @@ func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta,

// Job is used to serialize a job.
type Job struct {
Region string
ID string
Name string
Type string
Expand Down Expand Up @@ -122,26 +130,42 @@ type JobListStub struct {
ModifyIndex uint64
}

// JobIDSort is used to sort jobs by their job ID's.
type JobIDSort []*JobListStub

func (j JobIDSort) Len() int {
return len(j)
}

func (j JobIDSort) Less(a, b int) bool {
return j[a].ID < j[b].ID
}

func (j JobIDSort) Swap(a, b int) {
j[a], j[b] = j[b], j[a]
}

// NewServiceJob creates and returns a new service-style job
// for long-lived processes using the provided name, ID, and
// relative job priority.
func NewServiceJob(id, name string, pri int) *Job {
return newJob(id, name, JobTypeService, pri)
func NewServiceJob(id, name, region string, pri int) *Job {
return newJob(id, name, region, JobTypeService, pri)
}

// NewBatchJob creates and returns a new batch-style job for
// short-lived processes using the provided name and ID along
// with the relative job priority.
func NewBatchJob(id, name string, pri int) *Job {
return newJob(id, name, JobTypeBatch, pri)
func NewBatchJob(id, name, region string, pri int) *Job {
return newJob(id, name, region, JobTypeBatch, pri)
}

// newJob is used to create a new Job struct.
func newJob(jobID, jobName, jobType string, pri int) *Job {
func newJob(id, name, region, typ string, pri int) *Job {
return &Job{
ID: jobID,
Name: jobName,
Type: jobType,
Region: region,
ID: id,
Name: name,
Type: typ,
Priority: pri,
}
}
Expand Down
25 changes: 23 additions & 2 deletions api/jobs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package api

import (
"reflect"
"sort"
"strings"
"testing"
)
Expand Down Expand Up @@ -218,8 +219,9 @@ func TestJobs_ForceEvaluate(t *testing.T) {
}

func TestJobs_NewBatchJob(t *testing.T) {
job := NewBatchJob("job1", "myjob", 5)
job := NewBatchJob("job1", "myjob", "region1", 5)
expect := &Job{
Region: "region1",
ID: "job1",
Name: "myjob",
Type: JobTypeBatch,
Expand All @@ -231,8 +233,9 @@ func TestJobs_NewBatchJob(t *testing.T) {
}

func TestJobs_NewServiceJob(t *testing.T) {
job := NewServiceJob("job1", "myjob", 5)
job := NewServiceJob("job1", "myjob", "region1", 5)
expect := &Job{
Region: "region1",
ID: "job1",
Name: "myjob",
Type: JobTypeService,
Expand Down Expand Up @@ -301,3 +304,21 @@ func TestJobs_Constrain(t *testing.T) {
t.Fatalf("expect: %#v, got: %#v", expect, job.Constraints)
}
}

func TestJobs_Sort(t *testing.T) {
jobs := []*JobListStub{
&JobListStub{ID: "job2"},
&JobListStub{ID: "job0"},
&JobListStub{ID: "job1"},
}
sort.Sort(JobIDSort(jobs))

expect := []*JobListStub{
&JobListStub{ID: "job0"},
&JobListStub{ID: "job1"},
&JobListStub{ID: "job2"},
}
if !reflect.DeepEqual(jobs, expect) {
t.Fatalf("\n\n%#v\n\n%#v", jobs, expect)
}
}
20 changes: 19 additions & 1 deletion api/nodes.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package api

import (
"sort"
"strconv"
)

Expand All @@ -16,11 +17,12 @@ func (c *Client) Nodes() *Nodes {

// List is used to list out all of the nodes
func (n *Nodes) List(q *QueryOptions) ([]*NodeListStub, *QueryMeta, error) {
var resp []*NodeListStub
var resp NodeIndexSort
qm, err := n.client.query("/v1/nodes", &resp, q)
if err != nil {
return nil, nil, err
}
sort.Sort(NodeIndexSort(resp))
return resp, qm, nil
}

Expand Down Expand Up @@ -51,6 +53,7 @@ func (n *Nodes) Allocations(nodeID string, q *QueryOptions) ([]*AllocationListSt
if err != nil {
return nil, nil, err
}
sort.Sort(AllocIndexSort(resp))
return resp, qm, nil
}

Expand Down Expand Up @@ -95,6 +98,21 @@ type NodeListStub struct {
ModifyIndex uint64
}

// NodeIndexSort reverse sorts nodes by CreateIndex
type NodeIndexSort []*NodeListStub

func (n NodeIndexSort) Len() int {
return len(n)
}

func (n NodeIndexSort) Less(i, j int) bool {
return n[i].CreateIndex > n[j].CreateIndex
}

func (n NodeIndexSort) Swap(i, j int) {
n[i], n[j] = n[j], n[i]
}

// nodeEvalResponse is used to decode a force-eval.
type nodeEvalResponse struct {
EvalID string
Expand Down
20 changes: 20 additions & 0 deletions api/nodes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package api

import (
"fmt"
"reflect"
"sort"
"strings"
"testing"

Expand Down Expand Up @@ -201,3 +203,21 @@ func TestNodes_ForceEvaluate(t *testing.T) {
t.Fatalf("err: %s", err)
}
}

func TestNodes_Sort(t *testing.T) {
nodes := []*NodeListStub{
&NodeListStub{CreateIndex: 2},
&NodeListStub{CreateIndex: 1},
&NodeListStub{CreateIndex: 5},
}
sort.Sort(NodeIndexSort(nodes))

expect := []*NodeListStub{
&NodeListStub{CreateIndex: 5},
&NodeListStub{CreateIndex: 2},
&NodeListStub{CreateIndex: 1},
}
if !reflect.DeepEqual(nodes, expect) {
t.Fatalf("\n\n%#v\n\n%#v", nodes, expect)
}
}
Loading