diff --git a/README.md b/README.md index 1106cc65..dbd82e42 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,55 @@ Karamel introduces Karamelfile to orchestrate the execution of Chef recipes. Kar We leverage Berkshelf to transparently download and install transitive cookbook dependencies, so large systems can be defined in a few lines of code. Finally, the Karamel runtime builds and manages the execution of the DAG of Chef recipes by executing them with Chef Solo. +### Configuration +#### Karamel working directory +By default Karamel (local & remote) working directory is `$HOME/.karamel` If you want to override this directory +use the environment variable `KARAMEL_WORKING_DIRECTORY` such as `export +KARAMEL_WORKING_DIRECTORY=/tmp/karamel_directory` + +#### Chef file cache +Chef will download files to `/tmp/chef-solo` before using them. To change the staging location use the +environment variable `CHEF_FILE_CACHE_PATH` such as `export CHEF_FILE_CACHE_PATH=/run/chef-solo` + +#### Elevated privileges program +Karamel requires to run with elevated privileges. The default program for that is `sudo` but if your +organization uses another program set it with the environment variable `CHEF_SUDO_BINARY` such as +`export CHEF_SUDO_BINARY=dzdo` + +#### Airgapped installations +If you are installing Hopsworks on an environment with no internet connectivity set `KARAMEL_AIRGAP=true` + +#### Ruby Gems server +To point Chef to a local Ruby gems server instead of https://rubygems.org export +`CHEF_RUBYGEMS_URL=http://IP_ADDRESS:8808` + ### Cluster Definition --- Cluster definition is an expressive DSL based on YAML as you can see in the following sample. Since Karamel can run several clusters simultaneously, name of the cluster must be unique in each Karamel-runtime. +#### Recipes execution parallelism +Karamel will execute recipes in parallel on all machines respecting any dependency stated in Karamelfile. Regardless +of the dependencies, the same recipes will be executed in parallel on all machines. For example if we run recipe +`ndb::ndbd` on three machines, they may be executed in parallel depending on how Karamel will schedule them. + +In some cases we want to guarantee only a portion of machines to execute recipes in parallel. For example in case of +a rolling restart/upgrade we want to execute `ndb::ndbd` serially. You can configure recipe parallelism under +`runtimeConfiguration/recipesParallelism` section in the cluster definition in the format `recipe_name: parallelism`. +Multiple recipes can be configured. For example: + +```yaml +runtimeConfiguration: + recipesParallelism: + consul::master: 1 + ndb::ndbd: 1 + ndb::mysqld: 2 + ndb::mysqld_tls: 2 + onlinefs::default: 1 + hops::nn: 1 + hops::rm: 1 +``` + +#### Cloud providers We support five cloud providers: Amazon EC2 (ec2), Google Compute Engine (gce), Openstack Nova (nova), OCCI and bare-metal (baremetal). You can define provider globally or per group. In the group scope, you can overwrite some attributes of the network/machines in the global scope or you can entirely choose another cloud provider, that's how we support multi-cloud deployment. Settings and properties for each provider is introduced in a following separate section. Cookbooks section introduces github references to the used cookbooks, it is also possible to refer to a specific version or branch for each github repository. diff --git a/karamel-common/pom.xml b/karamel-common/pom.xml index b0fba098..1fb59c5c 100644 --- a/karamel-common/pom.xml +++ b/karamel-common/pom.xml @@ -5,7 +5,7 @@ se.kth.karamel karamel-parent - 0.5 + 0.10 se.kth.karamel karamel-common diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Cookbook.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Cookbook.java index 133bfc91..4fe9d945 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Cookbook.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Cookbook.java @@ -1,39 +1,15 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef; -import se.kth.karamel.common.cookbookmeta.CookbookUrls; -import se.kth.karamel.common.exception.CookbookUrlException; - -/** - * - * @author kamal - */ public class Cookbook { private String github; - private String version; private String branch; - private String cookbook; public Cookbook() { } - public Cookbook(Cookbook cookbook) { - this.github = cookbook.getGithub(); - this.version = cookbook.getVersion(); - this.branch = cookbook.getBranch(); - this.cookbook = cookbook.cookbook; - } - - public String getBranch() { - return branch; - } - - public void setBranch(String branch) { + public Cookbook(String github, String branch) { + this.github = github; this.branch = branch; } @@ -45,32 +21,11 @@ public void setGithub(String github) { this.github = github; } - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getCookbook() { - return cookbook; + public String getBranch() { + return branch; } - public void setCookbook(String cookbook) { - this.cookbook = cookbook; - } - - public CookbookUrls getUrls() throws CookbookUrlException { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - builder.url(github); - if (branch != null) - builder.branchOrVersion(branch); - else if (version != null) - builder.branchOrVersion(version); - if (cookbook != null) - builder.cookbookRelPath(cookbook); - CookbookUrls urls = builder.build(); - return urls; + public void setBranch(String branch) { + this.branch = branch; } } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Scope.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Scope.java index 9e0c7a19..8168405b 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Scope.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Scope.java @@ -5,6 +5,7 @@ */ package se.kth.karamel.common.clusterdef; +import se.kth.karamel.common.clusterdef.yaml.RuntimeConfiguration; import se.kth.karamel.common.exception.ValidationException; /** @@ -16,6 +17,7 @@ public abstract class Scope { private Ec2 ec2; private Vagrant vagrant; private Baremetal baremetal; + private RuntimeConfiguration runtimeConfiguration = new RuntimeConfiguration(); private Gce gce; private Nova nova; private Occi occi; @@ -27,6 +29,7 @@ public Scope(Scope scope) { this.ec2 = scope.getEc2(); this.vagrant = scope.getVagrant(); this.baremetal = scope.getBaremetal(); + this.runtimeConfiguration = scope.getRuntimeConfiguration(); this.gce = scope.getGce(); this.nova = scope.getNova(); this.occi = scope.getOcci(); @@ -81,17 +84,17 @@ public Gce getGce() { public void setGce(Gce gce) { this.gce = gce; } - + public Occi getOcci() { return occi; } - + public void setOcci(Occi occi) { this.occi = occi; } public Nova getNova() {return nova;} - + public void setNova(Nova nova) { this.nova = nova; } @@ -116,5 +119,12 @@ public void validate() throws ValidationException { occi.validate(); } } -; + + public RuntimeConfiguration getRuntimeConfiguration() { + return runtimeConfiguration; + } + + public void setRuntimeConfiguration(RuntimeConfiguration runtimeConfiguration) { + this.runtimeConfiguration = runtimeConfiguration; + } } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCluster.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCluster.java index c2e9ee2c..970adc88 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCluster.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCluster.java @@ -1,40 +1,61 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.json; +import se.kth.karamel.common.clusterdef.Cookbook; import se.kth.karamel.common.clusterdef.yaml.YamlCluster; import se.kth.karamel.common.clusterdef.yaml.YamlGroup; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; + +import se.kth.karamel.common.cookbookmeta.Attribute; +import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.exception.KaramelException; -/** - * - * @author kamal - */ public class JsonCluster extends JsonScope { private String name; - + private Map rootCookbooks = new HashMap<>(); private List groups = new ArrayList<>(); public JsonCluster() { } public JsonCluster(YamlCluster cluster) throws KaramelException { - super(cluster, cluster); - this.name = cluster.getName(); + super(cluster); + name = cluster.getName(); + rootCookbooks = cluster.getCookbooks(); + attributes = cluster.flattenAttrs(); + Set validAttrs = new HashSet<>(); + + cookbooks.addAll(CACHE.loadAllKaramelizedCookbooks(cluster)); + + //filtering invalid(not defined in metadata.rb) attributes from yaml model + // Get all the valid attributes, also for transient dependency + for (KaramelizedCookbook kcb : cookbooks) { + validAttrs.addAll(kcb.getMetadataRb().getAttributes()); + } + + // TODO(Fabio): I think that this map should be . But I don't want to see + // what happen if I change it. + Map invalidAttrs = new HashMap<>(); + + for (String usedAttr: attributes.keySet()) { + if (!validAttrs.contains(new Attribute(usedAttr))) { + invalidAttrs.put(usedAttr, attributes.get(usedAttr)); + } + } + + if (!invalidAttrs.isEmpty()) { + throw new KaramelException(String.format("Invalid attributes, all used attributes must be defined in metadata.rb " + + "files: %s", invalidAttrs.keySet().toString())); + } + Set> entrySet = cluster.getGroups().entrySet(); for (Map.Entry entry : entrySet) { - String gName = entry.getKey(); - YamlGroup group = entry.getValue(); - JsonGroup jsonGroup = new JsonGroup(cluster, group, gName); - this.groups.add(jsonGroup); + groups.add(new JsonGroup(entry.getValue(), entry.getKey(), cookbooks)); } } @@ -55,4 +76,11 @@ public void setGroups(List groups) { this.groups = groups; } + public Map getRootCookbooks() { + return rootCookbooks; + } + + public void setRootCookbooks(Map rootCookbooks) { + this.rootCookbooks = rootCookbooks; + } } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCookbook.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCookbook.java deleted file mode 100644 index 9be5f340..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCookbook.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.clusterdef.json; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; -import se.kth.karamel.common.cookbookmeta.CookbookUrls; - -/** - * - * @author kamal - */ -public class JsonCookbook { - - String id; - String alias; - String name; - //values of attrs could be string or array of string - Map attrs = new HashMap<>(); - Set recipes = new HashSet<>(); - @JsonIgnore - KaramelizedCookbook karamelizedCookbook; - - public JsonCookbook() { - } - - public JsonCookbook(String id, String alias, String name, Map attrs, - KaramelizedCookbook karamelizedCookbook) { - this.id = id; - this.alias = alias; - this.name = name; - this.attrs = attrs; - this.karamelizedCookbook = karamelizedCookbook; - } - - public String getName() throws KaramelException { - return name; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - public Map getAttrs() { - return attrs; - } - - public void setAttrs(Map attrs) { - this.attrs = attrs; - } - - public Set getRecipes() { - return recipes; - } - - public void setRecipes(Set recipes) { - this.recipes = recipes; - } - - @JsonIgnore - public KaramelizedCookbook getKaramelizedCookbook() { - return karamelizedCookbook; - } - - public CookbookUrls getUrls() throws KaramelException { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - CookbookUrls urls = builder.buildById(id); - return urls; - } - -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonGroup.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonGroup.java index 7001ffee..0f64ea57 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonGroup.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonGroup.java @@ -1,14 +1,9 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.json; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import se.kth.karamel.common.clusterdef.Baremetal; import se.kth.karamel.common.cookbookmeta.Attribute; @@ -16,86 +11,63 @@ import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.exception.RecipeNotfoundException; -import se.kth.karamel.common.clusterdef.yaml.YamlCluster; import se.kth.karamel.common.clusterdef.yaml.YamlGroup; import se.kth.karamel.common.exception.ValidationException; -/** - * - * @author kamal - */ public class JsonGroup extends JsonScope { private String name; private int size; + private List recipes = new ArrayList<>(); + public JsonGroup() { } - public JsonGroup(YamlCluster cluster, YamlGroup group, String name) throws KaramelException { - // We are doing the same work several times here - super(cluster, group); + public JsonGroup(YamlGroup group, String name, List allCookbooks) throws KaramelException { + super(group); setName(name); this.size = group.getSize(); - List recipes = group.getRecipes(); - for (String rec : recipes) { + + List clusterDefRecipes = group.getRecipes(); + for (String rec : clusterDefRecipes) { String[] comp = rec.split(Settings.COOKBOOK_DELIMITER); - JsonCookbook cookbook = null; - for (JsonCookbook cb : getCookbooks()) { - if (cb.getName().equals(comp[0])) { + KaramelizedCookbook cookbook = null; + for (KaramelizedCookbook cb : allCookbooks) { + if (cb.getCookbookName().equals(comp[0])) { cookbook = cb; break; } } if (cookbook == null) { throw new RecipeNotfoundException(String.format("Opps!! Import cookbook for '%s'", rec)); + } else { + recipes.add(new JsonRecipe(cookbook, comp.length == 2 ? comp[1] : "default")); + cookbooks.add(cookbook); } - JsonRecipe jsonRecipe = new JsonRecipe(); - jsonRecipe.setName(rec); - cookbook.getRecipes().add(jsonRecipe); } - getCookbooks().removeIf(jsonCb -> jsonCb.getRecipes().isEmpty()); + attributes = new HashMap<>(group.flattenAttrs()); + Set allValidAttrs = new HashSet<>(); + for (KaramelizedCookbook kcb : cookbooks) { + allValidAttrs.addAll(kcb.getMetadataRb().getAttributes()); + } - Map groupAttrs = new HashMap<>(group.flattenAttrs()); - Set usedAttributes = new HashSet<>(); - for (JsonCookbook jc : getCookbooks()) { - KaramelizedCookbook kcb = jc.getKaramelizedCookbook(); - Set allValidAttrs = new HashSet<>(kcb.getMetadataRb().getAttributes()); - for (KaramelizedCookbook depKcb : kcb.getDependencies()) { - allValidAttrs.addAll(depKcb.getMetadataRb().getAttributes()); - } + // I think that this map should be . But I don't want to see + // what happen if I change it. + Set invalidAttrs = new HashSet<>(); - // I think that this map should be . But I don't want to see - // what happen if I change it. - Map validUsedAttrs = new HashMap<>(); - for (String usedAttr: groupAttrs.keySet()) { - if (allValidAttrs.contains(new Attribute(usedAttr))) { - validUsedAttrs.put(usedAttr, groupAttrs.get(usedAttr)); - usedAttributes.add(usedAttr); - } + for (String usedAttr: attributes.keySet()) { + if (!allValidAttrs.contains(new Attribute(usedAttr))) { + invalidAttrs.add(usedAttr); } - jc.setAttrs(validUsedAttrs); } - if (!usedAttributes.containsAll(groupAttrs.keySet())){ - Set invalidAttrs = groupAttrs.keySet(); - invalidAttrs.removeAll(usedAttributes); + if (!invalidAttrs.isEmpty()) { throw new KaramelException(String.format("Undefined attributes: %s", invalidAttrs.toString())); } } - public Set flattenRecipes() { - Set recipes = new HashSet<>(); - for (JsonCookbook cb : getCookbooks()) { - Set recipes1 = cb.getRecipes(); - for (JsonRecipe jsonRecipe : recipes1) { - recipes.add(jsonRecipe.getCanonicalName()); - } - } - return recipes; - } - public String getName() { return name; } @@ -116,6 +88,10 @@ public void setSize(int size) { this.size = size; } + public List getRecipes() { + return recipes; + } + @Override public void validate() throws ValidationException { super.validate(); diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonRecipe.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonRecipe.java index 72871ae7..1e9dd28a 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonRecipe.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonRecipe.java @@ -1,20 +1,17 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - package se.kth.karamel.common.clusterdef.json; +import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.util.Settings; -/** - * - * @author kamal - */ public class JsonRecipe implements Comparable{ - String name; + private KaramelizedCookbook cookbook; + private String name; + + public JsonRecipe(KaramelizedCookbook cookbook, String name) { + this.cookbook = cookbook; + this.name = name; + } public String getName() { return name; @@ -23,14 +20,22 @@ public String getName() { public void setName(String name) { this.name = name; } - + + public KaramelizedCookbook getCookbook() { + return cookbook; + } + + public void setCookbook(KaramelizedCookbook cookbook) { + this.cookbook = cookbook; + } + public String getCanonicalName() { - return Settings.RECIPE_CANONICAL_NAME(name); + return cookbook.getCookbookName() + Settings.COOKBOOK_DELIMITER + name; } @Override public int compareTo(JsonRecipe o) { - return getCanonicalName().compareTo(o.getCanonicalName()); + return name.compareTo(o.name); } - + } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonScope.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonScope.java index 2e07f719..d5069304 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonScope.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonScope.java @@ -1,100 +1,54 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.json; - import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.clusterdef.Scope; -import se.kth.karamel.common.cookbookmeta.Attribute; import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; -import se.kth.karamel.common.clusterdef.yaml.YamlCluster; import se.kth.karamel.common.clusterdef.yaml.YamlScope; import se.kth.karamel.common.exception.ValidationException; import se.kth.karamel.common.cookbookmeta.CookbookCache; import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -/** - * - * @author kamal - */ public class JsonScope extends Scope { - private final List cookbooks = new ArrayList<>(); + protected final List cookbooks = new ArrayList<>(); + + public Map attributes; + public static CookbookCache CACHE; - + public JsonScope() { } - public JsonScope(YamlCluster cluster, YamlScope scope) throws KaramelException { + public JsonScope(YamlScope scope) throws KaramelException { super(scope); - Map usedAttrs = cluster.flattenAttrs(); - List allCookbooks = CACHE.loadAllKaramelizedCookbooks(cluster); - //filtering invalid(not defined in metadata.rb) attributes from yaml model - for (KaramelizedCookbook kcb : allCookbooks) { - // Get all the valid attributes, also for transient dependency - Set allValidAttrs = new HashSet<>(kcb.getMetadataRb().getAttributes()); - for (KaramelizedCookbook depKcb : kcb.getDependencies()) { - allValidAttrs.addAll(depKcb.getMetadataRb().getAttributes()); - } - - // I think that this map should be . But I don't want to see - // what happen if I change it. - Map validUsedAttrs = new HashMap<>(); - for (String usedAttr: usedAttrs.keySet()) { - if (allValidAttrs.contains(new Attribute(usedAttr))) { - validUsedAttrs.put(usedAttr, usedAttrs.get(usedAttr)); - } - } - - JsonCookbook jck = new JsonCookbook(kcb.getUrls().id, kcb.getMetadataRb().getName(), - kcb.getMetadataRb().getName(), validUsedAttrs, kcb); - cookbooks.add(jck); - } - - Map invalidAttrs = new HashMap<>(); - invalidAttrs.putAll(usedAttrs); - for (JsonCookbook jc : cookbooks) { - Map attrs1 = jc.getAttrs(); - for (Map.Entry entry : attrs1.entrySet()) { - String key = entry.getKey(); - if (invalidAttrs.containsKey(key)) { - invalidAttrs.remove(key); - } - } - } - - if (!invalidAttrs.isEmpty()) { - throw new KaramelException(String.format("Invalid attributes, all used attributes must be defined in metadata.rb " - + "files: %s", invalidAttrs.keySet().toString())); - } - } - public List getCookbooks() { + public List getCookbooks() { return cookbooks; } - public void setCookbooks(List cookbooks) { + public void setCookbooks(List cookbooks) { this.cookbooks.addAll(cookbooks); } @Override - public String getAttr(String key) { - throw new UnsupportedOperationException("Not supported yet."); + public Object getAttr(String key) { + return null; + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; } @Override public void validate() throws ValidationException { super.validate(); } - } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/RuntimeConfiguration.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/RuntimeConfiguration.java new file mode 100644 index 00000000..5e525288 --- /dev/null +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/RuntimeConfiguration.java @@ -0,0 +1,16 @@ +package se.kth.karamel.common.clusterdef.yaml; + +import java.util.HashMap; +import java.util.Map; + +public class RuntimeConfiguration { + private Map recipesParallelism = new HashMap<>(); + + public Map getRecipesParallelism() { + return recipesParallelism; + } + + public void setRecipesParallelism(Map recipeParallelism) { + this.recipesParallelism = recipeParallelism; + } +} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlCluster.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlCluster.java index 7fe5eabf..6c17a931 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlCluster.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlCluster.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.yaml; import java.util.HashMap; @@ -13,14 +8,8 @@ import se.kth.karamel.common.clusterdef.json.JsonGroup; import se.kth.karamel.common.cookbookmeta.CookbookCache; import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.common.exception.MetadataParseException; import se.kth.karamel.common.exception.ValidationException; -import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; -/** - * - * @author kamal - */ public class YamlCluster extends YamlScope { public static CookbookCache CACHE; @@ -31,7 +20,7 @@ public class YamlCluster extends YamlScope { public YamlCluster() { } - public YamlCluster(JsonCluster jsonCluster) throws MetadataParseException, KaramelException { + public YamlCluster(JsonCluster jsonCluster) throws KaramelException { super(jsonCluster); this.name = jsonCluster.getName(); List jsonGroups = jsonCluster.getGroups(); @@ -40,14 +29,8 @@ public YamlCluster(JsonCluster jsonCluster) throws MetadataParseException, Karam groups.put(jsonGroup.getName(), yamlGroup); } - List allCbs = CACHE.loadRootKaramelizedCookbooks(jsonCluster); - for (KaramelizedCookbook kc : allCbs) { - Cookbook ck = new Cookbook(); - ck.setBranch(kc.getUrls().branch); - ck.setCookbook(kc.getUrls().cookbookRelPath); - ck.setGithub(kc.getUrls().orgRepo); - cookbooks.put(kc.getMetadataRb().getName(), ck); - } + attrs.putAll(jsonCluster.getAttributes()); + cookbooks.putAll(jsonCluster.getRootCookbooks()); } public String getName() { diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlGroup.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlGroup.java index aff025a9..a03fa587 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlGroup.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlGroup.java @@ -1,25 +1,18 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.yaml; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; + import se.kth.karamel.common.clusterdef.Baremetal; import se.kth.karamel.common.clusterdef.json.JsonGroup; +import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.exception.MetadataParseException; import se.kth.karamel.common.exception.ValidationException; -/** - * - * @author kamal - */ public class YamlGroup extends YamlScope { private int size; - private final List recipes = new ArrayList<>(); + private List recipes = null; public YamlGroup() { } @@ -27,7 +20,8 @@ public YamlGroup() { YamlGroup(JsonGroup jsonGroup) throws MetadataParseException { super(jsonGroup); this.size = jsonGroup.getSize(); - recipes.addAll(jsonGroup.flattenRecipes()); + attrs.putAll(jsonGroup.getAttributes()); + recipes = jsonGroup.getRecipes().stream().map(JsonRecipe::getCanonicalName).collect(Collectors.toList()); } public int getSize() { @@ -43,13 +37,7 @@ public List getRecipes() { } public void setRecipes(List recipes) { - for (String recipe : recipes) { - setRecipe(recipe); - } - } - - public void setRecipe(String recipe) { - this.recipes.add(recipe); + this.recipes = recipes; } @Override diff --git a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlScope.java b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlScope.java index 056761c7..a96d4000 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlScope.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/clusterdef/yaml/YamlScope.java @@ -1,17 +1,13 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.clusterdef.yaml; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import se.kth.karamel.common.clusterdef.Scope; +import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.json.JsonScope; import se.kth.karamel.common.util.CollectionsUtil; import se.kth.karamel.common.exception.MetadataParseException; @@ -23,16 +19,17 @@ */ public abstract class YamlScope extends Scope { - private Map attrs = new HashMap<>(); + protected Map attrs = new HashMap<>(); public YamlScope() { } public YamlScope(JsonScope scope) throws MetadataParseException { super(scope); - List cookbooks = scope.getCookbooks(); - for (JsonCookbook cb : cookbooks) { - Set> entries = cb.getAttrs().entrySet(); + List cookbooks = scope.getCookbooks(); + for (KaramelizedCookbook cb : cookbooks) { + // TODO(Fabio) this is probably wrong. + Set> entries = new HashSet<>(); for (Map.Entry entry : entries) { foldOutAttr(entry.getKey(), entry.getValue(), attrs); } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Attribute.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Attribute.java index e8b33098..fb8ed644 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Attribute.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Attribute.java @@ -1,17 +1,7 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - package se.kth.karamel.common.cookbookmeta; import com.google.gson.annotations.SerializedName; -/** - * - * @author kamal - */ public class Attribute { String name; String displayName; diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Berksfile.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Berksfile.java deleted file mode 100644 index 5cca7a53..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/Berksfile.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.cookbookmeta; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.log4j.Logger; -import se.kth.karamel.common.clusterdef.Cookbook; -import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.exception.CookbookUrlException; -import se.kth.karamel.common.util.StringUtils; - -/** - * - * @author kamal - */ -public class Berksfile { - - private static final Logger logger = Logger.getLogger(Berksfile.class); - private final List fileLines; - private final Map deps = new HashMap<>(); - public static Pattern LINE_PATTERN = Pattern.compile( - "^cookbook\\s*(\\'[^,^'^\\\"]+\\'|\\\"[^,^'^\\\"]+\\\")\\s*," - + "\\s*github\\s*:\\s*(\\'[^,^'^\\\"]+\\'|\\\"[^,^'^\\\"]+\\\")\\s*" - + "(,\\s*(branch|tag|version)\\s*:\\s*(\\'[^,^'^\\\"]+\\'|\\\"[^,^'^\\\"]+\\\")\\s*)?$"); - public static Set validUrls = new HashSet<>(); - - public Berksfile(String content) throws CookbookUrlException { - this.fileLines = StringUtils.toLines(content); - loadDependencies(); - validateGithubUrls(); - } - - public Map getDeps() { - return deps; - } - - private void loadDependencies() { - for (String line : fileLines) { - String cbName = null; - String cbUrl = null; - String branch = null; - Matcher matcher = LINE_PATTERN.matcher(line); - if (matcher.matches()) { - cbName = matcher.group(1); - cbName = cbName.replaceAll("'|\"", ""); - cbUrl = matcher.group(2); - cbUrl = cbUrl.replaceAll("'|\"", ""); - if (matcher.group(3) != null) { - branch = matcher.group(5); - branch = branch.replaceAll("'|\"", ""); - } else { - branch = Settings.GITHUB_DEFAULT_BRANCH; - } - Cookbook cb = new Cookbook(); - cb.setGithub(cbUrl); - cb.setBranch(branch); - deps.put(cbName, cb); - } - - } - } - - private void validateGithubUrls() throws CookbookUrlException { - if (Settings.CB_CLASSPATH_MODE) { - logger.debug("Skip cookbook dependency check in the classpath mode"); - return; - } - - for (Map.Entry entry : deps.entrySet()) { - String name = entry.getKey(); - Cookbook cb = entry.getValue(); - String homeUrl = cb.getUrls().cookbookUrl; - String errorMsg = String.format("Cookbook-dependency '%s' doesn't refer to a valid url in Berksfile", name); - try { - if (validUrls.contains(homeUrl)) { - logger.debug(String.format("Skip validating url '%s' since it was already validated", homeUrl)); - } else { - logger.debug(String.format("Validating url '%s'", homeUrl)); - URL u = new URL(homeUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - int code = huc.getResponseCode(); - if (code != 200) { - throw new CookbookUrlException(errorMsg); - } - validUrls.add(homeUrl); - } - } catch (IOException ex) { - throw new CookbookUrlException(errorMsg, ex); - } - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - boolean skipLines = true; - - // append all lines that appear after 'metadata' in the Berksfile template - for (String s : fileLines) { - if (!skipLines) { - sb.append(s).append(System.lineSeparator()); - } - if (s.compareToIgnoreCase("metadata") == 0) { - skipLines = false; - } - } - return sb.toString(); - } - -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookCache.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookCache.java index b4e4e7a2..07ff0185 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookCache.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookCache.java @@ -1,37 +1,16 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -/** - * It caches cookbooks' metadata that being read from Github - */ package se.kth.karamel.common.cookbookmeta; import java.util.List; -import java.util.Set; import se.kth.karamel.common.clusterdef.json.JsonCluster; import se.kth.karamel.common.clusterdef.yaml.YamlCluster; import se.kth.karamel.common.exception.KaramelException; -/** - * - * @author kamal - */ public interface CookbookCache { - public KaramelizedCookbook readNew(String cookbookUrl) throws KaramelException; + KaramelizedCookbook get(String cookbookUrl) throws KaramelException; - public KaramelizedCookbook get(String cookbookUrl) throws KaramelException; - - public void prepareParallel(Set cookbookUrls) throws KaramelException; - - public void prepareNewParallel(Set cookbookUrls) throws KaramelException; - - public List loadRootKaramelizedCookbooks(JsonCluster cluster) throws KaramelException; + List loadAllKaramelizedCookbooks(JsonCluster cluster) throws KaramelException; - public List loadAllKaramelizedCookbooks(YamlCluster cluster) throws KaramelException; + List loadAllKaramelizedCookbooks(YamlCluster cluster) throws KaramelException; - public List loadAllKaramelizedCookbooks(JsonCluster cluster) throws KaramelException; - } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookInfoJson.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookInfoJson.java index c022504e..f6bd04b2 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookInfoJson.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookInfoJson.java @@ -13,15 +13,13 @@ * @author kamal */ public class CookbookInfoJson { - private final String id; private final String name; private final String description; private final String version; private final List attributes; private final List recipes; - public CookbookInfoJson(String id, MetadataRb metadataRb) { - this.id = id; + public CookbookInfoJson(MetadataRb metadataRb) { this.name = metadataRb.getName(); this.description = metadataRb.getDescription(); this.version = metadataRb.getVersion(); @@ -29,10 +27,6 @@ public CookbookInfoJson(String id, MetadataRb metadataRb) { this.recipes = metadataRb.getRecipes(); } - public String getId() { - return id; - } - public String getName() { return name; } @@ -52,6 +46,4 @@ public List getAttributes() { public List getRecipes() { return recipes; } - - } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookUrls.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookUrls.java deleted file mode 100644 index 46834448..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/CookbookUrls.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.cookbookmeta; - -import java.util.regex.Matcher; -import se.kth.karamel.common.util.Settings; -import static se.kth.karamel.common.util.Settings.CB_CLASSPATH_MODE; -import static se.kth.karamel.common.util.Settings.COOKBOOKS_PATH; -import static se.kth.karamel.common.util.Settings.CB_BERKSFILE_REL_URL; -import static se.kth.karamel.common.util.Settings.CB_CONFIGFILE_REL_URL; -import static se.kth.karamel.common.util.Settings.CB_DEFAULTRB_REL_URL; -import static se.kth.karamel.common.util.Settings.CB_KARAMELFILE_REL_URL; -import static se.kth.karamel.common.util.Settings.CB_METADATARB_REL_URL; -import static se.kth.karamel.common.util.Settings.GITHUB_BASE_URL; -import static se.kth.karamel.common.util.Settings.GITHUB_RAW_URL; -import static se.kth.karamel.common.util.Settings.GITHUB_REPO_NO_BRANCH_PATTERN; -import static se.kth.karamel.common.util.Settings.GITHUB_REPO_WITH_BRANCH_PATTERN; -import static se.kth.karamel.common.util.Settings.GITHUB_REPO_WITH_SUBCOOKBOOK_PATTERN; -import static se.kth.karamel.common.util.Settings.REPO_NO_BRANCH_PATTERN; -import static se.kth.karamel.common.util.Settings.REPO_WITH_BRANCH_PATTERN; -import static se.kth.karamel.common.util.Settings.REPO_WITH_SUBCOOKBOOK_PATTERN; -import static se.kth.karamel.common.util.Settings.SLASH; -import static se.kth.karamel.common.util.Settings.TEST_CB_ROOT_FOLDER; -import static se.kth.karamel.common.util.Settings.USE_CLONED_REPO_FILES; -import se.kth.karamel.common.exception.CookbookUrlException; - -/** - * - * @author kamal - */ -public class CookbookUrls { - - public String orgName; - public String repoName; - public String orgRepo; - public String repoUrl; - public String branch; - public String cookbookRelPath; - public String id; - public String cookbookUrl; - public String cookbookRawUrl; - public String attrFile; - public String metadataFile; - public String karamelFile; - public String berksFile; - public String configFile; - public String recipesHome; - - public CookbookUrls(String orgName, String repoName, String orgRepo, String repoUrl, String branch, - String cookbookRelPath, String id, String home, String rawHome, String attrFile, String metadataFile, - String karamelFile, String berksFile, String configFile, String recipesHome) { - this.orgName = orgName; - this.repoName = repoName; - this.orgRepo = orgRepo; - this.repoUrl = repoUrl; - this.branch = branch; - this.cookbookRelPath = cookbookRelPath; - this.id = id; - this.cookbookUrl = home; - this.cookbookRawUrl = rawHome; - this.attrFile = attrFile; - this.metadataFile = metadataFile; - this.karamelFile = karamelFile; - this.berksFile = berksFile; - this.configFile = configFile; - this.recipesHome = recipesHome; - } - - public static class Builder { - - String id; - String url; - String repo; - String org; - String branch; - String cookbookRelPath; - - /** - * - * @param url url to reposiory name if files == false. Otherwise the name of the reo - * @return - * @throws CookbookUrlException - */ - public Builder url(String url) throws CookbookUrlException { - if (url.isEmpty()) { - throw new CookbookUrlException("Cookbook url is empty."); - } - this.url = url.trim(); - - return this; - } - - public Builder branchOrVersion(String branch) { - this.branch = branch; - return this; - } - - public Builder cookbookRelPath(String subCookbook) { - this.cookbookRelPath = subCookbook; - return this; - } - - public CookbookUrls buildById(String id) throws CookbookUrlException { - if (id.isEmpty()) { - throw new CookbookUrlException("Cookbook id is empty."); - } - this.id = id.trim(); - - return build(); - } - - public CookbookUrls build() throws CookbookUrlException { - if (id != null) { - //id based data extraction - boolean found = false; - Matcher matcher = GITHUB_REPO_WITH_SUBCOOKBOOK_PATTERN.matcher(id); - if (matcher.matches()) { - org = matcher.group(1); - repo = matcher.group(2); - branch = matcher.group(3); - cookbookRelPath = matcher.group(4); - found = true; - } - - if (!found) { - matcher = GITHUB_REPO_WITH_BRANCH_PATTERN.matcher(id); - if (matcher.matches()) { - org = matcher.group(1); - repo = matcher.group(2); - branch = matcher.group(3); - found = true; - } - - } - - if (!found) { - throw new CookbookUrlException(String.format("'%s' is not a valid cookbook id, it must be the following " - + "format: \n'http(s)://github.com///tree//'", id)); - } - } else { - //url based data extraction - boolean found = false; - Matcher matcher = REPO_WITH_SUBCOOKBOOK_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - - if (!found) { - matcher = GITHUB_REPO_WITH_SUBCOOKBOOK_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - } - - if (found) { - cookbookRelPath = matcher.group(4); - } - - if (!found) { - matcher = REPO_WITH_BRANCH_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - } - - if (!found) { - matcher = GITHUB_REPO_WITH_BRANCH_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - } - - if (found && branch == null) { - branch = matcher.group(3); - } - - if (!found) { - matcher = REPO_NO_BRANCH_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - } - - if (!found) { - matcher = GITHUB_REPO_NO_BRANCH_PATTERN.matcher(url); - if (matcher.matches()) { - found = true; - } - } - - if (found) { - org = matcher.group(1); - repo = matcher.group(2); - if (branch == null) { - branch = Settings.GITHUB_DEFAULT_BRANCH; - } - } else { - throw new CookbookUrlException(String.format("'%s' is not a valid Github url, it must be one the following " - + "formats: \n'http(s)://github.com//', " - + "\n'http(s)://github.com///tree/', " - + "\n'http(s)://github.com///tree//', " - + "\n'/', \n'//tree/'," - + " or \n'//tree//'", url)); - } - } - - String base = CB_CLASSPATH_MODE ? TEST_CB_ROOT_FOLDER : GITHUB_BASE_URL; - String raw = CB_CLASSPATH_MODE ? TEST_CB_ROOT_FOLDER : GITHUB_RAW_URL; - - id = GITHUB_BASE_URL + SLASH + org + SLASH + repo + SLASH + "tree" + SLASH + branch; - String home = base + SLASH + org + SLASH + repo; - String repoHome = base + SLASH + org + SLASH + repo; - String rawHome = raw + SLASH + org + SLASH + repo + SLASH + branch; - - if (cookbookRelPath != null && !cookbookRelPath.isEmpty()) { - String subPath = SLASH + cookbookRelPath; - id += subPath; - home += subPath; - rawHome += subPath; - } - - if (USE_CLONED_REPO_FILES) { - rawHome = COOKBOOKS_PATH + SLASH + repo; - } - - String attrFile = rawHome + CB_DEFAULTRB_REL_URL; - String metadataFile = rawHome + CB_METADATARB_REL_URL; - String karamelFile = rawHome + CB_KARAMELFILE_REL_URL; - String berksFile = rawHome + CB_BERKSFILE_REL_URL; - String configFile = rawHome + CB_CONFIGFILE_REL_URL; - String recipesHome = rawHome + "/recipes/"; - CookbookUrls urls = new CookbookUrls(org, repo, org + SLASH + repo, repoHome, branch, cookbookRelPath, id, home, - rawHome, attrFile, metadataFile, karamelFile, berksFile, configFile, recipesHome); - return urls; - } - - } -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipe.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipe.java deleted file mode 100644 index 84a0a7fd..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipe.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.cookbookmeta; - -import se.kth.karamel.common.util.Settings; - -public class ExperimentRecipe { - - private final String recipeName; - private final String scriptContents; - private final String scriptType; - private final String configFileName; - private final String configFileContents; - - public ExperimentRecipe(String recipeName, String scriptType, - String scriptContents, String configFileName, String configFileContents) { - if (recipeName.contains(Settings.COOKBOOK_DELIMITER)) { - recipeName = recipeName.split(Settings.COOKBOOK_DELIMITER)[1]; - } - this.recipeName = recipeName; - this.scriptContents = scriptContents; - this.scriptType = scriptType; - this.configFileName = configFileName; - this.configFileContents = configFileContents; - } - - public String getScriptContents() { - return scriptContents; - } - - public String getRecipeName() { - return recipeName; - } - - public String getScriptType() { - return scriptType; - } - - public String getConfigFileContents() { - return configFileContents; - } - - public String getConfigFileName() { - return configFileName; - } - -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipeParser.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipeParser.java deleted file mode 100644 index ec42353d..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/ExperimentRecipeParser.java +++ /dev/null @@ -1,63 +0,0 @@ -package se.kth.karamel.common.cookbookmeta; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import se.kth.karamel.common.exception.RecipeParseException; - -public class ExperimentRecipeParser { - - public static Pattern EXPERIMENT_SCRIPT = Pattern.compile("script \'run_experiment\'"); - public static Pattern EXPERIMENT_DELIMITER = Pattern.compile("EOM"); - public static Pattern EXPERIMENT_INTERPRETER = Pattern.compile("interpreter \"(.*)\""); - - /** - * - * @param name - * @param recipeContent - * @param configFileName - * @param configFileContents - * @return an experiment recipe - * @throws se.kth.karamel.common.exception.RecipeParseException - */ - public static ExperimentRecipe parse(String name, String recipeContent, String configFileName, - String configFileContents) throws RecipeParseException { - - Matcher mp = EXPERIMENT_SCRIPT.matcher(recipeContent); - boolean foundPreScript = mp.find(); - if (!foundPreScript) { - throw new RecipeParseException( - "Could not find in the recipe any chef code before a script resource like \"script 'run_experiment' do\" "); - } - // +1 to skip the newline char - String postScript = recipeContent.substring(mp.start()+1); - - Matcher ms = EXPERIMENT_DELIMITER.matcher(postScript); - boolean foundStart = ms.find(); - if (!foundStart) { - throw new RecipeParseException( - "Could not find in the recipe a script resource like \"script 'run_experiment' do\" "); - } - int startPos = ms.end()+1; - - boolean foundEnd = ms.find(); - if (!foundEnd) { - throw new RecipeParseException( - "Could not find in the recipe a script resource like \"script 'run_experiment' do\" "); - } - int endPos = ms.start()-1; - - String script = postScript.substring(startPos, endPos); - - Matcher mi = EXPERIMENT_INTERPRETER.matcher(postScript); - boolean foundInterpreter = mi.find(); - if (!foundInterpreter) { - throw new RecipeParseException( - "Could not find in the Interpreter in script experiment recipe."); - } - String interpreter = mi.group(1); - - - return new ExperimentRecipe(name, interpreter, script, configFileName, configFileContents); - } - -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbook.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbook.java index 05edd5ac..66fcd7bb 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbook.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbook.java @@ -1,182 +1,25 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.cookbookmeta; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import se.kth.karamel.common.util.IoUtils; -import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.exception.CookbookUrlException; -import se.kth.karamel.common.exception.MetadataParseException; -import se.kth.karamel.common.exception.NoKaramelizedCookbookException; -import se.kth.karamel.common.exception.RecipeParseException; -import se.kth.karamel.common.exception.ValidationException; - -/** - * Represents a coobook located in github - * - * @author kamal - */ public class KaramelizedCookbook { - private static final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(KaramelizedCookbook.class); - - private final CookbookUrls urls; - private final DefaultRb defaultRb; - private final MetadataRb metadataRb; - private final KaramelFile karamelFile; - private final Berksfile berksFile; - private final List experimentRecipes = new ArrayList<>(); - private final Set dependencies = new HashSet<>(); - private InstallRecipe installRecipe; + private String cookbookName; + private MetadataRb metadataRb; + private KaramelFile karamelFile; private String json; - private boolean reciepsLoaded = false; - - /** - * - * @param homeUrl url or canonical path to the cookbook - * @param local true if it is a canonical path (to a cloned cookbook) and not a URL. - * @throws CookbookUrlException - * @throws MetadataParseException - * @throws se.kth.karamel.common.exception.ValidationException - * @throws se.kth.karamel.common.exception.NoKaramelizedCookbookException - */ - public KaramelizedCookbook(String homeUrl, boolean local) throws CookbookUrlException, MetadataParseException, - ValidationException, NoKaramelizedCookbookException { - if (local) { - Settings.USE_CLONED_REPO_FILES = true; - } - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - this.urls = builder.url(homeUrl).build(); - String karamelFileC; - try { - karamelFileC = IoUtils.readContent(urls.karamelFile); - } catch (IOException e) { - throw new NoKaramelizedCookbookException(e); - } - try { - String defaultRbC = IoUtils.readContent(urls.attrFile); - String metadataRbC = IoUtils.readContent(urls.metadataFile); - String berksfileC = IoUtils.readContent(urls.berksFile); - this.defaultRb = new DefaultRb(defaultRbC); - this.metadataRb = MetadataParser.parse(metadataRbC); - this.metadataRb.normalizeRecipeNames(); - this.metadataRb.setDefaults(defaultRb); - this.karamelFile = new KaramelFile(karamelFileC); - this.berksFile = new Berksfile(berksfileC); - } catch (IOException e) { - Settings.USE_CLONED_REPO_FILES = false; - throw new CookbookUrlException("", e); - } - } - - public KaramelizedCookbook(CookbookUrls urls, String defaultRbC, String metadataRbC, String karamelFileC, - String berksfileC) throws CookbookUrlException, MetadataParseException, ValidationException { - this.urls = urls; - this.defaultRb = new DefaultRb(defaultRbC); - this.metadataRb = MetadataParser.parse(metadataRbC); - this.metadataRb.normalizeRecipeNames(); - this.metadataRb.setDefaults(defaultRb); - this.karamelFile = new KaramelFile(karamelFileC); - this.berksFile = new Berksfile(berksfileC); - } - - private synchronized void loadrecipes() throws MetadataParseException, CookbookUrlException { - if (!reciepsLoaded) { - List recipes = this.metadataRb.getRecipes(); - for (Recipe r : recipes) { - String name = r.getName(); - if (name == null || name.isEmpty()) { - throw new MetadataParseException("Invalid recipe name in metadata.rb"); - } - String[] recipeData = r.getName().split(Settings.COOKBOOK_DELIMITER); - // assume recipe name is 'default' - String experimentFilename = "default.rb"; - if (recipeData.length > 1) { - experimentFilename = recipeData[1] + ".rb"; - } - String description = r.getDescription(); - String searchStr = "configFile="; - int confPos = description.indexOf(searchStr); - String configFileName = ""; - String configFileContents = ""; - String experimentContent; - if (confPos != -1 && confPos < description.length() + searchStr.length()) { - String desc = description.substring(confPos + searchStr.length()); - int pos = desc.indexOf(";"); - if (pos != -1) { - configFileName = desc.substring(0, pos); - int pathPos = configFileName.lastIndexOf("/"); - if (pathPos != -1) { - configFileName = configFileName.substring(pathPos + 1); - } - } - } - if (!configFileName.isEmpty()) { - String configFileUrl = urls.cookbookRawUrl + File.separator + "templates" + File.separator - + "defaults" + File.separator + configFileName + ".erb"; - try { - configFileContents = IoUtils.readContent(configFileUrl); - } catch (IOException ex) { - logger.debug("Not found in this cookbook: " + urls.recipesHome + experimentFilename, ex); - } - } - - ExperimentRecipe er = null; - try { - // Only parse experiment recipes here, parse the install.rb recipe later. - if (experimentFilename.compareTo(Settings.INSTALL_RECIPE + ".rb") != 0) { - experimentContent = IoUtils.readContent(urls.recipesHome + experimentFilename); - er = ExperimentRecipeParser.parse(r.getName(), experimentContent, configFileName, configFileContents); - } - } catch (IOException ex) { - logger.debug("This cookbook does not have a karamelized experiment: " + urls.recipesHome + experimentFilename - + " - " + ex.getMessage()); - } catch (RecipeParseException ex) { - logger.warn("The recipe is not in a karamelized format: " + urls.recipesHome + experimentFilename - + " - " + ex.getMessage()); - } - if (er != null) { - experimentRecipes.add(er); - } - - } - - try { - String installContent = IoUtils.readContent(urls.recipesHome + "install.rb"); - this.installRecipe = InstallRecipeParser.parse(installContent); - } catch (IOException ex) { - throw new CookbookUrlException( - "Could not find the file 'recipes/install.rb'. Does the file exist? Is the Internet working?"); - } catch (RecipeParseException ex) { - logger.warn("Install recipe not in a format that can be used by Karamel Experiments: " - + urls.recipesHome + "install.rb" + " - " + ex.getMessage()); - - } finally { - Settings.USE_CLONED_REPO_FILES = false; - } - } - reciepsLoaded = true; - } - - public Berksfile getBerksFile() { - return berksFile; + public KaramelizedCookbook(MetadataRb metadata, KaramelFile karamelFile) { + this.cookbookName = metadata.getName(); + this.metadataRb = metadata; + this.karamelFile = karamelFile; } + // TODO(Fabio): this is probably useless here. public String getInfoJson() { if (json == null) { - CookbookInfoJson cookbookInfoJson = new CookbookInfoJson(urls.id, metadataRb); + CookbookInfoJson cookbookInfoJson = new CookbookInfoJson(metadataRb); GsonBuilder builder = new GsonBuilder(); builder.disableHtmlEscaping(); Gson gson = builder.setPrettyPrinting().create(); @@ -193,31 +36,19 @@ public KaramelFile getKaramelFile() { return karamelFile; } - public List getExperimentRecipes() throws MetadataParseException, CookbookUrlException { - loadrecipes(); - return experimentRecipes; + public String getCookbookName() { + return cookbookName; } - public InstallRecipe getInstallRecipe() throws MetadataParseException, CookbookUrlException { - loadrecipes(); - return installRecipe; + public void setCookbookName(String cookbookName) { + this.cookbookName = cookbookName; } - public DefaultRb getDefaultRb() { - return defaultRb; + public void setMetadataRb(MetadataRb metadataRb) { + this.metadataRb = metadataRb; } - public CookbookUrls getUrls() { - return urls; + public void setKaramelFile(KaramelFile karamelFile) { + this.karamelFile = karamelFile; } - - public void addDependency(KaramelizedCookbook cookbook) { - this.dependencies.add(cookbook); - } - - public void addDependencies(Set cookbooks) { - this.dependencies.addAll(cookbooks); - } - - public Set getDependencies() { return this.dependencies; } } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/MetadataRb.java b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/MetadataRb.java index 33acf4b3..2ad149e1 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/MetadataRb.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/cookbookmeta/MetadataRb.java @@ -1,27 +1,15 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.cookbookmeta; import java.util.ArrayList; import java.util.List; -import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.exception.ValidationException; -/** - * Represents Chef metadata.rb file - * - * @author kamal - */ public class MetadataRb { - String name; - String description; - String version; - List recipes = new ArrayList<>(); - List attributes = new ArrayList<>(); + private String name; + private String description; + private String version; + private List recipes = new ArrayList<>(); + private List attributes = new ArrayList<>(); public void setAttributes(List attributes) { this.attributes = attributes; @@ -62,27 +50,4 @@ public void setVersion(String version) { public String getVersion() { return version; } - - public void setDefaults(DefaultRb defaultRb) { - for (Attribute attr : attributes) { - if (defaultRb.getValue(attr.getName()) != null) { - attr.setDefault(defaultRb.getValue(attr.getName())); - } - } - } - - public void normalizeRecipeNames() throws ValidationException { - if (this.name == null || this.name.isEmpty()) { - throw new ValidationException("name of cookbook is mandatory in metadata file"); - } - for (Recipe recipe : recipes) { - if (!recipe.getName().contains(Settings.COOKBOOK_DELIMITER)) { - if (!recipe.getName().equals(name)) { - recipe.setName(name + Settings.COOKBOOK_DELIMITER + recipe.getName()); - } - } - - } - } - } diff --git a/karamel-common/src/main/java/se/kth/karamel/common/util/Confs.java b/karamel-common/src/main/java/se/kth/karamel/common/util/Confs.java index e6eee7c3..457e1041 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/util/Confs.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/util/Confs.java @@ -10,6 +10,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; import java.util.HashSet; import java.util.Properties; @@ -43,7 +44,7 @@ public static void setMemConfs(Confs confs) { } public void writeKaramelConfs() { - File folder = new File(Settings.KARAMEL_ROOT_PATH); + File folder = new File(Settings.getKaramelRootPath()); writeConfs(folder); } @@ -102,7 +103,7 @@ public static Confs loadAllConfsForCluster(String clusterName) { public static Confs loadKaramelConfs() { if (memConfs == null) { - return loadConfs(Settings.KARAMEL_ROOT_PATH); + return loadConfs(Settings.getKaramelRootPath()); } else { return applyDefaults(memConfs); } @@ -158,9 +159,31 @@ public static Confs applyDefaults(Confs prop) { prop.put(Settings.CHEFDK_VERSION_KEY, Settings.CHEFDK_VERSION_DEFAULT); } + String chefFileCachePath = prop.getProperty(Settings.CHEF_FILE_CACHE_PATH); + if (chefFileCachePath == null) { + String tmp = System.getProperty("java.io.tmpdir"); + prop.put(Settings.CHEF_FILE_CACHE_PATH, Paths.get(tmp, "chef-solo").toString()); + } + + String sudoBinary = prop.getProperty(Settings.CHEF_SUDO_BINARY); + if (sudoBinary == null) { + prop.put(Settings.CHEF_SUDO_BINARY, Settings.DEFAULT_CHEF_SUDO_BINARY); + } + + String airgap = prop.getProperty(Settings.KARAMEL_AIRGAP); + if (airgap == null) { + prop.put(Settings.KARAMEL_AIRGAP, Settings.DEFAULT_KARAMEL_AIRGAP); + } return prop; } + + public String getProperty(String key) { + String envVariableName = key.replaceAll("\\.", "_").toUpperCase(); + String value = System.getenv(envVariableName); + return value != null ? value : super.getProperty(key); + } + @Override public synchronized Confs clone() { Confs clone = new Confs(); diff --git a/karamel-common/src/main/java/se/kth/karamel/common/util/IoUtils.java b/karamel-common/src/main/java/se/kth/karamel/common/util/IoUtils.java deleted file mode 100644 index b452ee35..00000000 --- a/karamel-common/src/main/java/se/kth/karamel/common/util/IoUtils.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.util; - -import com.google.common.base.Charsets; -import com.google.common.io.Files; -import com.google.common.io.Resources; -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; - -/** - * - * @author kamal - */ -public class IoUtils { - - static class Worker implements Runnable { - - ConcurrentHashMap map; - String url; - - public Worker(ConcurrentHashMap map, String url) { - this.map = map; - this.url = url; - } - - @Override - public void run() { - try { - String content = readContent(url); - map.put(url, content); - } catch (IOException ex) { - } - } - } - - public static List readLines(String url) throws IOException { - if (Settings.CB_CLASSPATH_MODE) { - return readLinesFromClasspath(url); - } else if (Settings.USE_CLONED_REPO_FILES) { - return readLinesFromPath(url); - } else { - return readLinesFromWeb(url); - } - } - - public static String readContent(String path) throws IOException { - if (Settings.CB_CLASSPATH_MODE) { - return readContentFromClasspath(path); - } else if (Settings.USE_CLONED_REPO_FILES) { - return readContentFromPath(path); - } else { - return readContentFromWeb(path); - } - } - - public static Map readContentParallel(Set paths, ExecutorService tp) { - ConcurrentHashMap map = new ConcurrentHashMap<>(); - Set workers = new HashSet<>(); - Collection> futures = new LinkedList<>(); - - for (String path : paths) { - Worker worker = new Worker(map, path); - workers.add(worker); - futures.add(tp.submit(worker)); - } - for (Future future : futures) { - try { - future.get(); - } catch (InterruptedException ex) { - } catch (ExecutionException ex) { - } - } - return map; - } - - public static String readContentFromClasspath(String path) throws IOException { - URL url = Resources.getResource(path); - if (url == null) { - throw new IOException("No config.props file found in cookbook"); - } - return Resources.toString(url, Charsets.UTF_8); - } - - public static String readContentFromPath(String path) throws IOException { - return Files.toString(new File(path), Charsets.UTF_8); - } - - public static List readLinesFromClasspath(String url) throws IOException { - return Resources.readLines(Resources.getResource(url), Charsets.UTF_8); - } - - public static List readLinesFromPath(String url) throws IOException { - return Files.readLines(new File(url), Charsets.UTF_8); - } - - public static List readLinesFromWeb(String url) throws IOException { - URL fileUrl = new URL(url); - return Resources.readLines(fileUrl, Charsets.UTF_8); - } - - public static String readContentFromWeb(String url) throws IOException { - URL fileUrl = new URL(url); - return Resources.toString(fileUrl, Charsets.UTF_8); - } -} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/util/IpAddressUtil.java b/karamel-common/src/main/java/se/kth/karamel/common/util/IpAddressUtil.java index 2fa84e5c..9a297c2c 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/util/IpAddressUtil.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/util/IpAddressUtil.java @@ -19,13 +19,6 @@ public class IpAddressUtil { public static Pattern IP_PATTERN = Pattern.compile(Settings.IP_REGEX); - public static void main(String[] args) { - try { - ipRange("192.168.0.1", "192.168.1.3"); - } catch (IpAddressException ex) { - } - } - public static List ipRange(String ipStr) throws IpAddressException { if (ipStr.contains("-")) { String[] indivIp = ipStr.split("-"); diff --git a/karamel-common/src/main/java/se/kth/karamel/common/util/ProcOutputConsumer.java b/karamel-common/src/main/java/se/kth/karamel/common/util/ProcOutputConsumer.java new file mode 100644 index 00000000..2a997e0b --- /dev/null +++ b/karamel-common/src/main/java/se/kth/karamel/common/util/ProcOutputConsumer.java @@ -0,0 +1,43 @@ +package se.kth.karamel.common.util; + +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.concurrent.Callable; + +public class ProcOutputConsumer implements Callable { + + private static final Logger LOGGER = Logger.getLogger(ProcOutputConsumer.class.toString()); + + private InputStream in; + + public ProcOutputConsumer(InputStream in) { + this.in = in; + } + + @Override + public String call() throws Exception { + + StringBuilder outputBuilder = new StringBuilder(); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + + char[] charArray = new char[1000]; + try { + int actualBuffered = 0; + while ((actualBuffered = br.read(charArray, 0, 1000)) != -1) { + outputBuilder.append(charArray, 0, actualBuffered); + } + } catch (IOException e) { + LOGGER.error("Could not read the process output", e); + } finally { + try { + br.close(); + } catch (IOException e) { + } + } + return outputBuilder.toString(); + } +} diff --git a/karamel-common/src/main/java/se/kth/karamel/common/util/Settings.java b/karamel-common/src/main/java/se/kth/karamel/common/util/Settings.java index ce9e0777..62349581 100644 --- a/karamel-common/src/main/java/se/kth/karamel/common/util/Settings.java +++ b/karamel-common/src/main/java/se/kth/karamel/common/util/Settings.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.common.util; import org.apache.log4j.Logger; @@ -12,6 +7,7 @@ import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -25,12 +21,14 @@ */ public class Settings { + // ---- Added by Fabio + public static final String WORKING_DIR = "/tmp/karamel"; + public static boolean USE_CLONED_REPO_FILES = true; + private static final Logger logger = Logger.getLogger(Settings.class); //test public static boolean CB_CLASSPATH_MODE = false; - // files - public static boolean USE_CLONED_REPO_FILES = false; //read public static final String ATTR_DELIMITER = "/"; @@ -78,7 +76,6 @@ public class Settings { public static final String SCRIPT_PATH_ROOT = "se/kth/karamel/backend/shellscripts/"; public static final String SCRIPT_PATH_APTGET_ESSENTIALS = SCRIPT_PATH_ROOT + "aptget_essentials.sc"; public static final String SCRIPT_FIND_OSTYPE = SCRIPT_PATH_ROOT + "find_ostype.sc"; - public static final String SCRIPT_PATH_SUDO_PASSWORD_CHECK = SCRIPT_PATH_ROOT + "sudo_password_check.sc"; public static final String SCRIPT_PATH_CLONE_VENDOR_COOKBOOK = SCRIPT_PATH_ROOT + "clone_vendor_cookbook.sb"; public static final String SCRIPET_PATH_PREPARE_STORAGE = SCRIPT_PATH_ROOT + "prepare_storages.sh"; public static final String SCRIPT_NAME_INSTALL_CHEFDK = "install_chefdk.sh"; @@ -106,6 +103,15 @@ public static final List UNIQUE_VM_NAMES(String provider, String cluster public static final String SKIP_EXISTINGTASKS_KEY = "karamel.skip.existing.tasks"; public static final String SKIP_EXISTINGTASKS_DEFAULT = "false"; + public static final String CHEF_FILE_CACHE_PATH = "chef.file_cache.path"; + public static final String CHEF_RUBYGEMS_URL = "chef.rubygems_url"; + public static final String CHEF_SUDO_BINARY = "chef.sudo_binary"; + public static final String DEFAULT_CHEF_SUDO_BINARY = "sudo"; + + public static final String KARAMEL_AIRGAP = "karamel.airgap"; + public static final String DEFAULT_KARAMEL_AIRGAP = "false"; + private static final String KARAMEL_WORKING_DIRECTORY = "KARAMEL_WORKING_DIRECTORY"; + //--------------------------------------------Baremetal--------------------------------------------------------------- public static final String PROVIDER_BAREMETAL_DEFAULT_USERNAME = "root"; public static final int BAREMETAL_DEFAULT_SSH_PORT = 22; @@ -125,7 +131,6 @@ public static final List UNIQUE_VM_NAMES(String provider, String cluster public static final String AWS_ACCESSKEY_ENV_VAR = "AWS_ACCESS_KEY_ID"; public static final String AWS_SECRETKEY_KEY = "aws.secret.key"; public static final String AWS_SECRETKEY_ENV_VAR = "AWS_SECRET_ACCESS_KEY"; - public static final String AWS_KEYPAIR_NAME_KEY = "aws.keypair.name"; public static final Integer AWS_BATCH_SIZE_DEFAULT = 1; public static final String AWS_BATCH_SIZE_KEY = "aws.batch.size"; public static final int AWS_RETRY_INTERVAL = 6 * 1000; @@ -175,25 +180,17 @@ public static final String GCE_UNIQUE_FIREWALL_NAME(String networkName, String p public static final String OCCI_DEFAULT_IMAGE_SIZE = "atlas"; public static final String OCCI_USER_CERTIFICATE_PATH = "/tmp/x509up_u1000"; public static final String OCCI_CERTIFICATE_DIR = "/etc/grid-security/certificates/"; - public static final List OCCI_VM_PORTS_DEFAULT = Arrays.asList(new String[]{"22"}); //------------------------------------Cookbooks on Github------------------------------------------------------------- - public static final String CB_DEFAULTRB_REL_URL = "/attributes/default.rb"; public static final String CB_METADATARB_REL_URL = "/metadata.rb"; public static final String CB_KARAMELFILE_REL_URL = "/Karamelfile"; public static final String CB_BERKSFILE_REL_URL = "/Berksfile"; - public static final String CB_CONFIGFILE = "config.props"; - public static final String CB_CONFIGFILE_REL_URL = "/templates/default/" + CB_CONFIGFILE; // ---------------------------------Cookbooks Scaffolding on Karamel Machine------------------------------------------ public static final String CB_TEMPLATE_PATH_ROOT = "se" + File.separator + "kth" + File.separator + "karamel" + File.separator + "backend" + File.separator + "templates" + File.separator; public static final String CB_TEMPLATE_RECIPE_INSTALL = CB_TEMPLATE_PATH_ROOT + "recipe_install"; - public static final String CB_TEMPLATE_RECIPE_EXPERIMENT = CB_TEMPLATE_PATH_ROOT + "recipe_experiment"; - public static final String CB_TEMPLATE_CONFIG_PROPS = CB_TEMPLATE_PATH_ROOT + CB_CONFIGFILE; - public static final String CB_TEMPLATE_KITCHEN_YML = CB_TEMPLATE_PATH_ROOT + "kitchen_yml"; public static final String CB_TEMPLATE_METADATA = CB_TEMPLATE_PATH_ROOT + "metadata"; - public static final String CB_TEMPLATE_KARAMELFILE = CB_TEMPLATE_PATH_ROOT + "Karamelfile"; public static final String CB_TEMPLATE_BERKSFILE = CB_TEMPLATE_PATH_ROOT + "Berksfile"; public static final String CB_TEMPLATE_README = CB_TEMPLATE_PATH_ROOT + "README.md"; public static final String CB_TEMPLATE_ATTRIBUTES_DEFAULT = CB_TEMPLATE_PATH_ROOT + "attributes_default"; @@ -280,10 +277,14 @@ public static String EXPERIMENT_RESULT_REMOTE_PATH(String recipeName) { } public static String REMOTE_USER_HOME_PATH(String sshUserName) { - return REMOTE_HOME_ROOT + "/" + sshUserName; + return USER_HOME != null ? USER_HOME : Paths.get(REMOTE_HOME_ROOT, sshUserName).toString(); } public static String REMOTE_WORKING_DIR_PATH(String sshUserName) { + String karamelWorkingDirectory = System.getenv(KARAMEL_WORKING_DIRECTORY); + if (karamelWorkingDirectory != null) { + return karamelWorkingDirectory; + } return REMOTE_USER_HOME_PATH(sshUserName) + "/" + REMOTE_WORKING_DIR_NAME; } @@ -307,9 +308,6 @@ public static String REMOTE_OSTYPE_PATH(String sshUserName) { return REMOTE_INSTALL_DIR_PATH(sshUserName) + "/" + OSTYPE_FILE_NAME; } - public static String REMOTE_PIDFILE_PATH(String sshUserName) { - return REMOTE_INSTALL_DIR_PATH(sshUserName) + "/" + PID_FILE_NAME; - } //------------------------------------------Karamel Machine----------------------------------------------------------- public static final String USER_HOME = System.getProperty("user.home"); public static final String USER_NAME = System.getProperty("user.name"); @@ -322,18 +320,25 @@ public static String REMOTE_PIDFILE_PATH(String sshUserName) { public static final String SSH_PUBKEY_PATH_KEY = "ssh.publickey.path"; public static final String SSH_PRIVKEY_PATH_KEY = "ssh.privatekey.path"; public static final String TEST_CB_ROOT_FOLDER = "testgithub"; - public static final String KARAMEL_ROOT_PATH = USER_HOME + File.separator + ".karamel"; - public static final String COOKBOOKS_PATH = KARAMEL_ROOT_PATH + File.separator + "cookbooks"; + public static final String COOKBOOKS_PATH = getKaramelRootPath() + File.separator + "cookbooks"; public static final String YAML_FILE_NAME = "definition.yaml"; public static final String KARAMEL_CONF_NAME = "conf"; public static final String SSH_FOLDER_NAME = ".ssh"; public static final String STATS_FOLDER_NAME = "stats"; - public static final String KARAMEL_SSH_PATH = KARAMEL_ROOT_PATH + File.separator + SSH_FOLDER_NAME; - public static final String KARAMEL_TMP_PATH = KARAMEL_ROOT_PATH + File.separator + TMP_FOLDER_NAME; + public static final String KARAMEL_SSH_PATH = getKaramelRootPath() + File.separator + SSH_FOLDER_NAME; + public static final String KARAMEL_TMP_PATH = getKaramelRootPath() + File.separator + TMP_FOLDER_NAME; public static final String SSH_PUBKEY_FILENAME = "ida_rsa.pub"; public static final String SSH_PRIVKEY_FILENAME = "ida_rsa"; public static final String RECIPE_RESULT_POSFIX = "__out.json"; + public static String getKaramelRootPath() { + String karamelWorkingDirectory = System.getenv(KARAMEL_WORKING_DIRECTORY); + if (karamelWorkingDirectory != null) { + return karamelWorkingDirectory; + } + return Paths.get(USER_HOME, ".karamel").toString(); + } + public static String loadIpAddress() { String address = "UnknownHost"; try { @@ -357,7 +362,7 @@ public static String TASK_LOG_FILE_PATH(String clusterName, String machinIp, Str } public static String CLUSTER_ROOT_PATH(String clusterName) { - return KARAMEL_ROOT_PATH + File.separator + clusterName.toLowerCase(); + return getKaramelRootPath() + File.separator + clusterName.toLowerCase(); } public static String CLUSTER_SSH_PATH(String clusterName) { @@ -421,7 +426,7 @@ public static String EXPERIMENT_RESULT_LOCAL_PATH(String recipeName, String clus SimpleDateFormat sdf = new SimpleDateFormat("yyMMddHHmmssZ"); - return KARAMEL_ROOT_PATH + File.separator + "results" + File.separator + clusterName.toLowerCase() + return getKaramelRootPath() + File.separator + "results" + File.separator + clusterName.toLowerCase() + File.separator + recName.replace(COOKBOOK_DELIMITER, REMOTE_CB_FS_PATH_DELIMITER) + File.separator + recName.replace(COOKBOOK_DELIMITER, REMOTE_CB_FS_PATH_DELIMITER) + "-" + sdf.format(new Date(System.currentTimeMillis())) + ".out"; diff --git a/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbookTest.java b/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbookTest.java index a77ad469..2d29a918 100644 --- a/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbookTest.java +++ b/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/KaramelizedCookbookTest.java @@ -11,9 +11,7 @@ import org.junit.Test; import se.kth.karamel.common.exception.CookbookUrlException; import se.kth.karamel.common.exception.MetadataParseException; -import se.kth.karamel.common.exception.RecipeParseException; import se.kth.karamel.common.exception.ValidationException; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.Settings; import java.io.IOException; @@ -177,22 +175,4 @@ public void testLoadDependencies() throws CookbookUrlException, IOException { String content = IoUtils.readContentFromClasspath("testgithub/testorg/testrepo/master/cookbooks/hopshadoop/hopsworks-chef/Berksfile"); Berksfile berksfile = new Berksfile(content); } - - @Test - public void testParseRecipes() throws CookbookUrlException, IOException { - try { - Settings.CB_CLASSPATH_MODE = true; - String recipe = Resources.toString(Resources.getResource( - "testgithub/testorg/testrepo/master/cookbooks/hopshadoop/hopsworks-chef/recipes/experiment.rb"), Charsets.UTF_8); - ExperimentRecipe er = ExperimentRecipeParser.parse("experiment", recipe, "config.props", "x=y"); - assertEquals("experiment", er.getRecipeName()); - assertEquals(er.getConfigFileName().isEmpty(), false); - assertEquals(er.getConfigFileContents().isEmpty(), false); - assertEquals(er.getScriptContents().isEmpty(), false); - assertEquals(er.getScriptType(), "bash"); - } catch (RecipeParseException ex) { - Assert.fail(ex.toString()); - } - - } } diff --git a/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/MetadataParserTest.java b/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/MetadataParserTest.java index 4e23efc4..97b19513 100644 --- a/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/MetadataParserTest.java +++ b/karamel-common/src/test/java/se/kth/karamel/common/cookbookmeta/MetadataParserTest.java @@ -8,7 +8,6 @@ import com.google.common.collect.Lists; import org.junit.Test; import se.kth.karamel.common.exception.MetadataParseException; -import se.kth.karamel.common.util.IoUtils; import java.io.IOException; import java.util.ArrayList; diff --git a/karamel-common/src/test/java/se/kth/karamel/common/util/IoUtilTestIT.java b/karamel-common/src/test/java/se/kth/karamel/common/util/IoUtilTestIT.java deleted file mode 100644 index 4ac32920..00000000 --- a/karamel-common/src/test/java/se/kth/karamel/common/util/IoUtilTestIT.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.common.util; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Executors; -import org.junit.Test; - -/** - * - * @author kamal - */ -public class IoUtilTestIT { - - @Test - public void testduration() throws IOException { - Set paths = new HashSet<>(); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/Karamelfile"); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/metadata.rb"); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/attributes/default.rb"); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/Berksfile"); - - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/README.md"); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/Rakefile"); - paths.add("https://raw.githubusercontent.com/hopshadoop/apache-hadoop-chef/master/wrongurl"); - - long start = System.currentTimeMillis(); - Map map = IoUtils.readContentParallel(paths, Executors.newFixedThreadPool(7)); - long end = System.currentTimeMillis(); - System.out.println("it took " + ((end - start)/1000) + "s for " + paths.size() + " files, it returned " - + map.size() + " contet"); - } -} diff --git a/karamel-core/pom.xml b/karamel-core/pom.xml index 369d9de2..175c960f 100644 --- a/karamel-core/pom.xml +++ b/karamel-core/pom.xml @@ -5,7 +5,7 @@ se.kth.karamel karamel-parent - 0.5 + 0.10 se.kth.karamel @@ -83,6 +83,11 @@ google-compute-engine ${jclouds.version} + + org.apache.jclouds.provider + azureblob + ${jclouds.version} + commons-io commons-io @@ -152,6 +157,12 @@ 1.0 + + + src/main/resources + true + + diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterContext.java b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterContext.java index fc374bad..4f2b078a 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterContext.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterContext.java @@ -6,7 +6,6 @@ package se.kth.karamel.backend; import se.kth.karamel.backend.converter.UserClusterDataExtractor; -import se.kth.karamel.backend.github.GithubApi; import se.kth.karamel.backend.launcher.amazon.Ec2Context; import se.kth.karamel.backend.launcher.google.GceContext; import se.kth.karamel.backend.launcher.nova.NovaContext; @@ -43,11 +42,6 @@ public void setSudoAccountPassword(String sudoAccountPassword) { this.sudoAccountPassword = sudoAccountPassword; } - public String getGithubUsername() { - return GithubApi.getEmail().isEmpty() ? "karamel" : GithubApi.getEmail().substring(0, - GithubApi.getEmail().lastIndexOf("@")); - } - public String getSudoAccountPassword() { return sudoAccountPassword; } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterDefinitionService.java b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterDefinitionService.java index 7ea84651..47719864 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterDefinitionService.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterDefinitionService.java @@ -74,8 +74,7 @@ public static String jsonToYaml(JsonCluster jsonCluster) throws KaramelException yamlPropertyRepresenter.addClassTag(YamlGroup.class, Tag.MAP); yamlPropertyRepresenter.addClassTag(HashSet.class, Tag.MAP); Yaml yaml = new Yaml(yamlPropertyRepresenter, options); - String content = yaml.dump(yamlCluster); - return content; + return yaml.dump(yamlCluster); } public static void saveYaml(String yaml) throws KaramelException { @@ -105,8 +104,7 @@ public static String loadYaml(String clusterName) throws KaramelException { if (!file.exists()) { throw new KaramelException(String.format("yaml '%s' is not available", yamlPath)); } - String yaml = Files.toString(file, Charsets.UTF_8); - return yaml; + return Files.toString(file, Charsets.UTF_8); } catch (IOException ex) { throw new KaramelException("Could not save the yaml ", ex); } @@ -123,7 +121,7 @@ public static void removeDefinition(String clusterName) throws KaramelException public static List listClusters() throws KaramelException { List clusters = new ArrayList(); - File folder = new File(Settings.KARAMEL_ROOT_PATH); + File folder = new File(Settings.getKaramelRootPath()); if (folder.exists()) { File[] files = folder.listFiles(); for (File file : files) { @@ -142,16 +140,13 @@ public static List listClusters() throws KaramelException { public static String jsonToYaml(String json) throws KaramelException { Gson gson = new Gson(); - JsonCluster jsonCluster = gson.fromJson(json, JsonCluster.class - ); + JsonCluster jsonCluster = gson.fromJson(json, JsonCluster.class); return jsonToYaml(jsonCluster); } - public static JsonCluster jsonToJsonObject(String json) throws KaramelException { + public static JsonCluster jsonToJsonObject(String json) { Gson gson = new Gson(); - JsonCluster jsonCluster = gson.fromJson(json, JsonCluster.class - ); - return jsonCluster; + return gson.fromJson(json, JsonCluster.class); } public static JsonCluster yamlToJsonObject(String yaml) throws KaramelException { @@ -176,11 +171,10 @@ public static String yamlToJson(String yaml) throws KaramelException { return serializeJson(jsonObj); } - public static String serializeJson(JsonCluster jsonCluster) throws KaramelException { + public static String serializeJson(JsonCluster jsonCluster) { GsonBuilder builder = new GsonBuilder(); builder.disableHtmlEscaping(); Gson gson = builder.setPrettyPrinting().create(); - String json = gson.toJson(jsonCluster); - return json; + return gson.toJson(jsonCluster); } } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterManager.java b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterManager.java index 821ccd6e..d002b636 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterManager.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterManager.java @@ -39,6 +39,7 @@ import se.kth.karamel.common.stats.PhaseStat; import se.kth.karamel.common.util.Settings; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -347,7 +348,7 @@ private void runDag(boolean installDag) throws Exception { generateClusterChefJsonsForPurge(definition, runtime); currentDag = DagBuilder.getPurgingDag(definition, runtime, stats, machinesMonitor, chefJsons); } - currentDag.start(); + currentDag.start(definition); } catch (Exception ex) { runtime.issueFailure(new Failure(Failure.Type.DAG_FAILURE, ex.getMessage())); throw ex; @@ -365,6 +366,9 @@ private void runDag(boolean installDag) throws Exception { logger.info(String.format("\\o/\\o/\\o/\\o/\\o/'%s' DAG IS DONE \\o/\\o/\\o/\\o/\\o/", definition.getName())); if (ClusterManager.EXIT_ON_COMPLETION) { + Duration cooldown = Duration.ofMinutes(2); + logger.info(String.format("Cooling down for %d minutes before exiting", cooldown.toMinutes())); + TimeUnit.MINUTES.sleep(cooldown.toMinutes()); System.exit(0); } } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterService.java b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterService.java index 03b96116..adf6c4e3 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/ClusterService.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/ClusterService.java @@ -83,28 +83,21 @@ public synchronized void registerEc2Context(String clusterName, Ec2Context ec2Co clusterContexts.put(name, clusterContext); } - public synchronized void registerGceContext(GceContext gceContext) throws KaramelException { + public synchronized void registerGceContext(GceContext gceContext) { commonContext.setGceContext(gceContext); } public synchronized void registerSshKeyPair(SshKeyPair sshKeyPair) throws KaramelException { File pubKey = new File(sshKeyPair.getPublicKeyPath()); - if (pubKey.exists() == false) { + if (!pubKey.exists()) { throw new KaramelException("Could not find public key: " + sshKeyPair.getPublicKeyPath()); } File privKey = new File(sshKeyPair.getPrivateKeyPath()); - if (privKey.exists() == false) { + if (!privKey.exists()) { throw new KaramelException("Could not find private key: " + sshKeyPair.getPrivateKeyPath()); } sshKeyPair.setNeedsPassword(SshKeyService.checkIfPasswordNeeded(sshKeyPair)); -// boolean needsPassword = SshKeyService.checkIfPasswordNeeded(sshKeyPair); -// if (needsPassword) { -// if (sshKeyPair.getPassphrase() == null || sshKeyPair.getPassphrase().isEmpty()) { -// throw new KaramelException("The passphrase needs to be entered for the OpenSshKey."); -// } -// sshKeyPair.setNeedsPassword(true); -// } commonContext.setSshKeyPair(sshKeyPair); } @@ -138,9 +131,10 @@ public synchronized void startCluster(String json) throws KaramelException { JsonCluster jsonCluster = gson.fromJson(json, JsonCluster.class); ClusterDefinitionValidator.validate(jsonCluster); String yml = ClusterDefinitionService.jsonToYaml(jsonCluster); + // TODO(Fabio): This is total BS - The result of writing spaghetti code. //We have to do it again otherwise the global scope attributes get lost //for more info refer to https://github.com/karamelchef/karamel/issues/28 - jsonCluster = ClusterDefinitionService.yamlToJsonObject(yml); + //jsonCluster = ClusterDefinitionService.yamlToJsonObject(yml); ClusterDefinitionService.saveYaml(yml); logger.debug(String.format("Let me see if I can start '%s' ...", jsonCluster.getName())); String clusterName = jsonCluster.getName(); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/Experiment.java b/karamel-core/src/main/java/se/kth/karamel/backend/Experiment.java deleted file mode 100644 index fd4a13cd..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/Experiment.java +++ /dev/null @@ -1,282 +0,0 @@ -package se.kth.karamel.backend; - -import java.util.ArrayList; -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement -public class Experiment { - - /** - * username to run program as - */ - private String user = "karamel"; - - /** - * groupname to run program as - */ - private String group = "karamel"; - - /** - * Repository on GitHub - */ - private String githubRepo = ""; - - /** - * Description of the experiment cookbook. - */ - private String description = "Karamel experiment repository description placeholder"; - - private String githubOwner = ""; - - /** - * Comma-separated String of Cookbook::recipe global dependencies used to generate the KaramelFile - */ - private String localDependencies=""; - - /** - * Comma-separated String of Cookbook::recipe local dependencies used to generate the KaramelFile - */ - private String globalDependencies=""; - - /** - * Comma-separated list of Berksfile dependencies. Each entry is enclosed in double-quotation marks. - */ - private String berksfile = ""; - - /** - * Url for the experiment binary. Typically, a .tar.gz extention. - */ - private String urlBinary = ""; - - /** - * Url for the experiment source code. Typically, a github URL. - */ - private String urlGitClone = ""; - - /** - * Maven command to build experiment source code. - */ - private String mavenCommand = ""; - - /** - * Chef code to be executed before the experiment in the Install phase - */ - private String experimentSetupCode = ""; - - - /** - * default/attributes.rb in Chef - */ - private String defaultAttributes = ""; - - /** - * YAML for the Cluster context - */ - private String clusterDefinition = ""; - - - private ArrayList code = new ArrayList<>(); - - - @XmlRootElement - public static class Code { - - private String name; - private String scriptContents; - private String configFileName; - private String configFileContents; - private String scriptType; - - /** - * Create an experiment as a Chef recipe. - * - * @param name - * @param scriptContents - * @param configFileName - * @param configFileContents - * @param scriptType - */ - public Code(String name, String scriptContents, String configFileName, String configFileContents, - String scriptType) { - this.name = name; - this.scriptContents = scriptContents; - this.configFileName = configFileName == null ? "" : configFileName; - this.configFileContents = configFileContents == null ? "" : configFileContents; - this.scriptType = scriptType; - } - - public Code() { - } - - public String getName() { - return name; - } - - public void setName(String experimentName) { - this.name = experimentName; - } - - public String getConfigFileContents() { - return configFileContents; - } - - public void setConfigFileContents(String configFileContents) { - this.configFileContents = configFileContents; - } - - - public String getConfigFileName() { - return configFileName; - } - - public void setConfigFileName(String configFileName) { - this.configFileName = configFileName; - } - - public void setScriptContents(String script) { - this.scriptContents = script; - } - - public String getScriptContents() { - return scriptContents; - } - - public String getScriptType() { - return scriptType; - } - - public String getScriptCommand() { - return scriptType; - } - - public void setScriptType(String scriptType) { - this.scriptType = scriptType; - } - - } - - public Experiment() { - } - - public ArrayList getCode() { - return code; - } - - public void setCode(ArrayList code) { - this.code = code; - } - - public String getExperimentSetupCode() { - return experimentSetupCode; - } - - public void setExperimentSetupCode(String experimentSetupCode) { - this.experimentSetupCode = experimentSetupCode; - } - - public void setGithubOwner(String githubOwner) { - this.githubOwner = githubOwner; - } - - public String getGithubOwner() { - return githubOwner; - } - - public String getGithubRepo() { - return githubRepo; - } - - public void setGithubRepo(String githubRepo) { - this.githubRepo = githubRepo; - } - - public String getUrlBinary() { - return urlBinary; - } - - public void setUrlBinary(String url) { - this.urlBinary = url; - } - - public String getUrlGitClone() { - return urlGitClone; - } - - public void setUrlGitClone(String urlGitClone) { - this.urlGitClone = urlGitClone; - } - - public String getGlobalDependencies() { - return globalDependencies; - } - - public void setGlobalDependencies(String globalDependencies) { - this.globalDependencies = globalDependencies; - } - - public String getLocalDependencies() { - return localDependencies; - } - - public void setLocalDependencies(String localDependencies) { - this.localDependencies = localDependencies; - } - - public void setMavenCommand(String mavenCommand) { - this.mavenCommand = mavenCommand; - } - - public String getMavenCommand() { - return mavenCommand; - } - - public String getGroup() { - return group; - } - - public String getClusterDefinition() { - return clusterDefinition; - } - - public void setClusterDefinition(String clusterDefinition) { - this.clusterDefinition = clusterDefinition; - } - - public String getUser() { - return user; - } - - - public void setGroup(String group) { - this.group = group; - } - - public void setUser(String user) { - this.user = user; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getDefaultAttributes() { - return defaultAttributes; - } - - public void setDefaultAttributes(String defaultAttributes) { - this.defaultAttributes = defaultAttributes; - } - - public String getBerksfile() { - return berksfile; - } - - public void setBerksfile(String berksfile) { - this.berksfile = berksfile; - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/command/CommandService.java b/karamel-core/src/main/java/se/kth/karamel/backend/command/CommandService.java index a402f1b5..774bc827 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/command/CommandService.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/command/CommandService.java @@ -5,7 +5,9 @@ */ package se.kth.karamel.backend.command; +import com.google.common.base.Charsets; import com.google.common.collect.Lists; +import com.google.common.io.Resources; import com.google.gson.JsonObject; import java.io.IOException; import java.util.ArrayList; @@ -43,7 +45,6 @@ import se.kth.karamel.common.stats.ClusterStats; import se.kth.karamel.common.stats.PhaseStat; import se.kth.karamel.common.stats.TaskStat; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.SshKeyPair; /** @@ -70,9 +71,12 @@ public class CommandService { static { try { - HELP_PAGE_TEMPLATE = IoUtils.readContentFromClasspath("se/kth/karamel/backend/command/helppage"); - HOME_PAGE_TEMPLATE = IoUtils.readContentFromClasspath("se/kth/karamel/backend/command/homepage"); - RUNNING_PAGE_TEMPLATE = IoUtils.readContentFromClasspath("se/kth/karamel/backend/command/running"); + HELP_PAGE_TEMPLATE = Resources.toString( + Resources.getResource("se/kth/karamel/backend/command/helppage"), Charsets.UTF_8); + HOME_PAGE_TEMPLATE = Resources.toString( + Resources.getResource("se/kth/karamel/backend/command/homepage"), Charsets.UTF_8); + RUNNING_PAGE_TEMPLATE = Resources.toString( + Resources.getResource("se/kth/karamel/backend/command/running"), Charsets.UTF_8); } catch (IOException e) { } @@ -226,10 +230,6 @@ public static CommandResponse processCommand(String command, String... args) thr builder.append(" but it is on pause.").append("\n"); } - if (clusterEntity.isFailed() && !clusterEntity.getFailures().isEmpty()) { - // builder.append(" List of failures: ").append("\n"); - // builder.append(failureTable(clusterEntity.getFailures().values(), true)); - } builder.append("\n\n"); builder.append("Passed Phases:"); builder.append("\n"); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/converter/ChefJsonGenerator.java b/karamel-core/src/main/java/se/kth/karamel/backend/converter/ChefJsonGenerator.java index 82aa6aa9..eb10b408 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/converter/ChefJsonGenerator.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/converter/ChefJsonGenerator.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.converter; import com.google.gson.Gson; @@ -19,16 +14,13 @@ import se.kth.karamel.backend.running.model.GroupRuntime; import se.kth.karamel.backend.running.model.MachineRuntime; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.json.JsonGroup; import se.kth.karamel.common.clusterdef.json.JsonRecipe; +import se.kth.karamel.common.clusterdef.json.JsonScope; +import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.exception.KaramelException; -/** - * - * @author kamal - */ public class ChefJsonGenerator { /** @@ -44,17 +36,16 @@ public class ChefJsonGenerator { * @return map of machineId-recipeName->json */ public static Map generateClusterChefJsonsForPurge(JsonCluster definition, - ClusterRuntime clusterEntity) throws KaramelException { + ClusterRuntime clusterEntity) { Map chefJsons = new HashMap<>(); JsonObject root = new JsonObject(); for (GroupRuntime groupEntity : clusterEntity.getGroups()) { JsonObject clone = cloneJsonObject(root); JsonGroup jsonGroup = UserClusterDataExtractor.findGroup(definition, groupEntity.getName()); - //Adding all attribtues to all chef-jsons - for (JsonCookbook cb : jsonGroup.getCookbooks()) { - addCookbookAttributes(cb, clone); - } - for (JsonCookbook cb : jsonGroup.getCookbooks()) { + //Adding all attributes to all chef-jsons + addScopeAttributes(jsonGroup, root); + + for (KaramelizedCookbook cb : jsonGroup.getCookbooks()) { Map gj = generatePurgeChefJsons(clone, cb, groupEntity); chefJsons.putAll(gj); } @@ -76,27 +67,23 @@ public static Map generateClusterChefJsonsForPurge(JsonClust * @return map of machineId-recipeName->json */ public static Map generateClusterChefJsonsForInstallation(JsonCluster definition, - ClusterRuntime clusterEntity) throws KaramelException { + ClusterRuntime clusterEntity) + throws KaramelException { + Map chefJsons = new HashMap<>(); JsonObject root = new JsonObject(); aggregateIpAddresses(root, definition, clusterEntity); // Add global attributes - for (JsonCookbook cb : definition.getCookbooks()) { - addCookbookAttributes(cb, root); - } + addScopeAttributes(definition, root); for (GroupRuntime groupEntity : clusterEntity.getGroups()) { JsonObject clone = cloneJsonObject(root); JsonGroup jsonGroup = UserClusterDataExtractor.findGroup(definition, groupEntity.getName()); //Adding all attribtues to all chef-jsons - for (JsonCookbook cb : jsonGroup.getCookbooks()) { - addCookbookAttributes(cb, clone); - } - for (JsonCookbook cb : jsonGroup.getCookbooks()) { - Map gj = generateRecipesChefJsons(clone, cb, groupEntity); - chefJsons.putAll(gj); - } + addScopeAttributes(jsonGroup, clone); + + chefJsons.putAll(generateRecipesChefJsons(clone, jsonGroup, groupEntity)); } return chefJsons; } @@ -109,12 +96,12 @@ public static Map generateClusterChefJsonsForInstallation(Js * @return * @throws KaramelException */ - public static Map generatePurgeChefJsons(JsonObject json, JsonCookbook cb, - GroupRuntime groupEntity) throws KaramelException { + private static Map generatePurgeChefJsons(JsonObject json, KaramelizedCookbook cb, + GroupRuntime groupEntity) { Map groupJsons = new HashMap<>(); for (MachineRuntime me : groupEntity.getMachines()) { - String purgeRecipeName = cb.getName() + Settings.COOKBOOK_DELIMITER + Settings.PURGE_RECIPE; + String purgeRecipeName = cb.getCookbookName() + Settings.COOKBOOK_DELIMITER + Settings.PURGE_RECIPE; JsonObject clone = addMachineNRecipeToJson(json, me, purgeRecipeName); groupJsons.put(me.getId() + purgeRecipeName, clone); } @@ -129,23 +116,25 @@ public static Map generatePurgeChefJsons(JsonObject json, Js * @return * @throws KaramelException */ - public static Map generateRecipesChefJsons(JsonObject json, JsonCookbook cb, - GroupRuntime groupEntity) throws KaramelException { + private static Map generateRecipesChefJsons(JsonObject json, JsonGroup jsonGroup, + GroupRuntime groupEntity) throws KaramelException { Map groupJsons = new HashMap<>(); for (MachineRuntime me : groupEntity.getMachines()) { - for (JsonRecipe recipe : cb.getRecipes()) { + for (JsonRecipe recipe : jsonGroup.getRecipes()) { JsonObject clone = addMachineNRecipeToJson(json, me, recipe.getCanonicalName()); groupJsons.put(me.getId() + recipe.getCanonicalName(), clone); } - String installRecipeName = cb.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; - JsonObject clone = addMachineNRecipeToJson(json, me, installRecipeName); - groupJsons.put(me.getId() + installRecipeName, clone); + for (KaramelizedCookbook cookbook : jsonGroup.getCookbooks()) { + String installRecipeName = cookbook.getCookbookName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; + JsonObject clone = addMachineNRecipeToJson(json, me, installRecipeName); + groupJsons.put(me.getId() + installRecipeName, clone); + } } return groupJsons; } - public static JsonObject addMachineNRecipeToJson(JsonObject json, MachineRuntime me, String recipeName) { + private static JsonObject addMachineNRecipeToJson(JsonObject json, MachineRuntime me, String recipeName) { JsonObject clone = cloneJsonObject(json); addMachineIps(clone, me); addRunListForRecipe(clone, recipeName); @@ -169,7 +158,7 @@ public static void addRunListForRecipe(JsonObject chefJson, String recipeName) { * @param json * @param machineEntity */ - public static void addMachineIps(JsonObject json, MachineRuntime machineEntity) { + private static void addMachineIps(JsonObject json, MachineRuntime machineEntity) { JsonArray ips = new JsonArray(); ips.add(new JsonPrimitive(machineEntity.getPrivateIp())); json.add("private_ips", ips); @@ -186,22 +175,15 @@ public static void addMachineIps(JsonObject json, MachineRuntime machineEntity) /** * It adds those attributes related to one cookbook into the json object. * For example [ndb/ports=[123, 134, 145], ndb/DataMemory=111] - * @param jc + * @param jsonScope * @param root */ - public static void addCookbookAttributes(JsonCookbook jc, JsonObject root) { - Set> entrySet = jc.getAttrs().entrySet(); + private static void addScopeAttributes(JsonScope jsonScope, JsonObject root) { + Set> entrySet = jsonScope.getAttributes().entrySet(); for (Map.Entry entry : entrySet) { String[] keyComps = entry.getKey().split(Settings.ATTR_DELIMITER); Object value = entry.getValue(); -// Object value = valStr; -// if (valStr.startsWith("$")) { -// if (valStr.contains(".")) { -// value = cluster.getVariable(valStr.substring(1)); -// } else { -// value = getVariable(valStr.substring(1)); -// } -// } + JsonObject o1 = root; for (int i = 0; i < keyComps.length; i++) { String comp = keyComps[i]; @@ -239,7 +221,7 @@ public static void addCookbookAttributes(JsonCookbook jc, JsonObject root) { * @param definition * @param clusterEntity */ - public static void aggregateIpAddresses(JsonObject json, JsonCluster definition, ClusterRuntime clusterEntity) { + private static void aggregateIpAddresses(JsonObject json, JsonCluster definition, ClusterRuntime clusterEntity) { Map> privateIps = new HashMap<>(); Map> publicIps = new HashMap<>(); Map> hosts = new HashMap<>(); @@ -247,25 +229,23 @@ public static void aggregateIpAddresses(JsonObject json, JsonCluster definition, for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(definition, ge.getName()); for (MachineRuntime me : ge.getMachines()) { - for (JsonCookbook jc : jg.getCookbooks()) { - for (JsonRecipe recipe : jc.getRecipes()) { - if (!recipe.getCanonicalName().endsWith(Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE)) { - String privateAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + - Settings.REMOTE_CHEFJSON_PRIVATEIPS_TAG; - String publicAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + - Settings.REMOTE_CHEFJSON_PUBLICIPS_TAG; - String hostsAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + - Settings.REMOTE_CHEFJSON_HOSTS_TAG; - if (!privateIps.containsKey(privateAttr)) { - privateIps.put(privateAttr, new HashSet()); - publicIps.put(publicAttr, new HashSet()); - hosts.put(hostsAttr, new HashMap()); - } - privateIps.get(privateAttr).add(me.getPrivateIp()); - publicIps.get(publicAttr).add(me.getPublicIp()); - hosts.get(hostsAttr).put(me.getPublicIp(), me.getName()); - hosts.get(hostsAttr).put(me.getPrivateIp(), me.getName()); + for (JsonRecipe recipe : jg.getRecipes()) { + if (!recipe.getCanonicalName().endsWith(Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE)) { + String privateAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + + Settings.REMOTE_CHEFJSON_PRIVATEIPS_TAG; + String publicAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + + Settings.REMOTE_CHEFJSON_PUBLICIPS_TAG; + String hostsAttr = recipe.getCanonicalName() + Settings.ATTR_DELIMITER + + Settings.REMOTE_CHEFJSON_HOSTS_TAG; + if (!privateIps.containsKey(privateAttr)) { + privateIps.put(privateAttr, new HashSet<>()); + publicIps.put(publicAttr, new HashSet<>()); + hosts.put(hostsAttr, new HashMap<>()); } + privateIps.get(privateAttr).add(me.getPrivateIp()); + publicIps.get(publicAttr).add(me.getPublicIp()); + hosts.get(hostsAttr).put(me.getPublicIp(), me.getName()); + hosts.get(hostsAttr).put(me.getPrivateIp(), me.getName()); } } } @@ -282,7 +262,7 @@ public static void aggregateIpAddresses(JsonObject json, JsonCluster definition, * @param root * @param attrs */ - public static void attrMap2Json(JsonObject root, Map> attrs) { + private static void attrMap2Json(JsonObject root, Map> attrs) { for (Map.Entry> entry : attrs.entrySet()) { String[] keyComps = entry.getKey().split(Settings.COOKBOOK_DELIMITER + "|" + Settings.ATTR_DELIMITER); JsonObject o1 = root; @@ -317,7 +297,7 @@ public static void attrMap2Json(JsonObject root, Map * @param root * @param attrs */ - public static void attr2Json(JsonObject root, Map> attrs) { + private static void attr2Json(JsonObject root, Map> attrs) { Set>> entrySet = attrs.entrySet(); for (Map.Entry> entry : entrySet) { String[] keyComps = entry.getKey().split(Settings.COOKBOOK_DELIMITER + "|" + Settings.ATTR_DELIMITER); @@ -344,11 +324,10 @@ public static void attr2Json(JsonObject root, Map> attrs) { } } - public static JsonObject cloneJsonObject(JsonObject jo) { + private static JsonObject cloneJsonObject(JsonObject jo) { Gson gson = new Gson(); JsonElement jelem = gson.fromJson(jo.toString(), JsonElement.class); - JsonObject clone = jelem.getAsJsonObject(); - return clone; + return jelem.getAsJsonObject(); } } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/converter/ShellCommandBuilder.java b/karamel-core/src/main/java/se/kth/karamel/backend/converter/ShellCommandBuilder.java index b681376c..78932e04 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/converter/ShellCommandBuilder.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/converter/ShellCommandBuilder.java @@ -1,66 +1,27 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.converter; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Scanner; +import java.util.regex.Matcher; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; import se.kth.karamel.backend.running.model.tasks.ShellCommand; -import se.kth.karamel.common.util.IoUtils; -/** - * - * @author kamal - */ public class ShellCommandBuilder { - public static List fileScript2LinebyLineCommands(String filePath, String... pairs) - throws IOException { - String script = IoUtils.readContentFromClasspath(filePath); - if (pairs.length > 0) { - for (int i = 0; i < pairs.length; i += 2) { - String key = pairs[i]; - String val = pairs[i + 1]; - script = script.replaceAll("%" + key + "%", val); - } - } - List cmds = makeLineByLineCommands(script); - return cmds; - } - - public static List makeLineByLineCommands(String script) throws IOException { - List tasks = new ArrayList<>(); - Scanner scanner = new Scanner(script); - while (scanner.hasNextLine()) { - String nextCmd = scanner.nextLine(); - if (nextCmd.contains("cat") && nextCmd.contains("END_OF_FILE")) { - StringBuilder cmdBuf = new StringBuilder(); - cmdBuf.append(nextCmd); - String newLine = null; - do { - newLine = scanner.nextLine(); - cmdBuf.append("\n").append(newLine); - } while (!newLine.contains("END_OF_FILE")); - tasks.add(new ShellCommand(cmdBuf.toString())); - } else { - tasks.add(new ShellCommand(nextCmd)); - } - } - return tasks; - } - public static List makeSingleFileCommand(String filePath, String... pairs) throws IOException { - String script = IoUtils.readContentFromClasspath(filePath); + String script = Resources.toString(Resources.getResource(filePath), Charsets.UTF_8); if (pairs.length > 0) { for (int i = 0; i < pairs.length; i += 2) { String key = pairs[i]; String val = pairs[i + 1]; - script = script.replaceAll("%" + key + "%", val); + if (key == null || val == null) { + continue; + } + script = script.replaceAll("%" + key + "%", Matcher.quoteReplacement(val)); } } List cmds = new ArrayList<>(); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/converter/UserClusterDataExtractor.java b/karamel-core/src/main/java/se/kth/karamel/backend/converter/UserClusterDataExtractor.java index fb8cbb01..492ffa80 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/converter/UserClusterDataExtractor.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/converter/UserClusterDataExtractor.java @@ -1,36 +1,27 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.converter; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.log4j.Logger; import se.kth.karamel.backend.ClusterDefinitionService; import se.kth.karamel.backend.running.model.ClusterRuntime; import se.kth.karamel.backend.running.model.GroupRuntime; import se.kth.karamel.backend.running.model.MachineRuntime; +import se.kth.karamel.common.clusterdef.Cookbook; +import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.clusterdef.Ec2; import se.kth.karamel.common.clusterdef.Provider; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.json.JsonGroup; -import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.cookbookmeta.CookbookCache; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; -import se.kth.karamel.common.cookbookmeta.CookbookUrls; import se.kth.karamel.common.cookbookmeta.MetadataRb; import se.kth.karamel.common.cookbookmeta.Recipe; -/** - * - * @author kamal - */ public class UserClusterDataExtractor { private static final Logger logger = Logger.getLogger(UserClusterDataExtractor.class); @@ -39,45 +30,36 @@ public class UserClusterDataExtractor { public static String clusterLinks(JsonCluster cluster, ClusterRuntime clusterEntity) throws KaramelException { StringBuilder builder = new StringBuilder(); - HashSet cbids = new HashSet<>(); - for (JsonGroup jg : cluster.getGroups()) { - for (JsonCookbook jc : jg.getCookbooks()) { - String cbid = jc.getId(); - cbids.add(cbid); - cookbookCache.prepareParallel(cbids); - } - } for (JsonGroup jg : cluster.getGroups()) { - for (JsonCookbook jc : jg.getCookbooks()) { - for (JsonRecipe rec : jc.getRecipes()) { - String cbid = jc.getId(); - KaramelizedCookbook cb = cookbookCache.get(cbid); - MetadataRb metadataRb = cb.getMetadataRb(); - List recipes = metadataRb.getRecipes(); - for (Recipe recipe : recipes) { - if (recipe.getCanonicalName().equalsIgnoreCase(rec.getCanonicalName())) { - Set links = recipe.getLinks(); - for (String link : links) { - if (link.contains(Settings.METADATA_INCOMMENT_HOST_KEY)) { - if (clusterEntity != null) { - GroupRuntime ge = findGroup(clusterEntity, jg.getName()); - if (ge != null) { - List machines = ge.getMachines(); - if (machines != null) { - for (MachineRuntime me : ge.getMachines()) { - String l = link.replaceAll(Settings.METADATA_INCOMMENT_HOST_KEY, me.getPublicIp()); - builder.append(l).append("\n"); - } + + for (JsonRecipe rec : jg.getRecipes()) { + String cbid = rec.getCookbook().getCookbookName(); + KaramelizedCookbook cb = cookbookCache.get(cbid); + MetadataRb metadataRb = cb.getMetadataRb(); + List recipes = metadataRb.getRecipes(); + for (Recipe recipe : recipes) { + if (recipe.getCanonicalName().equalsIgnoreCase(rec.getCanonicalName())) { + Set links = recipe.getLinks(); + for (String link : links) { + if (link.contains(Settings.METADATA_INCOMMENT_HOST_KEY)) { + if (clusterEntity != null) { + GroupRuntime ge = findGroup(clusterEntity, jg.getName()); + if (ge != null) { + List machines = ge.getMachines(); + if (machines != null) { + for (MachineRuntime me : ge.getMachines()) { + String l = link.replaceAll(Settings.METADATA_INCOMMENT_HOST_KEY, me.getPublicIp()); + builder.append(l).append("\n"); } } } - } else { - builder.append(link).append("\n"); } - + } else { + builder.append(link).append("\n"); } } + } } } @@ -128,12 +110,10 @@ public static Provider getGroupProvider(JsonCluster cluster, String groupName) { return provider; } - public static String makeVendorPath(String sshUser, List rootCookbooks) throws KaramelException { + public static String makeVendorPath(String sshUser, Map rootCookbooks) throws KaramelException { Set paths = new HashSet<>(); - for (KaramelizedCookbook kcb : rootCookbooks) { - CookbookUrls urls = kcb.getUrls(); - String cookbookPath = urls.repoName; - paths.add(Settings.REMOTE_COOKBOOK_VENDOR_PATH(sshUser, cookbookPath)); + for (Map.Entry cookbook : rootCookbooks.entrySet()) { + paths.add(Settings.REMOTE_COOKBOOK_VENDOR_PATH(sshUser, cookbook.getKey())); } Object[] arr = paths.toArray(); StringBuilder buffer = new StringBuilder(); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/dag/Dag.java b/karamel-core/src/main/java/se/kth/karamel/backend/dag/Dag.java index 0ab38e33..c1497554 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/dag/Dag.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/dag/Dag.java @@ -12,7 +12,10 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + import org.apache.log4j.Logger; +import se.kth.karamel.common.clusterdef.json.JsonCluster; import se.kth.karamel.common.exception.DagConstructionException; /** @@ -24,6 +27,21 @@ public class Dag { private static final Logger logger = Logger.getLogger(Dag.class); private final Map allNodes = new HashMap<>(); + private final Map serializableRecipes = new ConcurrentHashMap<>(); + + public void addSerializableRecipe(String id, Integer parallelism) { + String safeId = id.trim(); + if (serializableRecipes.containsKey(safeId)) { + return; + } + logger.info("Adding serializable recipe " + safeId + " with parallelism " + parallelism); + serializableRecipes.put(safeId, new RecipeSerialization(parallelism)); + } + + public RecipeSerialization getSerializableRecipeCounter(String id) { + return serializableRecipes.get(id.trim()); + } + public void addNode(String nodeId) { if (!allNodes.containsKey(nodeId)) { allNodes.put(nodeId, new DagNode(nodeId)); @@ -42,7 +60,7 @@ public void addTask(DagTask task) throws DagConstructionException { logger.debug("Adding task: " + task.dagNodeId()); DagNode node = null; if (!allNodes.containsKey(task.dagNodeId())) { - node = new DagNode(task.dagNodeId(), task); + node = new DagNode(task.dagNodeId(), task, this); allNodes.put(task.dagNodeId(), node); } else { node = allNodes.get(task.dagNodeId()); @@ -62,7 +80,7 @@ public boolean addDependency(String first, String next) throws DagConstructionEx } if (first.equals(next)) { - throw new DagConstructionException(String.format("Cyrcular dependency is not allowed: %s -> %s", first, next)); + throw new DagConstructionException(String.format("Circular dependency is not allowed: %s -> %s", first, next)); } logger.debug("Adding dependency: " + first + " -> " + next); @@ -94,12 +112,12 @@ public void updateLabel(String nodeId, String label) throws DagConstructionExcep } } - public void start() throws DagConstructionException { + public void start(JsonCluster clusterDefinition) throws DagConstructionException { validate(); logger.debug("Dag is starting: \n" + print()); String prob = UUID.randomUUID().toString(); for (DagNode node : findRootNodes()) { - node.prepareToStart(prob); + node.prepareToStart(prob, clusterDefinition); } for (DagNode node : findRootNodes()) { node.start(); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagNode.java b/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagNode.java index 279fc6e5..11d07ec0 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagNode.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagNode.java @@ -9,7 +9,12 @@ import java.util.HashSet; import java.util.List; import java.util.Set; + import org.apache.log4j.Logger; +import se.kth.karamel.backend.running.model.tasks.RunRecipeTask; +import se.kth.karamel.backend.running.model.tasks.Task; +import se.kth.karamel.common.clusterdef.json.JsonCluster; +import se.kth.karamel.common.clusterdef.yaml.RuntimeConfiguration; import se.kth.karamel.common.exception.DagConstructionException; import se.kth.karamel.common.exception.KaramelException; @@ -35,20 +40,26 @@ public static enum Status { private Status status = Status.WAITING; private int indention = 1; private String label; + private final Dag dag; public DagNode(String id) { - this.id = id; + this(id, null, null); } - public DagNode(String id, DagTask task) { + public DagNode(String id, DagTask task, Dag dag) { this.id = id; this.task = task; + this.dag = dag; } public String getId() { return id; } + public Dag getDag() { + return dag; + } + public Status getStatus() { return status; } @@ -97,15 +108,26 @@ public void removePredecessor(DagNode predecessor) { predecessors.remove(predecessor); } - public void prepareToStart(String prob) throws DagConstructionException { + public void prepareToStart(String prob, JsonCluster clusterDefinition) throws DagConstructionException { if (probs.contains(prob)) { return; } probs.add(prob); task.prepareToStart(); + if (task instanceof RunRecipeTask) { + String recipeCanonicalName = ((RunRecipeTask) task).getRecipeCanonicalName(); + if (clusterDefinition != null) { + RuntimeConfiguration runtimeConfiguration = clusterDefinition.getRuntimeConfiguration(); + // Recipe canonical name is for example "ndb::ndbd" + Integer recipeParallelism = runtimeConfiguration.getRecipesParallelism().get(recipeCanonicalName); + if (recipeParallelism != null && recipeParallelism > 0) { + dag.addSerializableRecipe(recipeCanonicalName, recipeParallelism); + } + } + } for (DagNode succ : successors) { - succ.prepareToStart(prob); + succ.prepareToStart(prob, clusterDefinition); } } @@ -201,6 +223,7 @@ public void start() { public void succeed() { logger.debug(String.format("Done '%s'", id)); status = Status.DONE; + releaseSerializedTask(); signalChildren(); } @@ -208,6 +231,7 @@ public void succeed() { public void skipped() { logger.debug(String.format("Skip '%s'", id)); status = Status.SKIPPED; + releaseSerializedTask(); signalChildren(); } @@ -216,10 +240,32 @@ public void terminate() { for (DagNode succ : successors) { succ.terminate(); } + releaseSerializedTask(); task.terminate(); } } + private void releaseSerializedTask() { + if (task instanceof RunRecipeTask) { + RunRecipeTask recipeTask = (RunRecipeTask) task; + RecipeSerialization serialization = dag.getSerializableRecipeCounter(recipeTask.getRecipeCanonicalName()); + if (serialization != null) { + synchronized (serialization) { + serialization.setFailedStatus(recipeTask.getStatus().equals(Task.Status.FAILED)); + logger.info(String.format("%s: Recipe %s RecipeSerializationID: %s Has recipe failed: %s", + ((RunRecipeTask) task).getMachine().getId(), + ((RunRecipeTask) task).getName(), + serialization.hashCode(), + serialization.hasFailed())); + logger.debug(String.format("%s: Recipe %s NotifyingAll", ((RunRecipeTask) task).getMachine().getId(), + ((RunRecipeTask) task).getName())); + serialization.notifyAll(); + } + serialization.release(recipeTask); + } + } + } + @Override public void terminated() { status = Status.TERMINATED; @@ -252,6 +298,7 @@ public synchronized void signal(DagNode pred) throws KaramelException { @Override public void failed(String reason) { logger.error(String.format("Failed '%s' because '%s', DAG is stuck here :(", id, reason)); + releaseSerializedTask(); status = Status.FAILED; } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagTaskCallback.java b/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagTaskCallback.java index ac085b8e..fae84317 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagTaskCallback.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/dag/DagTaskCallback.java @@ -25,4 +25,6 @@ public interface DagTaskCallback { public void terminated(); public void skipped(); + + Dag getDag(); } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerialization.java b/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerialization.java new file mode 100644 index 00000000..c1e36ac1 --- /dev/null +++ b/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerialization.java @@ -0,0 +1,88 @@ +package se.kth.karamel.backend.dag; + +import org.apache.log4j.Logger; +import se.kth.karamel.backend.running.model.tasks.RunRecipeTask; + +import java.util.Comparator; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.Semaphore; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class RecipeSerialization { + private static final Logger logger = Logger.getLogger(RecipeSerialization.class); + private final Semaphore parallelism; + private final Integer maxParallelism; + private final Set claims; + private final ReentrantReadWriteLock claimsLock; + private boolean failed = false; + + public RecipeSerialization(Integer parallelism) { + this.parallelism = new Semaphore(parallelism, true); + this.maxParallelism = parallelism; + claims = new TreeSet<>(new Comparator() { + @Override + public int compare(RecipeSerializationClaim t0, RecipeSerializationClaim t1) { + return t0.getClaimedAt().compareTo(t1.getClaimedAt()); + } + }); + claimsLock = new ReentrantReadWriteLock(true); + } + + public void release(RunRecipeTask task) { + parallelism.release(); + claimsLock.writeLock().lock(); + try { + claims.remove(new RecipeSerializationClaim(task)); + } finally { + claimsLock.writeLock().unlock(); + } + logger.info("Released serializable execution of " + task.getRecipeCanonicalName() + " on " + + task.getMachineId()); + } + + public void prepareToExecute(RunRecipeTask task) throws InterruptedException { + logger.info("Prepare to run " + task.getRecipeCanonicalName() + " on " + task.getMachineId()); + if (!parallelism.tryAcquire()) { + logger.info("Could not run " + task.getRecipeCanonicalName() + " on " + task.getMachineId() + + " at the moment because parallelism is limited. Available parallelism permits: " + + parallelism.availablePermits() + "/" + maxParallelism + " - we wait until a permit becomes available." + + " Current claims: " + printableClaims()); + task.blocked(); + parallelism.acquire(); + } + claimsLock.writeLock().lock(); + try { + claims.add(new RecipeSerializationClaim(task)); + } finally { + claimsLock.writeLock().unlock(); + } + logger.info("Proceed with running " + task.getRecipeCanonicalName() + " on " + task.getMachineId()); + } + + private String printableClaims() { + claimsLock.readLock().lock(); + try { + StringBuffer sb = new StringBuffer(); + Iterator i = claims.iterator(); + int o = 0; + while (i.hasNext()) { + sb.append(String.format("[%d] ", o)); + sb.append(i.next().getId()).append(" "); + o++; + } + return sb.toString(); + } finally { + claimsLock.readLock().unlock(); + } + } + + public synchronized void setFailedStatus(boolean failed) { + this.failed = failed; + } + + public synchronized boolean hasFailed() { + return failed; + } +} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerializationClaim.java b/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerializationClaim.java new file mode 100644 index 00000000..05d653f3 --- /dev/null +++ b/karamel-core/src/main/java/se/kth/karamel/backend/dag/RecipeSerializationClaim.java @@ -0,0 +1,41 @@ +package se.kth.karamel.backend.dag; + +import se.kth.karamel.backend.running.model.tasks.RunRecipeTask; + +import java.time.Instant; +import java.util.Objects; + +public class RecipeSerializationClaim { + private final String id; + private final Instant claimedAt; + + public static String serializationClaimId(RunRecipeTask task) { + return String.format("%s@%s", task.getRecipeCanonicalName(), task.getMachineId()); + } + + public RecipeSerializationClaim(RunRecipeTask task) { + id = serializationClaimId(task); + claimedAt = Instant.now(); + } + + public String getId() { + return id; + } + + public Instant getClaimedAt() { + return claimedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RecipeSerializationClaim that = (RecipeSerializationClaim) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubApi.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubApi.java deleted file mode 100644 index 509dd396..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubApi.java +++ /dev/null @@ -1,467 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.backend.github; - -import org.apache.commons.io.FileUtils; -import org.eclipse.egit.github.core.Repository; -import org.eclipse.egit.github.core.SearchRepository; -import org.eclipse.egit.github.core.User; -import org.eclipse.egit.github.core.client.GitHubClient; -import org.eclipse.egit.github.core.service.OrganizationService; -import org.eclipse.egit.github.core.service.RepositoryService; -import org.eclipse.egit.github.core.service.UserService; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; -import org.kohsuke.github.GHOrganization; -import org.kohsuke.github.GHRepository; -import org.kohsuke.github.GitHub; -import se.kth.karamel.common.CookbookScaffolder; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.common.util.Confs; -import se.kth.karamel.common.util.Settings; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * 1. Call registerCredentials() to store your github credentials in memory. 2. Then call methods like addFile(), - * commitPush(repo,..) - * - */ -public class GithubApi { - - private static volatile String user = ""; - private static volatile String email = ""; - private static volatile String password = ""; - - private static final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(GithubApi.class); - - private static final GitHubClient client = GitHubClient.createClient("http://github.com"); - - private static final Map> cachedOrgs = new HashMap<>(); - private static final Map> cachedRepos = new HashMap<>(); - - // Singleton - private GithubApi() { - } - - /** - * Blindly accepts user credentials, no validation with github. - * - * @param user - * @param password - * @return primary github email for the user - * @throws se.kth.karamel.common.exception.KaramelException - */ - public synchronized static GithubUser registerCredentials(String user, String password) throws KaramelException { - try { - GithubApi.user = user; - GithubApi.password = password; - client.setCredentials(user, password); - client.getUser(); - Confs confs = Confs.loadKaramelConfs(); - confs.put(Settings.GITHUB_USER_KEY, user); - confs.put(Settings.GITHUB_PASSWORD_KEY, password); - confs.writeKaramelConfs(); - UserService us = new UserService(client); - if (us == null) { - throw new KaramelException("Could not find user or password incorret: " + user); - } - User u = us.getUser(); - if (u == null) { - throw new KaramelException("Could not find user or password incorret: " + user); - } - GithubApi.email = u.getEmail(); - } catch (IOException ex) { - logger.warn("Problem connecting to GitHub: " + ex.getMessage()); - } - return new GithubUser(GithubApi.user, GithubApi.password, GithubApi.email); - } - - /** - * - * @return email or null if not set yet. - */ - public static String getEmail() { - return GithubApi.email; - } - - public static GithubUser loadGithubCredentials() throws KaramelException { - Confs confs = Confs.loadKaramelConfs(); - GithubApi.user = confs.getProperty(Settings.GITHUB_USER_KEY); - GithubApi.password = confs.getProperty(Settings.GITHUB_PASSWORD_KEY); - if (GithubApi.user != null && GithubApi.password != null) { - registerCredentials(GithubApi.user, GithubApi.password); - } - return new GithubUser(GithubApi.user, GithubApi.password, GithubApi.email); - } - - public synchronized static String getUser() { - return GithubApi.user; - } - - public synchronized static String getPassword() { - return GithubApi.password; - } - - public synchronized static int getRemainingRequests() { - return client.getRemainingRequests(); - } - - public synchronized static int getRequestLimit() { - return client.getRequestLimit(); - } - - /** - * - * @return List of github orgs for authenticated user - * @throws KaramelException - */ - public synchronized static List getOrganizations() throws KaramelException { - if (cachedOrgs.get(GithubApi.getUser()) != null) { - return cachedOrgs.get(GithubApi.getUser()); - } - try { - List orgs = new ArrayList<>(); - OrganizationService os = new OrganizationService(client); - List longOrgsList = os.getOrganizations(); - List orgsList = new ArrayList<>(); - for (User u : longOrgsList) { - orgsList.add(new OrgItem(u.getLogin(), u.getAvatarUrl())); - } - cachedOrgs.put(GithubApi.getUser(), orgsList); - - return orgsList; - } catch (IOException ex) { - throw new KaramelException("Problem listing GitHub organizations: " + ex.getMessage()); - } - } - - /** - * Gets all repositories for a given organization/user. - * - * @param orgName - * @return List of repositories - * @throws KaramelException - */ - public synchronized static List getRepos(String orgName) throws KaramelException { - if (cachedRepos.get(orgName) != null) { - return cachedRepos.get(orgName); - } - - try { - RepositoryService rs = new RepositoryService(client); - List repos; - // If we are looking for the repositories for the current user - if (GithubApi.getUser().equalsIgnoreCase(orgName)) { - repos = rs.getRepositories(orgName); - } else { // If we are looking for the repositories for a given organization - repos = rs.getOrgRepositories(orgName); - } - - List repoItems = new ArrayList<>(); - for (Repository r : repos) { - repoItems.add(new RepoItem(r.getName(), r.getDescription(), r.getSshUrl())); - } - cachedRepos.put(orgName, repoItems); - return repoItems; - } catch (IOException ex) { - throw new KaramelException("Problem listing GitHub repositories: " + ex.getMessage()); - } - } - - public synchronized static boolean repoExists(String owner, String repoName) throws KaramelException { - List repos = GithubApi.getRepos(owner); - if (repos == null) { - return false; - } - boolean found = false; - for (RepoItem r : repos) { - if (r.getName().compareToIgnoreCase(repoName) == 0) { - found = true; - break; - } - } - return found; - } - - /** - * Gets local directory for a given repository name. - * - * @param repoName - * @return File representing the local directory - */ - public static File getRepoDirectory(String repoName) { - File targetDir = new File(Settings.COOKBOOKS_PATH); - if (targetDir.exists() == false) { - targetDir.mkdirs(); - } - return new File(Settings.COOKBOOKS_PATH + File.separator + repoName); - } - - /** - * Create a repository for a given organization with a description - * - * @param org - * @param repoName - * @param description - * @return RepoItem bean/json object - * @throws KaramelException - */ - public synchronized static RepoItem createRepoForOrg(String org, String repoName, String description) throws - KaramelException { - try { - OrganizationService os = new OrganizationService(client); - RepositoryService rs = new RepositoryService(client); - Repository r = new Repository(); - r.setName(repoName); - r.setOwner(os.getOrganization(org)); - r.setDescription(description); - rs.createRepository(org, r); - cloneRepo(org, repoName); - cachedRepos.remove(org); - return new RepoItem(repoName, description, r.getSshUrl()); - } catch (IOException ex) { - throw new KaramelException("Problem creating the repository " + repoName + " for organization " + org - + " : " + ex.getMessage()); - } - } - - /** - * Create a repository in a given github user's local account. - * - * @param repoName - * @param description - * @throws KaramelException - */ - public synchronized static void createRepoForUser(String repoName, String description) throws KaramelException { - try { - UserService us = new UserService(client); - RepositoryService rs = new RepositoryService(client); - Repository r = new Repository(); - r.setName(repoName); - r.setOwner(us.getUser()); - r.setDescription(description); - rs.createRepository(r); - cloneRepo(getUser(), repoName); - cachedRepos.remove(GithubApi.getUser()); - } catch (IOException ex) { - throw new KaramelException("Problem creating " + repoName + " for user " + ex.getMessage()); - } - } - - /** - * Clone an existing github repo. - * - * @param owner - * @param repoName - * @throws se.kth.karamel.common.exception.KaramelException - */ - public synchronized static void cloneRepo(String owner, String repoName) throws KaramelException { - Git result = null; - try { - RepositoryService rs = new RepositoryService(client); - Repository r = rs.getRepository(owner, repoName); - - String cloneURL = r.getSshUrl(); - // prepare a new folder for the cloned repository - File localPath = new File(Settings.COOKBOOKS_PATH + File.separator + repoName); - if (localPath.isDirectory() == false) { - localPath.mkdirs(); - } else { - throw new KaramelException("Local directory already exists. Delete it first: " + localPath); - } - - logger.debug("Cloning from " + cloneURL + " to " + localPath); - result = Git.cloneRepository() - .setURI(cloneURL) - .setDirectory(localPath) - .call(); - // Note: the call() returns an opened repository already which needs to be closed to avoid file handle leaks! - logger.debug("Cloned repository: " + result.getRepository().getDirectory()); - } catch (IOException | GitAPIException ex) { - throw new KaramelException("Problem cloning repo: " + ex.getMessage()); - } finally { - if (result != null) { - result.close(); - } - } - - } - - public synchronized static void removeRepo(String owner, String repoName) throws KaramelException { - - try { - GitHub gitHub = GitHub.connectUsingPassword(GithubApi.getUser(), GithubApi.getPassword()); - if (!gitHub.isCredentialValid()) { - throw new KaramelException("Invalid GitHub credentials"); - } - GHRepository repo = null; - if (owner.compareToIgnoreCase(GithubApi.getUser()) != 0) { - GHOrganization org = gitHub.getOrganization(owner); - repo = org.getRepository(repoName); - } else { - repo = gitHub.getRepository(owner + "/" + repoName); - } - repo.delete(); - - } catch (IOException ex) { - throw new KaramelException("Problem authenticating with gihub-api when trying to remove a repository"); - } - } - - public synchronized static void removeLocalRepo(String owner, String repoName) throws KaramelException { - File path = getRepoDirectory(repoName); - try { - FileUtils.deleteDirectory(path); - } catch (IOException ex) { - throw new KaramelException("Couldn't find the path to delete for Repo: " + repoName + " with owner: " + owner); - } - } - - /** - * Adds a file to the Github repo's index. If the file already exists, it will delete it and replace its contents with - * the new contents. You wil subsequenty need to commit the change and push the commit to github. - * - * @param owner - * @param repoName - * @param fileName - * @param contents - * @throws KaramelException - */ - public synchronized static void addFile(String owner, String repoName, String fileName, String contents) - throws KaramelException { - File repoDir = getRepoDirectory(repoName); - Git git = null; - try { - git = Git.open(repoDir); - - new File(repoDir + File.separator + fileName).delete(); - new File(repoDir + File.separator + fileName).getParentFile().mkdirs(); - try (PrintWriter out = new PrintWriter(repoDir + File.separator + fileName)) { - out.println(contents); - } - git.add().addFilepattern(fileName).call(); - } catch (IOException | GitAPIException ex) { - throw new KaramelException(ex.getMessage()); - } finally { - if (git != null) { - git.close(); - } - - } - } - - public synchronized static void removeFile(String owner, String repoName, String fileName) - throws KaramelException { - File repoDir = getRepoDirectory(repoName); - Git git = null; - try { - git = Git.open(repoDir); - new File(repoDir + File.separator + fileName).delete(); - git.add().addFilepattern(fileName).call(); - git.commit().setAuthor(user, email).setMessage("File removed by Karamel.") - .setAll(true).call(); - git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider(user, password)).call(); - RepoItem toRemove = null; - List repos = cachedRepos.get(owner); - for (RepoItem r : repos) { - if (r.getName().compareToIgnoreCase(repoName) == 0) { - toRemove = r; - } - } - if (toRemove != null) { - repos.remove(toRemove); - } - } catch (IOException | GitAPIException ex) { - throw new KaramelException(ex.getMessage()); - } finally { - if (git != null) { - git.close(); - } - } - } - - /** - * Scaffolds a Karamel/chef project for an experiment and adds it to the github repo. You still need to commit and - * push the changes to github. - * - * @param repoName - * @throws KaramelException - */ - public static void scaffoldRepo(String repoName) throws KaramelException { - File repoDir = getRepoDirectory(repoName); - - Git git = null; - try { - git = Git.open(repoDir); - - CookbookScaffolder.create(repoName); - - git.add().addFilepattern("Berksfile").addFilepattern("metadata.rb") - .addFilepattern("Karamelfile") - .addFilepattern(".kitchen.yml").addFilepattern("attributes").addFilepattern("recipes") - .addFilepattern("templates").addFilepattern("README.md").call(); - - } catch (IOException | GitAPIException ex) { - throw new KaramelException("Problem scaffolding a new Repository: " + ex.getMessage()); - } finally { - if (git != null) { - git.close(); - } - } - } - - /** - * Synchronizes your updates on your local repository with github. - * - * @param owner - * @param repoName - * @throws KaramelException - */ - public synchronized static void commitPush(String owner, String repoName) - throws KaramelException { - if (email == null || user == null) { - throw new KaramelException("You forgot to call registerCredentials. You must call this method first."); - } - File repoDir = getRepoDirectory(repoName); - Git git = null; - try { - git = Git.open(repoDir); - - git.commit().setAuthor(user, email).setMessage("Code generated by Karamel.") - .setAll(true).call(); - git.push().setCredentialsProvider(new UsernamePasswordCredentialsProvider(user, password)).call(); - } catch (IOException | GitAPIException ex) { - logger.error("error during github push", ex); - throw new KaramelException(ex.getMessage()); - } finally { - if (git != null) { - git.close(); - } - - } - } - - /** - * Search github for organizations/repositories/users using the GitHub API. - * - * @param query - * @throws IOException - */ - public synchronized static void searchRepos(String query) throws IOException { - RepositoryService rs = new RepositoryService(client); - - List listRepos = rs.searchRepositories(query); - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubUser.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubUser.java deleted file mode 100644 index 6b76e786..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/GithubUser.java +++ /dev/null @@ -1,44 +0,0 @@ -package se.kth.karamel.backend.github; - -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement -public class GithubUser { - private String user; - private String password; - private String email; - - public GithubUser(String user, String password, String email) { - this.user = (user == null) ? "" : user; - this.password = (password == null) ? "" : password; - this.email = (email == null) ? "" : email; - } - - public GithubUser() { - } - - public void setEmail(String email) { - this.email = email; - } - - public String getEmail() { - return email; - } - - public String getUser() { - return user; - } - - public String getPassword() { - return password; - } - - public void setUser(String email) { - this.user = email; - } - - public void setPassword(String password) { - this.password = password; - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/OrgItem.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/OrgItem.java deleted file mode 100644 index 271462e1..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/OrgItem.java +++ /dev/null @@ -1,34 +0,0 @@ -package se.kth.karamel.backend.github; - -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement -public class OrgItem { - private String name; - private String gravitar; - - public OrgItem(String name, String gravitar) { - this.name = name; - this.gravitar = gravitar; - } - - public OrgItem() { - } - - public String getGravitar() { - return gravitar; - } - - public String getName() { - return name; - } - - public void setGravitar(String gravitar) { - this.gravitar = gravitar; - } - - public void setName(String name) { - this.name = name; - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/RepoItem.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/RepoItem.java deleted file mode 100644 index 76a798f4..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/RepoItem.java +++ /dev/null @@ -1,46 +0,0 @@ -package se.kth.karamel.backend.github; - -import javax.xml.bind.annotation.XmlRootElement; - -@XmlRootElement -public class RepoItem { - - private String name; - private String description; - private String sshUrl; - - public RepoItem(String name, String description, String sshUrl) { - this.name = name; - this.description = description; - this.sshUrl = sshUrl; - } - - public RepoItem() { - } - - - public String getDescription() { - return description; - } - - public String getName() { - return name; - } - - public String getSshUrl() { - return sshUrl; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setName(String name) { - this.name = name; - } - - public void setSshUrl(String sshUrl) { - this.sshUrl = sshUrl; - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/ChefExperimentExtractor.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/util/ChefExperimentExtractor.java deleted file mode 100644 index 07e4a9b3..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/ChefExperimentExtractor.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.backend.github.util; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import se.kth.karamel.backend.Experiment; -import se.kth.karamel.backend.Experiment.Code; -import se.kth.karamel.backend.github.GithubApi; -import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.exception.KaramelException; - -/** - * How to use. Invoke methods in this order: (1) @see ChefExperimentExtractor#parseAttributesAddToGit() (2) @see - * ChefExperimentExtractor#parseRecipesAddToGit() - * - */ -public class ChefExperimentExtractor { - - private static final String YAML_DEPENDENCY_PREFIX = " - "; - private static final String YAML_RECIPE_PREFIX = " - recipe: "; - - // pair added to attributes/default.rb - private static final SortedMap attrs = new TreeMap<>(); - private static final Map> configFiles = new HashMap<>(); - - /** - * Parses all scripts and config files and outputs to metadata.rb and attributes/default.rb the configuration values - * found. - * - * @param owner org/user on github - * @param repoName name of github repository - * @param experiment input scripts/config filenames and content - * @throws KaramelException - */ - public static void parseAttributesAddToGit(String owner, String repoName, Experiment experiment) - throws KaramelException { - - attrs.clear(); - configFiles.clear(); - - StringBuilder recipeDescriptions = new StringBuilder(); - List experiments = experiment.getCode(); - - // Extract all the configFileNames: write them to metadata.rb later - // Extract all the from the configFile contents: write them to attributes/default.rb later - // No conflict detection for duplicate key-value pairs yet. Should be done in Javascript in Browser. - for (Code code : experiments) { - String configFileName = code.getConfigFileName(); - Map cfs = configFiles.get(configFileName); - if (cfs == null) { - cfs = new HashMap<>(); - configFiles.put(configFileName, cfs); - } - recipeDescriptions.append("recipe \"").append(repoName).append(Settings.COOKBOOK_DELIMITER). - append(code.getName()).append("\", \"configFile=").append(configFileName) - .append("; Experiment name: ").append(code.getName()).append("\"").append(System.lineSeparator()); - String str = code.getConfigFileContents(); - Pattern p = Pattern.compile("\\s*(.*)\\s*=\\s*(.*)\\s*"); - Matcher m = p.matcher(str); - while (m.find()) { - String name = m.group(1); - String value = m.group(2); - if (!name.isEmpty()) { - cfs.put(name, value); -// attrs.put(name, value); - } - } - } - - String email = (GithubApi.getEmail() == null) ? "karamel@karamel.io" : GithubApi.getEmail(); - try { - StringBuilder defaults_rb = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_ATTRIBUTES_DEFAULT - // , "name", repoName, - // "user", experiment.getUser(), - // "group", experiment.getGroup(), - // "http_binaries", experiment.getUrlBinary() - ); - - String str = experiment.getDefaultAttributes(); - Pattern p = Pattern.compile("\\s*(.*)\\s*=\\s*(.*)\\s*"); - Matcher m = p.matcher(str); - while (m.find()) { - String name = m.group(1); - String value = m.group(2); - if (!name.isEmpty()) { - attrs.put(name, value); - } - } - - // Add all key-value pairs from the config files to the default attributes - for (String key : attrs.keySet()) { - String entry = "default[:" + repoName + "][:" + key + "] = \"" + attrs.get(key) + "\""; - defaults_rb.append(entry).append(System.lineSeparator()); - } - - StringBuilder metadata_rb = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_METADATA, - "name", repoName, - "user", experiment.getUser(), - "email", email, - "depends", "", - "resolve_ips", "", - "build_command", experiment.getMavenCommand(), - "url_binary", experiment.getUrlBinary(), - "url_gitclone", experiment.getUrlGitClone(), - "build_command", experiment.getMavenCommand(), - "ip_params", "", - "more_recipes", recipeDescriptions.toString() - ); - - for (String key : attrs.keySet()) { - String entry = "attribute \"" + repoName + "/" + key + "\"," + System.lineSeparator() - + ":description => \"" + key + " parameter value\"," + System.lineSeparator() - + ":type => \"string\""; - metadata_rb.append(entry).append(System.lineSeparator()).append(System.lineSeparator()); - } - - // 3. write them to files and push to github - GithubApi.addFile(owner, repoName, "attributes/default.rb", defaults_rb.toString()); - GithubApi.addFile(owner, repoName, "metadata.rb", metadata_rb.toString()); - - } catch (IOException ex) { - throw new KaramelException("Problem parsing attributes from GitHub: " + ex.getMessage()); - } - } - - /** - * Parses the user-defined script files and for each script, a recipe file is generated and added to the git repo. - * - * @param owner - * @param repoName - * @param experimentContext - * @throws se.kth.karamel.common.exception.KaramelException - * @throws KaramelExceptionntents.toString()); // Update Karamel - */ - public static void parseRecipesAddToGit(String owner, String repoName, Experiment experimentContext) - throws KaramelException { - - try { - - StringBuilder kitchenContents = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_KITCHEN_YML, - "name", repoName - ); - GithubApi.addFile(owner, repoName, ".kitchen.yml", kitchenContents.toString()); - - List experiments = experimentContext.getCode(); - - String localDependencies = repoName + Settings.COOKBOOK_DELIMITER + "install" + System.lineSeparator() - + experimentContext.getLocalDependencies(); - String[] lDeps = localDependencies.split(System.lineSeparator()); - - Set lDepsRemoveDuplicates = new HashSet<>(); - StringBuilder lDepsFinal = new StringBuilder(); - for (String s : lDeps) { - s = s.trim(); - if (!s.isEmpty()) { - lDepsRemoveDuplicates.add(s); - } - } - for (String s : lDepsRemoveDuplicates) { - lDepsFinal.append(YAML_DEPENDENCY_PREFIX).append(s).append(System.lineSeparator()); - } - - String globalDependencies = experimentContext.getGlobalDependencies(); - String[] gDeps = globalDependencies.split(System.lineSeparator()); - Set gDepsRemoveDuplicates = new HashSet<>(); - StringBuilder gDepsFinal = new StringBuilder(); - for (String s : gDeps) { - s = s.trim(); - if (!s.isEmpty()) { - gDepsRemoveDuplicates.add(s); - } - } - for (String s : gDepsRemoveDuplicates) { - gDepsFinal.append(YAML_DEPENDENCY_PREFIX).append(s).append(System.lineSeparator()); - } - - StringBuilder recipeDepsKaramelfile = new StringBuilder(); - for (Code experiment : experiments) { - String recipeName = experiment.getName(); - recipeDepsKaramelfile.append(YAML_RECIPE_PREFIX).append(repoName).append(Settings.COOKBOOK_DELIMITER).append( - recipeName) - .append(System.lineSeparator()); - recipeDepsKaramelfile.append(" local:").append(System.lineSeparator()); - recipeDepsKaramelfile.append(lDepsFinal.toString()); - recipeDepsKaramelfile.append(" global:").append(System.lineSeparator()); - recipeDepsKaramelfile.append(gDepsFinal.toString()); - } - - StringBuilder karamelContents = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_KARAMELFILE, - "name", repoName, - "next_recipes", recipeDepsKaramelfile.toString() - ); - // TODO - integration with cluster defn file -// String ymlString = experimentContext.getClusterDefinition(); - - String berksfile = experimentContext.getBerksfile(); - - StringBuilder berksContents = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_BERKSFILE, - "berks_dependencies", berksfile - ); - - GithubApi.addFile(owner, repoName, "Berksfile", berksContents.toString()); - - Map expConfigFileNames = new HashMap<>(); - Map expConfigFilePaths = new HashMap<>(); - - // 2. write them to recipes/default.rb and metadata.rb - for (Code experiment : experiments) { - String experimentName = experiment.getName(); - String configFilePath = experiment.getConfigFileName(); - String configFileContents = experiment.getConfigFileContents(); - - String configFileName = configFilePath; - int filePos = configFileName.lastIndexOf("/"); - if (filePos != -1) { - configFileName = configFileName.substring(filePos + 1); - } - - String email = (GithubApi.getEmail() == null) ? "karamel@karamel.io" : GithubApi.getEmail(); - - String username = attrs.containsKey("user") ? attrs.get("user") : experimentContext.getUser(); - String groupname = attrs.containsKey("group") ? attrs.get("group") : experimentContext.getUser(); - - StringBuilder recipe_rb = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_RECIPE_EXPERIMENT, - "cookbook", repoName, - "name", experimentName, - "interpreter", experiment.getScriptType(), - "user", username, - "group", groupname, - "script_contents", experiment.getScriptContents() - ); - - String recipeContents = recipe_rb.toString(); - - // Replace all parameters with chef attribute values - for (String attr : attrs.keySet()) { - recipeContents = recipeContents.replaceAll("%%" + attr + "%%", "#{node[:" + repoName + "][:" + attr + "]}"); - } - - for (String attr : attrs.keySet()) { - configFileContents = configFileContents.replaceAll("%%" + attr + "%%", - "<%= node[:" + repoName + "][:" + attr + "] =>"); - } - - if (!configFilePath.isEmpty()) { - expConfigFileNames.put(experimentName, configFileName); - expConfigFilePaths.put(experimentName, configFilePath); - } - - // 3. write them to files and push to github - GithubApi.addFile(owner, repoName, "recipes" + File.separator + experimentName + ".rb", recipeContents); - if (!configFileName.isEmpty()) { - GithubApi.addFile(owner, repoName, - "templates" + File.separator + "defaults" + File.separator + configFileName + ".erb", configFileContents); - } - - } - - StringBuilder configFilesTemplateDefns = new StringBuilder(); - for (String expName : expConfigFileNames.keySet()) { - String configFilePath = expConfigFileNames.get(expName); - String configFileName = expConfigFileNames.get(expName); - StringBuilder configProps = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_CONFIG_PROPS, - "name", expName, - "configFileName", configFileName, - "configFilePath", configFilePath, - "ip_params", "" - ); - configFilesTemplateDefns.append(configProps).append(System.lineSeparator()); - } - - StringBuilder install_rb = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_RECIPE_INSTALL, - "name", repoName, - "cookbook", repoName, - "checksum", "", - "resolve_ips", "", - "setup_code", experimentContext.getExperimentSetupCode(), - "config_files", configFilesTemplateDefns.toString() - ); - - GithubApi.addFile(owner, repoName, "recipes/install.rb", install_rb.toString()); - GithubApi.addFile(owner, repoName, "Karamelfile", karamelContents.toString()); - - } catch (IOException ex) { - throw new KaramelException("Problem parsing recipes from GitHub: " + ex.getMessage()); - } - - } -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/CookbookGenerator.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/util/CookbookGenerator.java deleted file mode 100644 index 106e29ad..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/CookbookGenerator.java +++ /dev/null @@ -1,43 +0,0 @@ -package se.kth.karamel.backend.github.util; - -import java.io.IOException; -import se.kth.karamel.common.util.IoUtils; - -public class CookbookGenerator { - - public static StringBuilder instantiateFromTemplate(String filePath, String... pairs) throws IOException { - StringBuilder sb = new StringBuilder(); - String script = IoUtils.readContentFromClasspath(filePath); - if (pairs.length > 0) { - for (int i = 0; i < pairs.length; i += 2) { - String key = pairs[i]; - String val = pairs[i + 1]; - script = script.replaceAll("%%" + key + "%%", val); - } - } - return sb.append(script); - } - - public static StringBuilder metadataAttribute(StringBuilder sb, String cbName, String desc, String type) { - return metadataAttribute(sb, cbName, desc, type, null); - } - - public static StringBuilder metadataAttribute(StringBuilder sb, String cbName, String desc, String type, - String defaultValue) { - desc = (desc == null) ? "" : desc; - sb.append(cbName).append(System.lineSeparator()); - sb.append(desc).append(System.lineSeparator()); - sb.append(type).append(System.lineSeparator()); - if (defaultValue != null) { - sb.append(defaultValue).append(System.lineSeparator()); - } - return sb; - } - - public static StringBuilder defaultAttribute(StringBuilder sb, String cbName, String attr, String type, - String value) { - - sb.append("default[:").append(cbName).append("][:").append(attr).append("] = ").append(value); - return sb; - } -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/GithubUrl.java b/karamel-core/src/main/java/se/kth/karamel/backend/github/util/GithubUrl.java deleted file mode 100644 index 502c9530..00000000 --- a/karamel-core/src/main/java/se/kth/karamel/backend/github/util/GithubUrl.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.backend.github.util; - -import se.kth.karamel.common.exception.KaramelException; - -public class GithubUrl { - - public static String getProtocol(String githubUrl) throws KaramelException { - if (githubUrl == null || githubUrl.isEmpty()) { - throw new KaramelException("Misformed empty url: " + githubUrl); - } - String protocol = githubUrl.substring(0,5); - - if (protocol.substring(0,4).compareToIgnoreCase("http") != 0 && - protocol.substring(0,4).compareToIgnoreCase("git@") != 0 && - protocol.compareToIgnoreCase("https") != 0 ) { - throw new KaramelException("Misformed url - only 'http' and 'git@' supported: " + githubUrl); - } - - return protocol.compareToIgnoreCase("https") == 0 ? protocol : protocol.substring(0,4); - } - - public static String extractRepoName(String githubUrl) throws KaramelException { - int e = githubUrl.lastIndexOf(".git"); - int s = githubUrl.lastIndexOf("/"); - if (s == -1 || e == -1) { - throw new KaramelException("Misformed url: " + githubUrl); - } - String repoName = githubUrl.substring(s+1, e); - if (repoName == null || repoName.isEmpty()) { - throw new KaramelException("Misformed url repo/owner: " + githubUrl); - } - return repoName; - } - - public static String extractUserName(String githubUrl) throws KaramelException { - String protocol = getProtocol(githubUrl); - - int e = githubUrl.lastIndexOf(".git"); - int s = githubUrl.lastIndexOf("/"); - if (s == -1 || e == -1) { - throw new KaramelException("Misformed url: " + githubUrl); - } - int s1 = -1; - if (protocol.compareToIgnoreCase("http") == 0 || protocol.compareToIgnoreCase("https") == 0) { - s1 = githubUrl.lastIndexOf("/", s-1); - } else if (protocol.compareToIgnoreCase("git@") == 0) { - s1 = githubUrl.lastIndexOf(":", s-1); - } else { - throw new KaramelException("Unrecognized protocol: " + protocol); - } - if (s1 == -1) { - throw new KaramelException("Misformed url: " + githubUrl); - } - String owner = githubUrl.substring(s1+1, s); - if (owner == null || owner.isEmpty()) { - throw new KaramelException("Misformed url repo/owner: " + githubUrl); - } - return owner; - } - -} diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/machines/SshMachine.java b/karamel-core/src/main/java/se/kth/karamel/backend/machines/SshMachine.java index 25be7fcb..4cc11825 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/machines/SshMachine.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/machines/SshMachine.java @@ -10,9 +10,12 @@ import java.io.SequenceInputStream; import java.nio.file.Files; import java.util.List; +import java.util.Optional; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; + +import com.google.common.base.Charsets; import net.schmizz.sshj.SSHClient; import net.schmizz.sshj.connection.ConnectionException; import net.schmizz.sshj.connection.channel.direct.Session; @@ -20,6 +23,7 @@ import net.schmizz.sshj.transport.verification.PromiscuousVerifier; import net.schmizz.sshj.userauth.keyprovider.KeyProvider; import org.apache.log4j.Logger; +import se.kth.karamel.backend.dag.RecipeSerialization; import se.kth.karamel.backend.running.model.MachineRuntime; import se.kth.karamel.backend.running.model.tasks.ShellCommand; import se.kth.karamel.backend.running.model.tasks.Task; @@ -30,6 +34,9 @@ import java.security.Security; import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantReadWriteLock; + import net.schmizz.sshj.userauth.UserAuthException; import net.schmizz.sshj.userauth.password.PasswordFinder; import net.schmizz.sshj.userauth.password.Resource; @@ -41,7 +48,6 @@ import se.kth.karamel.backend.running.model.tasks.KillSessionTask; import se.kth.karamel.backend.running.model.tasks.RunRecipeTask; import se.kth.karamel.common.util.Confs; -import se.kth.karamel.common.util.IoUtils; /** * @@ -61,13 +67,14 @@ public class SshMachine implements MachineInterface, Runnable { private SSHClient client; private long lastHeartbeat = 0; private final BlockingQueue taskQueue = new ArrayBlockingQueue<>(Settings.MACHINES_TASKQUEUE_SIZE); - private boolean stopping = false; - private boolean killing = false; + private final AtomicBoolean stopping = new AtomicBoolean(false); + private final AtomicBoolean killing = new AtomicBoolean(false); private final SshShell shell; private Task activeTask; private boolean isSucceedTaskHistoryUpdated = false; private final List succeedTasksHistory = new ArrayList<>(); private static Confs confs = Confs.loadKaramelConfs(); + private final ReentrantReadWriteLock clientConnectLock = new ReentrantReadWriteLock(); /** * This constructor is used for users with SSH keys protected by a password @@ -95,7 +102,7 @@ public SshShell getShell() { } public void setStopping(boolean stopping) { - this.stopping = stopping; + this.stopping.set(stopping); } public void pause() { @@ -126,16 +133,36 @@ private boolean anyFailure() { return anyfailure; } + private void prepareSerializedTask(Task task) throws InterruptedException { + Optional maybeRS = getRecipeSerialization(task); + if (maybeRS.isPresent()) { + RecipeSerialization recipeSerialization = maybeRS.get(); + recipeSerialization.prepareToExecute((RunRecipeTask) task); + } + } + + private Optional getRecipeSerialization(Task task) { + if (task instanceof RunRecipeTask) { + RunRecipeTask recipeTask = (RunRecipeTask) task; + RecipeSerialization serialization = task.getDagCallback().getDag() + .getSerializableRecipeCounter(recipeTask.getRecipeCanonicalName()); + return Optional.ofNullable(serialization); + } + return Optional.empty(); + } + @Override public void run() { logger.debug(String.format("%s: Started SSH_Machine d'-'", machineEntity.getId())); try { - while (!stopping) { + boolean hasAbortedDueToFailure = false; + while (!stopping.get()) { try { if (machineEntity.getLifeStatus() == MachineRuntime.LifeStatus.CONNECTED && (machineEntity.getTasksStatus() == MachineRuntime.TasksStatus.ONGOING || machineEntity.getTasksStatus() == MachineRuntime.TasksStatus.EMPTY)) { try { + boolean retry = false; if (activeTask == null) { if (taskQueue.isEmpty()) { machineEntity.setTasksStatus(MachineRuntime.TasksStatus.EMPTY, null, null); @@ -143,15 +170,74 @@ public void run() { activeTask = taskQueue.take(); logger.debug(String.format("%s: Taking a new task from the queue.", machineEntity.getId())); machineEntity.setTasksStatus(MachineRuntime.TasksStatus.ONGOING, null, null); + hasAbortedDueToFailure = false; } else { + retry = true; logger.debug( String.format("%s: Retrying a task that didn't complete on last execution attempt.", machineEntity.getId())); } logger.debug(String.format("%s: Task for execution.. '%s'", machineEntity.getId(), activeTask.getName())); + + Optional recipeSerialization = getRecipeSerialization(activeTask); + if (recipeSerialization.isPresent()) { + RecipeSerialization rs = recipeSerialization.get(); + // In retries we don't check if it has failed because we already know it has failed, hence retried + // If we do check, it will lead to deadlock + // + // After the task gotten the green light to proceed with running the task - parallelism claim + // has been satisfied, we check again if the task has failed. While waiting for the claim to be + // satisfied, the task might have failed on another node. In that case, we continue the loop without + // executing the task. Then we come here, where it is a retry but it has aborted due to failure + // and we wait until the failure is resolved + if (!retry || hasAbortedDueToFailure) { + logger.debug(String.format("%s: retry: %s hasAborted: %s", machineEntity.getId(), + activeTask.getName(), retry, hasAbortedDueToFailure)); + synchronized (rs) { + logger.debug(String.format("%s: Recipe %s RecipeSerializationID: %s Has failed: %s", + activeTask.getMachine().getId(), + activeTask.getName(), + rs.hashCode(), + rs.hasFailed())); + if (rs.hasFailed()) { + logger.info(String.format("%s: Recipe %s has failed on another node. Wait until it succeeds", + machineEntity.getId(), activeTask.getName())); + rs.wait(); + } else { + logger.debug(String.format("%s: Recipe %s has NOT failed on another node. Executing", + machineEntity.getId(), activeTask.getName())); + } + } + } else { + logger.debug(String.format("%s: %s it is a retry", machineEntity.getId(), activeTask.getName())); + } + } + + prepareSerializedTask(activeTask); + + if (recipeSerialization.isPresent()) { + RecipeSerialization rs = recipeSerialization.get(); + // For explanation read above + if (!retry || hasAbortedDueToFailure) { + synchronized (rs) { + logger.debug(String.format("%s: %s Checking again after getting hold of the lock if recipe has " + + "failed", machineEntity.getId(), activeTask.getName())); + if (rs.hasFailed()) { + logger.info(String.format("%s: %s Recipe has failed on another node, releasing the lock and " + + "continue", machineEntity.getId(), activeTask.getName())); + rs.release((RunRecipeTask) activeTask); + logger.debug(String.format("%s: %s Released and continue", machineEntity.getId(), + activeTask.getName())); + hasAbortedDueToFailure = true; + continue; + } + } + } + } runTask(activeTask); + hasAbortedDueToFailure = false; } catch (InterruptedException ex) { - if (stopping) { + if (stopping.get()) { logger.debug(String.format("%s: Stopping SSH_Machine", machineEntity.getId())); return; } else { @@ -167,7 +253,7 @@ public void run() { try { Thread.sleep(Settings.MACHINE_TASKRUNNER_BUSYWAITING_INTERVALS); } catch (InterruptedException ex) { - if (!stopping) { + if (!stopping.get()) { logger.error( String.format("%s: Got interrupted without having recieved stopping signal", machineEntity.getId())); @@ -207,7 +293,7 @@ public void killTaskSession(Task task) { if (activeTask == task) { logger.info(String.format("Killing '%s' on '%s'", task.getName(), task.getMachine().getPublicIp())); KillSessionTask killTask = new KillSessionTask(machineEntity); - killing = true; + killing.set(true); runTask(killTask); } else { logger.warn(String.format("Request to kill '%s' on '%s' but the task is not ongoing now", task.getName(), @@ -272,7 +358,7 @@ private void runTask(Task task) { for (ShellCommand cmd : commands) { if (cmd.getStatus() != ShellCommand.Status.DONE) { logger.debug(String.format("command to run %s", cmd.getCmdStr())); - runSshCmd(cmd, task, false); + runSshCmd2(cmd, task, false); if (cmd.getStatus() != ShellCommand.Status.DONE) { task.failed(String.format("%s: Command did not complete: %s", machineEntity.getId(), @@ -313,6 +399,123 @@ private void runTask(Task task) { } } + private void runSshCmd2(ShellCommand shellCommand, Task task, boolean killCommand) { + logger.info(getLogWithmachineId("Received task to run " + task.getName())); + logger.debug(getLogWithmachineId("Command to run " + shellCommand.getCmdStr())); + + int numCmdRetries = Settings.SSH_CMD_RETRY_NUM; + int delayBetweenRetries = Settings.SSH_CMD_RETRY_INTERVALS; + Session session = null; + + while (!stopping.get() && !killing.get()) { + logger.info(getLogWithmachineId(String.format("Running task: %s", task.getName()))); + shellCommand.setStatus(ShellCommand.Status.ONGOING); + Session.Command cmd = null; + try { + clientConnectLock.writeLock().lock(); + if (client == null || !client.isConnected()) { + logger.info(getLogWithmachineId("SSH client is disconnected. Connecting")); + connect(); + } + logger.debug(getLogWithmachineId("Starting new SSH session")); + session = client.startSession(); + logger.info(getLogWithmachineId("Started new SSH session")); + if (task.isSudoTerminalReqd()) { + session.allocateDefaultPTY(); + } + + logger.info(getLogWithmachineId("Executing remote command")); + String cmdStr = shellCommand.getCmdStr(); + String password = ClusterService.getInstance().getCommonContext().getSudoAccountPassword(); + if (password != null && !password.isEmpty()) { + cmd = session.exec(cmdStr.replaceAll("%password_hidden%", password)); + } else { + cmd = session.exec(cmdStr); + } + try { + logger.debug(getLogWithmachineId("Waiting for command to finish")); + cmd.join(Settings.SSH_CMD_MAX_TIOMEOUT, TimeUnit.MINUTES); + } catch (ConnectionException tex) { + logger.warn(getLogWithmachineId("Timeout reached while waiting for command to finish executing")); + throw tex; + } + updateHeartbeat(); + SequenceInputStream sequenceInputStream = new SequenceInputStream(cmd.getInputStream(), cmd.getErrorStream()); + LogService.serializeTaskLog(task, machineEntity.getPublicIp(), sequenceInputStream); + if (cmd.getExitStatus() == 0) { + logger.info(getLogWithmachineId("Command finished successfully")); + shellCommand.setStatus(ShellCommand.Status.DONE); + return; + } + String log = getLogWithmachineId("Command " + task.getName() + " failed with exit code " + cmd.getExitStatus()); + logger.warn(log); + throw new KaramelException(log); + } catch (Exception ex) { + if (ex instanceof ConnectionException || ex instanceof TransportException) { + if (session != null) { + try { + logger.warn(getLogWithmachineId("Closing SSH session after error")); + session.close(); + } catch (TransportException | ConnectionException cex) { + logger.warn(getLogWithmachineId("Error while closing session, but we ignore it"), cex); + } + } else { + logger.info(getLogWithmachineId("Will not close SSH session because it is null")); + } + + if (client != null) { + try { + logger.warn(getLogWithmachineId("Disconnecting SSH session after error")); + client.disconnect(); + } catch (IOException cex) { + logger.warn(getLogWithmachineId("Error while disconnecting client, but we ignore it"), cex); + } + } else { + logger.info(getLogWithmachineId("Will not disconnect SSH client because it is null")); + } + } + + logger.warn(getLogWithmachineId("Error while executing command")); + if (--numCmdRetries <= 0) { + logger.error(getLogWithmachineId("Terminal error while executing command"), ex); + logger.error(getLogWithmachineId(String.format("Exhausted all %d retries, giving up!!!", + Settings.SSH_CMD_RETRY_NUM))); + shellCommand.setStatus(ShellCommand.Status.FAILED); + return; + } + try { + TimeUnit.MILLISECONDS.sleep(delayBetweenRetries); + } catch (InterruptedException iex) { + if (!stopping.get() && !killing.get()) { + logger.warn(getLogWithmachineId("Interrupted waiting to retry a command. Continuing...")); + } + } + delayBetweenRetries *= Settings.SSH_CMD_RETRY_SCALE; + } finally { + killing.compareAndSet(true, false); + if (session != null && session.isOpen()) { + try { + session.close(); + } catch (TransportException | ConnectionException ex) { + logger.info(getLogWithmachineId("Error while closing SSH session, ignoring...")); + } + } + clientConnectLock.writeLock().unlock(); + } + } + } + + private String getLogWithmachineId(String log) { + return String.format("%s: %s", machineEntity.getId(), log); + } + + /* + * This method is not used any longer. It has been re-written into runShhCmd2 + * to address connectivity issues with the way it is handling the SSH client + * and sessions and concurrency. + * + * Leaving it here just for reference + */ private void runSshCmd(ShellCommand shellCommand, Task task, boolean killcommand) { logger.debug(String.format("recieved a command to run '%s'", shellCommand.getCmdStr())); int numCmdRetries = Settings.SSH_CMD_RETRY_NUM; @@ -320,7 +523,7 @@ private void runSshCmd(ShellCommand shellCommand, Task task, boolean killcommand boolean finished = false; Session session = null; - while (!stopping && !killing && !finished && numCmdRetries > 0) { + while (!stopping.get() && !killing.get() && !finished && numCmdRetries > 0) { shellCommand.setStatus(ShellCommand.Status.ONGOING); try { logger.info(String.format("%s: Running task: %s", machineEntity.getId(), task.getName())); @@ -353,7 +556,7 @@ private void runSshCmd(ShellCommand shellCommand, Task task, boolean killcommand try { Thread.sleep(timeBetweenRetries); } catch (InterruptedException ex3) { - if (!stopping && !killing) { + if (!stopping.get() && !killing.get()) { logger.warn(String.format("%s: Interrupted while waiting to start ssh session. Continuing...", machineEntity.getId())); } @@ -382,12 +585,12 @@ private void runSshCmd(ShellCommand shellCommand, Task task, boolean killcommand SequenceInputStream sequenceInputStream = new SequenceInputStream(cmd.getInputStream(), cmd.getErrorStream()); LogService.serializeTaskLog(task, machineEntity.getPublicIp(), sequenceInputStream); } catch (ConnectionException | TransportException ex) { - if (!killing + if (!killing.get() && getMachineEntity().getGroup().getCluster().getPhase() != ClusterRuntime.ClusterPhases.TERMINATING) { logger.error(String.format("%s: Couldn't excecute command", machineEntity.getId()), ex); } - if (killing) { + if (killing.get()) { logger.info(String.format("Killed '%s' on '%s' successfully...", task.getName(), machineEntity.getId())); } } @@ -399,7 +602,7 @@ && getMachineEntity().getGroup().getCluster().getPhase() != ClusterRuntime.Clust try { Thread.sleep(timeBetweenRetries); } catch (InterruptedException ex) { - if (!stopping && !killing) { + if (!stopping.get() && !killing.get()) { logger.warn( String.format("%s: Interrupted waiting to retry a command. Continuing...", machineEntity.getId())); } @@ -414,7 +617,7 @@ && getMachineEntity().getGroup().getCluster().getPhase() != ClusterRuntime.Clust logger.error(String.format("Couldn't close ssh session to '%s' ", machineEntity.getId()), ex); } } - killing = false; + killing.set(false); } } } @@ -438,6 +641,7 @@ private void connect() throws KaramelException { if (client == null || !client.isConnected()) { isSucceedTaskHistoryUpdated = false; try { + clientConnectLock.writeLock().lock(); KeyProvider keys; client = new SSHClient(); client.addHostKeyVerifier(new PromiscuousVerifier()); @@ -514,8 +718,9 @@ private void connect() throws KaramelException { throw exp; } catch (IOException e) { throw new KaramelException(e); + } finally { + clientConnectLock.writeLock().unlock(); } - return; } } @@ -534,6 +739,7 @@ public void ping() throws KaramelException { if (client != null && client.isConnected()) { updateHeartbeat(); } else { + logger.warn("Lost connection to " + machineEntity.getId() + " Reconnecting..."); connect(); } } @@ -564,7 +770,7 @@ private void loadSucceedListFromMachineToMemory() { //shoudn't throw this because I am deleting the local file already here } finally { try { - String list = IoUtils.readContentFromPath(localSucceedPath); + String list = com.google.common.io.Files.toString(new File(localSucceedPath), Charsets.UTF_8); String[] items = list.split("\n"); succeedTasksHistory.clear(); succeedTasksHistory.addAll(Arrays.asList(items)); @@ -579,20 +785,25 @@ private void loadSucceedListFromMachineToMemory() { public void downloadRemoteFile(String remoteFilePath, String localFilePath, boolean overwrite) throws KaramelException, IOException { - connect(); - SCPFileTransfer scp = client.newSCPFileTransfer(); - File f = new File(localFilePath); - f.mkdirs(); - // Don't collect logs of values, just overwrite - if (f.exists()) { - if (overwrite) { - f.delete(); - } else { - throw new KaramelException(String.format("%s: Local file already exist %s", - machineEntity.getId(), localFilePath)); + try { + clientConnectLock.writeLock().lock(); + connect(); + SCPFileTransfer scp = client.newSCPFileTransfer(); + File f = new File(localFilePath); + f.mkdirs(); + // Don't collect logs of values, just overwrite + if (f.exists()) { + if (overwrite) { + f.delete(); + } else { + throw new KaramelException(String.format("%s: Local file already exist %s", + machineEntity.getId(), localFilePath)); + } } + // If the file doesn't exist, it should quickly throw an IOException + scp.download(remoteFilePath, localFilePath); + } finally { + clientConnectLock.writeLock().unlock(); } - // If the file doesn't exist, it should quickly throw an IOException - scp.download(remoteFilePath, localFilePath); } } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/AptGetEssentialsTask.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/AptGetEssentialsTask.java index 34002233..8c517513 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/AptGetEssentialsTask.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/AptGetEssentialsTask.java @@ -9,7 +9,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import se.kth.karamel.backend.ClusterService; import se.kth.karamel.backend.converter.ShellCommandBuilder; import se.kth.karamel.backend.launcher.OsType; import se.kth.karamel.backend.machines.TaskSubmitter; @@ -35,12 +34,10 @@ public AptGetEssentialsTask(MachineRuntime machine, ClusterStats clusterStats, T public List getCommands() throws IOException { OsType osType = getMachine().getOsType(); String sudocommand = getSudoCommand(); - String githuuser = ClusterService.getInstance().getCommonContext().getGithubUsername(); String osfamily = osType.family.toString().toLowerCase(); if (commands == null) { commands = ShellCommandBuilder.makeSingleFileCommand(Settings.SCRIPT_PATH_APTGET_ESSENTIALS, "sudo_command", sudocommand, - "github_username", githuuser, "osfamily", osfamily, "task_id", getId(), "install_dir_path", Settings.REMOTE_INSTALL_DIR_PATH(getSshUser()), diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/DagBuilder.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/DagBuilder.java index 38bd80a5..a87b7af8 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/DagBuilder.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/DagBuilder.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.running.model.tasks; import com.google.gson.Gson; @@ -10,13 +5,15 @@ import com.google.gson.JsonObject; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; + import org.apache.log4j.Logger; import se.kth.karamel.backend.ClusterDefinitionService; import se.kth.karamel.backend.converter.ChefJsonGenerator; import se.kth.karamel.backend.converter.UserClusterDataExtractor; import se.kth.karamel.backend.dag.Dag; +import se.kth.karamel.common.clusterdef.Cookbook; +import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.launcher.amazon.InstanceType; import se.kth.karamel.backend.machines.TaskSubmitter; import se.kth.karamel.backend.running.model.ClusterRuntime; @@ -26,14 +23,11 @@ import se.kth.karamel.common.clusterdef.Ec2; import se.kth.karamel.common.clusterdef.Provider; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.json.JsonGroup; -import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.cookbookmeta.CookbookCache; import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.exception.DagConstructionException; import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.common.cookbookmeta.CookbookUrls; import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.cookbookmeta.KaramelFileYamlDeps; import se.kth.karamel.common.util.Confs; @@ -62,10 +56,8 @@ public static Dag getPurgingDag(JsonCluster cluster, ClusterRuntime clusterEntit TaskSubmitter submitter, Map chefJsons) throws KaramelException { Dag dag = new Dag(); Map allRecipeTasks = new HashMap<>(); - CookbookCache cache = ClusterDefinitionService.CACHE; - List kcbs = cache.loadRootKaramelizedCookbooks(cluster); - machineLevelTasks(cluster, clusterEntity, clusterStats, submitter, dag, kcbs); - cookbookLevelPurgingTasks(cluster, clusterEntity, clusterStats, chefJsons, submitter, allRecipeTasks, dag, kcbs); + machineLevelTasks(cluster, clusterEntity, clusterStats, submitter, dag); + cookbookLevelPurgingTasks(cluster, clusterEntity, clusterStats, chefJsons, submitter, allRecipeTasks, dag); return dag; } @@ -86,13 +78,10 @@ public static Dag getInstallationDag(JsonCluster cluster, ClusterRuntime cluster TaskSubmitter submitter, Map chefJsons) throws KaramelException { Dag dag = new Dag(); Map allRecipeTasks = new HashMap<>(); - CookbookCache cache = ClusterDefinitionService.CACHE; - List kcbs = cache.loadRootKaramelizedCookbooks(cluster); - machineLevelTasks(cluster, clusterEntity, clusterStats, submitter, dag, kcbs); - cookbookLevelInstallationTasks(cluster, clusterEntity, clusterStats, chefJsons, submitter, allRecipeTasks, dag, - kcbs); + machineLevelTasks(cluster, clusterEntity, clusterStats, submitter, dag); + cookbookLevelInstallationTasks(cluster, clusterEntity, clusterStats, chefJsons, submitter, allRecipeTasks, dag); Map> rlts = recipeLevelTasks(cluster, clusterEntity, clusterStats, chefJsons, submitter, - allRecipeTasks, dag, kcbs); + allRecipeTasks, dag); updateKaramelDependencies(allRecipeTasks, dag, rlts); return dag; } @@ -105,7 +94,6 @@ private static boolean updateKaramelDependencies(Map allR cbids.add(task.getCookbookId()); } CookbookCache cache = ClusterDefinitionService.CACHE; - cache.prepareParallel(cbids); for (RunRecipeTask task : allRecipeTasks.values()) { String tid = task.uniqueId(); KaramelizedCookbook kcb = cache.get(task.getCookbookId()); @@ -142,24 +130,20 @@ private static boolean updateKaramelDependencies(Map allR * @param submitter * @param allRecipeTasks * @param dag - * @param rootCookbooks * @return * @throws KaramelException */ public static Map> recipeLevelTasks(JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, Map chefJsons, TaskSubmitter submitter, - Map allRecipeTasks, Dag dag, - List rootCookbooks) throws KaramelException { + Map allRecipeTasks, Dag dag) throws KaramelException { Map> map = new HashMap<>(); for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName()); for (MachineRuntime me : ge.getMachines()) { - for (JsonCookbook jc : jg.getCookbooks()) { - for (JsonRecipe rec : jc.getRecipes()) { - JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName()); - addRecipeTaskForMachineIntoRecipesMap(rec.getCanonicalName(), me, clusterStats, map, json1, submitter, - jc.getId(), jc.getName(), allRecipeTasks, dag, rootCookbooks); - } + for (JsonRecipe rec : jg.getRecipes()) { + JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName()); + addRecipeTaskForMachineIntoRecipesMap(rec.getCanonicalName(), me, clusterStats, map, json1, submitter, + rec.getCookbook().getCookbookName(), allRecipeTasks, dag, cluster.getRootCookbooks()); } } } @@ -171,11 +155,12 @@ public static Map> recipeLevelTasks(JsonCluster cluste */ private static RunRecipeTask addRecipeTaskForMachineIntoRecipesMap(String recipeName, MachineRuntime machine, ClusterStats clusterStats, Map> map, JsonObject chefJson, TaskSubmitter submitter, - String cookbookId, String cookbookName, Map allRecipeTasks, Dag dag, - List rootCookbooks) - throws DagConstructionException { + String cookbookId, Map allRecipeTasks, Dag dag, + Map rootCookbooks) throws DagConstructionException { + RunRecipeTask t1 = makeRecipeTaskIfNotExist(recipeName, machine, clusterStats, chefJson, submitter, cookbookId, - cookbookName, allRecipeTasks, dag, rootCookbooks); + allRecipeTasks, dag, rootCookbooks); + Map map1 = map.get(recipeName); if (map1 == null) { map1 = new HashMap<>(); @@ -190,8 +175,9 @@ private static RunRecipeTask addRecipeTaskForMachineIntoRecipesMap(String recipe */ private static RunRecipeTask makeRecipeTaskIfNotExist(String recipeName, MachineRuntime machine, ClusterStats clusterStats, JsonObject chefJson, - TaskSubmitter submitter, String cookbookId, String cookbookName, Map allRecipeTasks, - Dag dag, List rootCookbooks) throws DagConstructionException { + TaskSubmitter submitter, String cookbookId, Map allRecipeTasks, + Dag dag, Map rootCookbooks) throws DagConstructionException { + String recId = RunRecipeTask.makeUniqueId(machine.getId(), recipeName); RunRecipeTask runRecipeTask = allRecipeTasks.get(recId); if (!allRecipeTasks.containsKey(recId)) { @@ -202,7 +188,7 @@ private static RunRecipeTask makeRecipeTaskIfNotExist(String recipeName, Machine String jsonString = gson.toJson(chefJson); runRecipeTask = new RunRecipeTask(machine, clusterStats, recipeName, jsonString, - submitter, cookbookId, cookbookName, rootCookbooks); + submitter, cookbookId, rootCookbooks); dag.addTask(runRecipeTask); } allRecipeTasks.put(recId, runRecipeTask); @@ -225,27 +211,28 @@ private static RunRecipeTask makeRecipeTaskIfNotExist(String recipeName, Machine */ public static Map> cookbookLevelPurgingTasks(JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, Map chefJsons, - TaskSubmitter submitter, Map allRecipeTasks, Dag dag, - List rootCookbooks) throws KaramelException { + TaskSubmitter submitter, Map allRecipeTasks, Dag dag) + throws KaramelException { + Map> map = new HashMap<>(); for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName()); for (MachineRuntime me : ge.getMachines()) { Map map1 = new HashMap<>(); - for (KaramelizedCookbook rcb : rootCookbooks) { - CookbookUrls urls = rcb.getUrls(); - VendorCookbookTask t1 = new VendorCookbookTask(me, clusterStats, submitter, urls.id, - Settings.REMOTE_COOKBOOKS_PATH(me.getSshUser()), - urls.repoUrl, urls.repoName, urls.cookbookRelPath, urls.branch); + + for (Map.Entry cb : cluster.getRootCookbooks().entrySet()) { + VendorCookbookTask t1 = new VendorCookbookTask(me, clusterStats, submitter, + Settings.REMOTE_COOKBOOKS_PATH(me.getSshUser()), cb.getKey(), cb.getValue()); + dag.addTask(t1); map1.put(t1.uniqueId(), t1); } - for (JsonCookbook jc : jg.getCookbooks()) { - CookbookUrls urls = jc.getUrls(); - String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.PURGE_RECIPE; + + for (KaramelizedCookbook kbc : jg.getCookbooks()) { + String recipeName = kbc.getCookbookName() + Settings.COOKBOOK_DELIMITER + Settings.PURGE_RECIPE; JsonObject json = chefJsons.get(me.getId() + recipeName); - RunRecipeTask t2 = makeRecipeTaskIfNotExist(recipeName, me, clusterStats, json, submitter, urls.id, - jc.getName(), allRecipeTasks, dag, rootCookbooks); + RunRecipeTask t2 = makeRecipeTaskIfNotExist(recipeName, me, clusterStats, json, submitter, + kbc.getCookbookName(), allRecipeTasks, dag, cluster.getRootCookbooks()); map1.put(t2.uniqueId(), t2); } logger.debug(String.format("Cookbook-level tasks for the machine '%s' in the group '%s' are: %s", @@ -274,29 +261,28 @@ public static Map> cookbookLevelPurgingTasks(JsonClust * @return * @throws KaramelException */ - public static Map> cookbookLevelInstallationTasks(JsonCluster cluster, + private static Map> cookbookLevelInstallationTasks(JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, Map chefJsons, - TaskSubmitter submitter, Map allRecipeTasks, Dag dag, - List rootCookbooks) throws KaramelException { + TaskSubmitter submitter, Map allRecipeTasks, Dag dag) + throws KaramelException { Map> map = new HashMap<>(); for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName()); for (MachineRuntime me : ge.getMachines()) { Map map1 = new HashMap<>(); - for (KaramelizedCookbook rcb : rootCookbooks) { - CookbookUrls urls = rcb.getUrls(); - VendorCookbookTask t1 = new VendorCookbookTask(me, clusterStats, submitter, urls.id, - Settings.REMOTE_COOKBOOKS_PATH(me.getSshUser()), - urls.repoUrl, urls.repoName, urls.cookbookRelPath, urls.branch); + + for (Map.Entry cb : cluster.getRootCookbooks().entrySet()) { + VendorCookbookTask t1 = new VendorCookbookTask(me, clusterStats, submitter, + Settings.REMOTE_COOKBOOKS_PATH(me.getSshUser()), cb.getKey(), cb.getValue()); dag.addTask(t1); map1.put(t1.uniqueId(), t1); } - for (JsonCookbook jc : jg.getCookbooks()) { - CookbookUrls urls = jc.getUrls(); - String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; + + for (KaramelizedCookbook kcb : jg.getCookbooks()) { + String recipeName = kcb.getCookbookName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; JsonObject json = chefJsons.get(me.getId() + recipeName); RunRecipeTask t2 = makeRecipeTaskIfNotExist(recipeName, me, clusterStats, - json, submitter, urls.id, jc.getName(), allRecipeTasks, dag, rootCookbooks); + json, submitter, kcb.getCookbookName(), allRecipeTasks, dag, cluster.getRootCookbooks()); map1.put(t2.uniqueId(), t2); } logger.debug(String.format("Cookbook-level tasks for the machine '%s' in the group '%s' are: %s", @@ -324,14 +310,16 @@ public static Map> cookbookLevelInstallationTasks(Json * @throws KaramelException */ public static void machineLevelTasks(JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, - TaskSubmitter submitter, Dag dag, List rootCookbooks) throws KaramelException { + TaskSubmitter submitter, Dag dag) throws KaramelException { Confs confs = Confs.loadKaramelConfs(); String prepStoragesConf = confs.getProperty(Settings.PREPARE_STORAGES_KEY); for (GroupRuntime ge : clusterEntity.getGroups()) { for (MachineRuntime me : ge.getMachines()) { - String vendorPath = UserClusterDataExtractor.makeVendorPath(me.getSshUser(), rootCookbooks); + + String vendorPath = UserClusterDataExtractor.makeVendorPath(me.getSshUser(), cluster.getRootCookbooks()); FindOsTypeTask findOs = new FindOsTypeTask(me, clusterStats, submitter); dag.addTask(findOs); + Provider provider = UserClusterDataExtractor.getGroupProvider(cluster, ge.getName()); boolean storagePreparation = (prepStoragesConf != null && prepStoragesConf.equalsIgnoreCase("true") && (provider instanceof Ec2)); @@ -342,6 +330,7 @@ public static void machineLevelTasks(JsonCluster cluster, ClusterRuntime cluster = new PrepareStoragesTask(me, clusterStats, submitter, instanceType.getStorageDevices()); dag.addTask(st); } + AptGetEssentialsTask t1 = new AptGetEssentialsTask(me, clusterStats, submitter, storagePreparation); InstallChefdkTask t2 = new InstallChefdkTask(me, clusterStats, submitter); MakeSoloRbTask t3 = new MakeSoloRbTask(me, vendorPath, clusterStats, submitter); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/FindOsTypeTask.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/FindOsTypeTask.java index f0c564c9..c2eb661a 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/FindOsTypeTask.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/FindOsTypeTask.java @@ -5,10 +5,14 @@ */ package se.kth.karamel.backend.running.model.tasks; +import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Collections; import java.util.List; import java.util.Set; + +import com.google.common.io.Files; import org.apache.log4j.Logger; import se.kth.karamel.backend.converter.ShellCommandBuilder; import se.kth.karamel.backend.launcher.OsType; @@ -17,7 +21,6 @@ import se.kth.karamel.backend.running.model.MachineRuntime; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.stats.ClusterStats; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.Settings; /** @@ -73,7 +76,7 @@ public void collectResults(MachineInterface sshMachine) throws KaramelException return; } try { - String content = IoUtils.readContentFromPath(localResultsFile); + String content = Files.toString(new File(localResultsFile), Charset.forName("UTF-8")); content = content.trim().toLowerCase(); if (content.isEmpty()) { throw new KaramelException(String.format("The OS-Type file for %s is empty", publicIp)); diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/MakeSoloRbTask.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/MakeSoloRbTask.java index a69061cd..135a2b8c 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/MakeSoloRbTask.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/MakeSoloRbTask.java @@ -23,7 +23,8 @@ public class MakeSoloRbTask extends Task { private final String vendorPath; - public MakeSoloRbTask(MachineRuntime machine, String vendorPath, ClusterStats clusterStats, TaskSubmitter submitter) { + public MakeSoloRbTask(MachineRuntime machine, String vendorPath, ClusterStats clusterStats, + TaskSubmitter submitter) { super("make solo.rb", "make solo.rb", false, machine, clusterStats, submitter); this.vendorPath = vendorPath; } @@ -31,11 +32,18 @@ public MakeSoloRbTask(MachineRuntime machine, String vendorPath, ClusterStats cl @Override public List getCommands() throws IOException { if (commands == null) { + String rubygemsUrlConf = ""; + String rubygemsUrl = conf.getProperty(Settings.CHEF_RUBYGEMS_URL); + if (rubygemsUrl != null) { + rubygemsUrlConf = String.format("rubygems_url \"%s\"", rubygemsUrl); + } commands = ShellCommandBuilder.makeSingleFileCommand(Settings.SCRIPT_PATH_MAKE_SOLO_RB, "install_dir_path", Settings.REMOTE_INSTALL_DIR_PATH(getSshUser()), "cookbooks_path", vendorPath, "sudo_command", getSudoCommand(), - "pid_file", Settings.PID_FILE_NAME); + "pid_file", Settings.PID_FILE_NAME, + "file_cache_path", conf.getProperty(Settings.CHEF_FILE_CACHE_PATH), + "rubygems_url", rubygemsUrlConf); } return commands; } diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/RunRecipeTask.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/RunRecipeTask.java index 189c7113..ee48d977 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/RunRecipeTask.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/RunRecipeTask.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.running.model.tasks; import com.google.gson.Gson; @@ -26,32 +21,26 @@ import se.kth.karamel.backend.machines.MachineInterface; import se.kth.karamel.backend.machines.TaskSubmitter; import se.kth.karamel.backend.running.model.MachineRuntime; -import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; +import se.kth.karamel.common.clusterdef.Cookbook; import se.kth.karamel.common.stats.ClusterStats; import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.exception.KaramelException; -/** - * - * @author kamal - */ public class RunRecipeTask extends Task { private static final Logger logger = Logger.getLogger(RunRecipeTask.class); private final String recipeCanonicalName; private String json; private final String cookbookId; - private final String cookbookName; - private final List rookCookbooks; + private final Map rookCookbooks; public RunRecipeTask(MachineRuntime machine, ClusterStats clusterStats, String recipe, String json, - TaskSubmitter submitter, String cookbookId, String cookbookName, List rookCookbooks) { + TaskSubmitter submitter, String cookbookId, Map rootCookbooks) { super(recipe, cookbookId + "/" + recipe, false, machine, clusterStats, submitter); this.recipeCanonicalName = recipe; this.json = json; this.cookbookId = cookbookId; - this.cookbookName = cookbookName; - this.rookCookbooks = rookCookbooks; + this.rookCookbooks = rootCookbooks; } /** @@ -134,13 +123,6 @@ public String getCookbookId() { return cookbookId; } - public String getRecipeName() { - return recipeCanonicalName.split(Settings.COOKBOOK_DELIMITER)[1]; - } - - public String getCookbookName() { - return cookbookName; - } public static String installRecipeIdFromCookbookName(String machineId, String cookbook) { String installName = cookbook + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; @@ -174,8 +156,8 @@ public Set dagDependencies() { String installId = installRecipeIdFromAnotherRecipeName(getMachineId(), recipeCanonicalName); String purgeId = purgeRecipeIdFromAnotherRecipeName(getMachineId(), recipeCanonicalName); if (uniqueId().equals(installId) || uniqueId().equals(purgeId)) { - for (KaramelizedCookbook kcb : rookCookbooks) { - String id = VendorCookbookTask.makeUniqueId(getMachineId(), kcb.getUrls().id); + for (Map.Entry rootCookbook : rookCookbooks.entrySet()) { + String id = VendorCookbookTask.makeUniqueId(getMachineId(), rootCookbook.getKey()); deps.add(id); } } else { diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/Task.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/Task.java index cf25d155..414af8c1 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/Task.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/Task.java @@ -19,6 +19,8 @@ import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.stats.ClusterStats; import se.kth.karamel.common.stats.TaskStat; +import se.kth.karamel.common.util.Confs; +import se.kth.karamel.common.util.Settings; /** * @@ -30,7 +32,7 @@ public abstract class Task implements DagTask, TaskCallback { public static enum Status { - WAITING, READY, EXIST, ONGOING, DONE, FAILED, SKIPPED; + WAITING, BLOCKED, READY, EXIST, ONGOING, DONE, FAILED, SKIPPED; } private Status status = Status.WAITING; private final String name; @@ -47,6 +49,7 @@ public static enum Status { private long startTime; private long duration = 0; private boolean markSkip = false; + protected final Confs conf; public Task(String name, String id, boolean idempotent, MachineRuntime machine, ClusterStats clusterStats, TaskSubmitter submitter) { @@ -59,6 +62,7 @@ public Task(String name, String id, boolean idempotent, MachineRuntime machine, this.uuid = UUID.randomUUID().toString(); this.clusterStats = clusterStats; this.submitter = submitter; + this.conf = Confs.loadKaramelConfs(); } public void setDuration(long duration) { @@ -80,7 +84,11 @@ public String getMachineId() { public String getSshUser() { return sshUser; } - + + public DagTaskCallback getDagCallback() { + return dagCallback; + } + public String getName() { return name; } @@ -184,6 +192,10 @@ public void started() { dagCallback.started(); } + public void blocked() { + status = Status.BLOCKED; + } + @Override public void succeed() { status = Status.DONE; @@ -227,7 +239,9 @@ public void downloadExperimentResults(MachineInterface sshMachine) throws Karame public String getSudoCommand() { String password = ClusterService.getInstance().getCommonContext().getSudoAccountPassword(); - return (password == null || password.isEmpty()) ? "sudo" : "echo \"%password_hidden%\" | sudo -S "; + String sudoBinary = conf.getProperty(Settings.CHEF_SUDO_BINARY); + return (password == null || password.isEmpty()) ? sudoBinary : "echo \"%password_hidden%\" | " + + sudoBinary + " -S "; } private void addStats() { diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/VendorCookbookTask.java b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/VendorCookbookTask.java index 80a09359..345d74ed 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/VendorCookbookTask.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/running/model/tasks/VendorCookbookTask.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.backend.running.model.tasks; import java.io.IOException; @@ -12,55 +7,47 @@ import se.kth.karamel.backend.converter.ShellCommandBuilder; import se.kth.karamel.backend.machines.TaskSubmitter; import se.kth.karamel.backend.running.model.MachineRuntime; +import se.kth.karamel.common.clusterdef.Cookbook; +import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.stats.ClusterStats; import se.kth.karamel.common.util.Settings; -/** - * - * @author kamal - */ public class VendorCookbookTask extends Task { - private final String cookbookId; + private final String cookbookName; private final String cookbooksHome; - private final String githubRepoName; private final String githubRepoUrl; - private final String subCookbookName; private final String branch; - public VendorCookbookTask(MachineRuntime machine, ClusterStats clusterStats, TaskSubmitter submitter, - String cookbookId, String cookbooksHome, String githubRepoUrl, String githubRepoName, String subCookbookName, - String branch) { - super("clone and vendor " + ((subCookbookName == null) ? githubRepoName : subCookbookName), - "clone and vendor " + cookbookId, true, machine, + public VendorCookbookTask(MachineRuntime machine, ClusterStats clusterStats, TaskSubmitter submitter, + String cookbooksHome, String cookbookName, Cookbook cookbook) + throws KaramelException { + super("clone and vendor " + cookbookName, + "clone and vendor " + cookbookName, true, machine, clusterStats, submitter); - this.cookbookId = cookbookId; + this.cookbookName = cookbookName; this.cookbooksHome = cookbooksHome; - this.githubRepoName = githubRepoName; - this.githubRepoUrl = githubRepoUrl; - this.subCookbookName = subCookbookName; - this.branch = branch; + this.githubRepoUrl = Settings.GITHUB_BASE_URL + "/" + cookbook.getGithub(); + this.branch = cookbook.getBranch(); } @Override public List getCommands() throws IOException { - String cookbookPath = githubRepoName; - if (subCookbookName != null && !subCookbookName.isEmpty()) { - cookbookPath += Settings.SLASH + subCookbookName; - } + boolean airgap = Boolean.parseBoolean(conf.getProperty(Settings.KARAMEL_AIRGAP)); if (commands == null) { commands = ShellCommandBuilder.makeSingleFileCommand(Settings.SCRIPT_PATH_CLONE_VENDOR_COOKBOOK, "cookbooks_home", cookbooksHome, - "github_repo_name", githubRepoName, - "cookbook_path", cookbookPath, + "github_repo_name", cookbookName, + "cookbook_path", cookbookName, "github_repo_url", githubRepoUrl, "branch_name", branch, - "vendor_path", Settings.REMOTE_COOKBOOK_VENDOR_PATH(getSshUser(), githubRepoName), + "vendor_path", Settings.REMOTE_COOKBOOK_VENDOR_PATH(getSshUser(), cookbookName), "sudo_command", getSudoCommand(), "task_id", getId(), "install_dir_path", Settings.REMOTE_INSTALL_DIR_PATH(getSshUser()), "succeedtasks_filepath", Settings.SUCCEED_TASKLIST_FILENAME, - "pid_file", Settings.PID_FILE_NAME); + "pid_file", Settings.PID_FILE_NAME, + "is_airgap", String.valueOf(airgap)); } return commands; } @@ -71,7 +58,7 @@ public static String makeUniqueId(String machineId, String cookbookId) { @Override public String uniqueId() { - return makeUniqueId(super.getMachineId(), cookbookId); + return makeUniqueId(super.getMachineId(), cookbookName); } @Override diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/stats/ClusterStatistics.java b/karamel-core/src/main/java/se/kth/karamel/backend/stats/ClusterStatistics.java index c5067fc9..36ce84f2 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/stats/ClusterStatistics.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/stats/ClusterStatistics.java @@ -14,7 +14,7 @@ */ public class ClusterStatistics { - public static final String DEFAULT_TIME_STAT_FILE_DIR = Settings.KARAMEL_ROOT_PATH; + public static final String DEFAULT_TIME_STAT_FILE_DIR = Settings.getKaramelRootPath(); private static String fileName = null; private static FileWriter writer = null; private static String experimentName = ""; diff --git a/karamel-core/src/main/java/se/kth/karamel/backend/stats/KaramelEvaluation.java b/karamel-core/src/main/java/se/kth/karamel/backend/stats/KaramelEvaluation.java index 240b0774..737fbb9c 100644 --- a/karamel-core/src/main/java/se/kth/karamel/backend/stats/KaramelEvaluation.java +++ b/karamel-core/src/main/java/se/kth/karamel/backend/stats/KaramelEvaluation.java @@ -37,7 +37,7 @@ public static void main(String[] args) throws IOException, KaramelException, Int sshKeys = api.generateSshKeysAndUpdateConf(clusterName); } api.registerSshKeys(sshKeys); - api.updateGceCredentialsIfValid(Settings.KARAMEL_ROOT_PATH + "/gce-key.json"); + api.updateGceCredentialsIfValid(Settings.getKaramelRootPath() + "/gce-key.json"); KaramelEvaluation evaluation = new KaramelEvaluation(); diff --git a/karamel-core/src/main/java/se/kth/karamel/client/api/CookbookCacheIml.java b/karamel-core/src/main/java/se/kth/karamel/client/api/CookbookCacheIml.java index 85259634..e329614c 100644 --- a/karamel-core/src/main/java/se/kth/karamel/client/api/CookbookCacheIml.java +++ b/karamel-core/src/main/java/se/kth/karamel/client/api/CookbookCacheIml.java @@ -1,38 +1,42 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -/** - * It caches cookbooks' metadata that being read from Github - */ package se.kth.karamel.client.api; +import com.google.common.base.Charsets; import org.apache.log4j.Logger; -import se.kth.karamel.backend.ClusterService; -import se.kth.karamel.backend.dag.Dag; -import se.kth.karamel.backend.dag.DagNode; import se.kth.karamel.common.clusterdef.Cookbook; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; -import se.kth.karamel.common.clusterdef.json.JsonGroup; import se.kth.karamel.common.clusterdef.yaml.YamlCluster; -import se.kth.karamel.common.cookbookmeta.CookbookUrls; +import se.kth.karamel.common.cookbookmeta.KaramelFile; +import se.kth.karamel.common.cookbookmeta.MetadataParser; +import se.kth.karamel.common.cookbookmeta.MetadataRb; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.cookbookmeta.CookbookCache; +import se.kth.karamel.common.exception.MetadataParseException; import se.kth.karamel.common.exception.NoKaramelizedCookbookException; -import se.kth.karamel.common.util.Settings; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import se.kth.karamel.common.util.ProcOutputConsumer; +import se.kth.karamel.common.util.Settings; /** * @@ -45,234 +49,108 @@ public class CookbookCacheIml implements CookbookCache { public Map cookbooks = new HashMap<>(); public Set problematics = new HashSet<>(); - @Override - public KaramelizedCookbook readNew(String cookbookUrl) throws KaramelException { - if (problematics.contains(cookbookUrl)) { - problematics.remove(cookbookUrl); - } - try { - KaramelizedCookbook cookbook = new KaramelizedCookbook(cookbookUrl, false); - cookbooks.put(cookbookUrl, cookbook); - return cookbook; - } catch (Exception e) { - problematics.add(cookbookUrl); - throw new NoKaramelizedCookbookException( - String.format("Cookbook should problem in the metadata '%s'", cookbookUrl), e); - } - } + private ExecutorService es = Executors.newFixedThreadPool(2); @Override - public KaramelizedCookbook get(String cookbookUrl) throws KaramelException { - if (!problematics.contains(cookbookUrl)) { - KaramelizedCookbook cb = cookbooks.get(cookbookUrl); + public KaramelizedCookbook get(String cookbookName) throws KaramelException { + if (!problematics.contains(cookbookName)) { + KaramelizedCookbook cb = cookbooks.get(cookbookName); if (cb == null) { - cb = readNew(cookbookUrl); + throw new NoKaramelizedCookbookException( + String.format("Cookbook could not be found '%s'", cookbookName)); } return cb; } else { - throw new NoKaramelizedCookbookException(String.format("Cookbook has problem in the metadata '%s'", cookbookUrl)); - } - } - - @Override - public synchronized void prepareParallel(Set urlsToPrepare) throws KaramelException { - Set allUrls = new HashSet<>(); - for (String cookbookUrl : urlsToPrepare) { - KaramelizedCookbook cb = cookbooks.get(cookbookUrl); - if (cb == null) { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - CookbookUrls urls = builder.url(cookbookUrl).build(); - allUrls.add(urls.attrFile); - allUrls.add(urls.metadataFile); - allUrls.add(urls.karamelFile); - allUrls.add(urls.berksFile); - } - } - Map contents = IoUtils.readContentParallel(allUrls, ClusterService.SHARED_GLOBAL_TP); - for (String cookbookUrl : urlsToPrepare) { - KaramelizedCookbook cb = cookbooks.get(cookbookUrl); - if (cb == null) { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - CookbookUrls urls = builder.url(cookbookUrl).build(); - if (contents.containsKey(urls.attrFile) - && contents.containsKey(urls.metadataFile) - && contents.containsKey(urls.karamelFile) - && contents.containsKey(urls.berksFile)) { - KaramelizedCookbook kc = new KaramelizedCookbook(urls, contents.get(urls.attrFile), - contents.get(urls.metadataFile), contents.get(urls.karamelFile), contents.get(urls.berksFile)); - cookbooks.put(cookbookUrl, kc); - } else { - problematics.add(cookbookUrl); - } - } - } - } - - @Override - public void prepareNewParallel(Set cookbookUrls) throws KaramelException { - Set allUrls = new HashSet<>(); - for (String cookbookUrl : cookbookUrls) { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - CookbookUrls urls = builder.url(cookbookUrl).build(); - allUrls.add(urls.attrFile); - allUrls.add(urls.metadataFile); - allUrls.add(urls.karamelFile); - allUrls.add(urls.berksFile); - } - Map contents = IoUtils.readContentParallel(allUrls, ClusterService.SHARED_GLOBAL_TP); - for (String cookbookUrl : cookbookUrls) { - CookbookUrls.Builder builder = new CookbookUrls.Builder(); - CookbookUrls urls = builder.url(cookbookUrl).build(); - if (contents.containsKey(urls.attrFile) - && contents.containsKey(urls.metadataFile) - && contents.containsKey(urls.karamelFile) - && contents.containsKey(urls.berksFile)) { - KaramelizedCookbook kc = new KaramelizedCookbook(urls, contents.get(urls.attrFile), - contents.get(urls.metadataFile), contents.get(urls.karamelFile), contents.get(urls.berksFile)); - cookbooks.put(cookbookUrl, kc); - } - } - } - - @Override - public List loadRootKaramelizedCookbooks(JsonCluster jsonCluster) throws KaramelException { - List jsonGroups = jsonCluster.getGroups(); - Set toLoad = new HashSet<>(); - for (JsonGroup jsonGroup : jsonGroups) { - for (JsonCookbook cb : jsonGroup.getCookbooks()) { - toLoad.add(cb.getId()); - } - } - for (JsonCookbook cb : jsonCluster.getCookbooks()) { - toLoad.add(cb.getId()); - } - Dag dag = new Dag(); - List kcbs = loadAllKaramelizedCookbooks(jsonCluster.getName(), toLoad, dag); - List roots = new ArrayList<>(); - for (DagNode node : dag.findRootNodes()) { - String id = node.getId(); - boolean found = false; - for (KaramelizedCookbook kcb : kcbs) { - if (kcb.getUrls().id.equals(id)) { - roots.add(kcb); - found = true; - break; - } - } - if (!found) { - throw new NoKaramelizedCookbookException("Could not load a root cookbook, " - + "make sure it is correctly karamelized " + id); - } + throw new NoKaramelizedCookbookException( + String.format("Cookbook has problem in the metadata '%s'", cookbookName)); } - return roots; } @Override public List loadAllKaramelizedCookbooks(JsonCluster jsonCluster) throws KaramelException { - List jsonGroups = jsonCluster.getGroups(); - Set toLoad = new HashSet<>(); - for (JsonGroup jsonGroup : jsonGroups) { - for (JsonCookbook cb : jsonGroup.getCookbooks()) { - toLoad.add(cb.getId()); - } - } - for (JsonCookbook cb : jsonCluster.getCookbooks()) { - toLoad.add(cb.getId()); + if (!cookbooks.isEmpty()) { + return new ArrayList<>(cookbooks.values()); } - return loadAllKaramelizedCookbooks(jsonCluster.getName(), toLoad, new Dag()); + + cloneAndVendorCookbooks(jsonCluster.getRootCookbooks()); + buildCookbookObjects(); + return new ArrayList<>(cookbooks.values()); } @Override public List loadAllKaramelizedCookbooks(YamlCluster cluster) throws KaramelException { - Set toLoad = new HashSet<>(); - for (Cookbook cb : cluster.getCookbooks().values()) { - toLoad.add(cb.getUrls().id); + if (!cookbooks.isEmpty()) { + return new ArrayList<>(cookbooks.values()); } - Dag dag = new Dag(); - // An hacky fix for a crappy code. The thing is that, where this function is called, I'd like to have - // also the attributes of dependencies (including the transient ones) for the filtering of valid attributes. - // So here we build a Topological sort of the dependency graph, reverse it, and traverse it. - // For each KaramelizedCookbook we add in a set all the *references* to the dependency, both direct and transient - // that's the reason of using the topological sort. - // As we are using references to the same set of KaramelizedCookbooks, the default methods for equals and hashcode - // (comparing addresses) works fine. + if (!Settings.USE_CLONED_REPO_FILES) { + cloneAndVendorCookbooks(cluster.getCookbooks()); + } - // Result ignored on purpose - loadAllKaramelizedCookbooks(cluster.getName(), toLoad, dag); - List topologicalSort = new ArrayList<>(); + buildCookbookObjects(); + return new ArrayList<>(cookbooks.values()); + } - Set rootNodes = dag.findRootNodes(); - while (!rootNodes.isEmpty()) { - for (DagNode dagNode : rootNodes) { - KaramelizedCookbook kcb = cookbooks.get(dagNode.getId()); - if (kcb != null) { - topologicalSort.add(cookbooks.get(dagNode.getId())); + private void cloneAndVendorCookbooks(Map toClone) throws KaramelException { + File workingDir = Paths.get(Settings.WORKING_DIR).toFile(); + if (!workingDir.exists()) { + workingDir.mkdir(); + } + + for (Map.Entry cb : toClone.entrySet()) { + // Clone the repository + try { + if (!Paths.get(Settings.WORKING_DIR, cb.getKey()).toFile().exists()) { + Git.cloneRepository() + // TODO(Fabio): make base url as setting in the cluster definition + // So we can support also GitLab/Bitbucket and so on. + .setURI(Settings.GITHUB_BASE_URL + "/" + cb.getValue().getGithub()) + .setBranch(cb.getValue().getBranch()) + .setDirectory(Paths.get(Settings.WORKING_DIR, cb.getKey()).toFile()) + .call(); } - dag.removeNode(dagNode); + // TODO(Fabio) try to update the branch + } catch (GitAPIException e) { + throw new KaramelException(e); } - rootNodes = dag.findRootNodes(); - } - // Reverse the list - Collections.reverse(topologicalSort); + // Vendor the repository + try { + Process vendorProcess = Runtime.getRuntime().exec("berks vendor --berksfile=" + + Paths.get(Settings.WORKING_DIR, cb.getKey(), "Berksfile") + " " + Settings.WORKING_DIR); + Future vendorOutput = es.submit(new ProcOutputConsumer(vendorProcess.getInputStream())); + vendorProcess.waitFor(10, TimeUnit.MINUTES); - for (KaramelizedCookbook kcb : topologicalSort) { - Map dependencies = kcb.getBerksFile().getDeps(); - for (Cookbook cookbook : dependencies.values()) { - KaramelizedCookbook depKbc = cookbooks.get(cookbook.getUrls().id); - if (depKbc != null) { - kcb.addDependency(depKbc); - kcb.addDependencies(depKbc.getDependencies()); + if (vendorProcess.exitValue() != 0) { + throw new KaramelException("Fail to vendor the cookbook: " + cb.getKey() + " " + vendorOutput.get()); } + } catch (IOException | InterruptedException | ExecutionException e) { + throw new KaramelException(e); } } - - return topologicalSort; } - private List loadAllKaramelizedCookbooks(String clusterName, Set toLoad, Dag dag) - throws KaramelException { - Set loaded = new HashSet<>(); - List all = new ArrayList<>(); - - int level = 0; - - while (!toLoad.isEmpty()) { - logger.info(String.format("%d-level cookbooks for %s is %d", level++, clusterName, toLoad.size())); - prepareParallel(toLoad); - for (String tl : toLoad) { - dag.addNode(tl); - if (problematics.contains(tl)) { - dag.updateLabel(tl, "NON_KARAMELIZED"); - } else { - dag.updateLabel(tl, "OK"); - } - } - toLoad.removeAll(problematics); - Set depsToLoad = new HashSet(); - for (String cbid : toLoad) { - KaramelizedCookbook kc = get(cbid); - all.add(kc); - if (!Settings.CB_CLASSPATH_MODE) { - Map deps = kc.getBerksFile().getDeps(); - for (Cookbook cb : deps.values()) { - String depId = cb.getUrls().id; - dag.addDependency(cbid, depId); - if (!loaded.contains(depId) && !problematics.contains(depId)) { - depsToLoad.add(depId); + private void buildCookbookObjects() throws KaramelException { + try (Stream paths = Files.find(Paths.get(Settings.WORKING_DIR), + Integer.MAX_VALUE, (path, attributes) -> attributes.isDirectory())) { + paths.forEach(path -> { + File rawKaramelFile = path.resolve("Karamelfile").toFile(); + File rawMetadataRb = path.resolve("metadata.rb").toFile(); + if (rawKaramelFile.exists()) { + try { + KaramelFile karamelFile = new KaramelFile(com.google.common.io.Files.toString( + rawKaramelFile, Charsets.UTF_8)); + MetadataRb metadataRb = MetadataParser.parse(com.google.common.io.Files.toString( + rawMetadataRb, Charsets.UTF_8)); + cookbooks.put(metadataRb.getName(), new KaramelizedCookbook(metadataRb, karamelFile)); + } catch (IOException | MetadataParseException e) { + logger.error(e); + throw new RuntimeException(e); } } - } - } - loaded.addAll(toLoad); - toLoad.clear(); - toLoad.addAll(depsToLoad); - + }); + } catch (IOException e) { + throw new KaramelException(e); } - logger.info(String.format("################## COOKBOOK TRANSIENT DEPENDENCIES FOR %s ############", clusterName)); - logger.info(dag.print()); - logger.info("############################################################################"); - return all; } } diff --git a/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApi.java b/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApi.java index f6ba5780..719137dd 100644 --- a/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApi.java +++ b/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApi.java @@ -1,27 +1,16 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.client.api; -import se.kth.karamel.backend.Experiment; import se.kth.karamel.backend.command.CommandResponse; -import se.kth.karamel.backend.github.GithubUser; -import se.kth.karamel.backend.github.OrgItem; -import se.kth.karamel.backend.github.RepoItem; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.util.Ec2Credentials; import se.kth.karamel.common.util.OcciCredentials; import se.kth.karamel.common.util.NovaCredentials; import se.kth.karamel.common.util.SshKeyPair; -import java.util.List; /** * The main API of Karamel-Core for Karamel clients * - * @author kamal */ public interface KaramelApi { @@ -31,7 +20,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public String commandCheatSheet() throws KaramelException; + String commandCheatSheet() throws KaramelException; /** * Parses the command, if valid fetches the result in string, result could have different formatting depends on the @@ -42,17 +31,16 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public CommandResponse processCommand(String command, String... args) throws KaramelException; + CommandResponse processCommand(String command, String... args) throws KaramelException; /** * Returns visible recipes and attributes of the cookbook with their detail as a json file * * @param cookbookUrl - * @param refresh * @return * @throws KaramelException */ - public String getCookbookDetails(String cookbookUrl, boolean refresh) throws KaramelException; + String getCookbookDetails(String cookbookUrl) throws KaramelException; /** * Converts json definition of the cluster into a yaml object @@ -61,7 +49,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public String jsonToYaml(String json) throws KaramelException; + String jsonToYaml(String json) throws KaramelException; /** * Converts yaml definition of the cluster into the json @@ -70,7 +58,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public String yamlToJson(String yaml) throws KaramelException; + String yamlToJson(String yaml) throws KaramelException; /** * Loads Karamel common keys @@ -78,7 +66,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair loadSshKeysIfExist() throws KaramelException; + SshKeyPair loadSshKeysIfExist() throws KaramelException; /** * Loads cluster specific keys @@ -87,7 +75,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair loadSshKeysIfExist(String clusterName) throws KaramelException; + SshKeyPair loadSshKeysIfExist(String clusterName) throws KaramelException; /** * Generates a common ssh keys in the karamel folder @@ -95,7 +83,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair generateSshKeysAndUpdateConf() throws KaramelException; + SshKeyPair generateSshKeysAndUpdateConf() throws KaramelException; /** * Generates cluster specific ssh keys @@ -104,7 +92,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair generateSshKeysAndUpdateConf(String clusterName) throws KaramelException; + SshKeyPair generateSshKeysAndUpdateConf(String clusterName) throws KaramelException; /** * Register ssh keys for the current runtime of karamel @@ -113,7 +101,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair registerSshKeys(SshKeyPair keypair) throws KaramelException; + SshKeyPair registerSshKeys(SshKeyPair keypair) throws KaramelException; /** * Register ssh keys for the specified cluster @@ -123,7 +111,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public SshKeyPair registerSshKeys(String clusterName, SshKeyPair keypair) throws KaramelException; + SshKeyPair registerSshKeys(String clusterName, SshKeyPair keypair) throws KaramelException; /** * Reads it from default karamel conf file @@ -131,7 +119,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public Ec2Credentials loadEc2CredentialsIfExist() throws KaramelException; + Ec2Credentials loadEc2CredentialsIfExist() throws KaramelException; /** * Validates user's credentials before starting the cluster @@ -140,7 +128,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public boolean updateEc2CredentialsIfValid(Ec2Credentials credentials) throws KaramelException; + boolean updateEc2CredentialsIfValid(Ec2Credentials credentials) throws KaramelException; /** * Starts running the cluster by launching machines and installing softwares It expect to receive a complete @@ -149,7 +137,7 @@ public interface KaramelApi { * @param json * @throws KaramelException */ - public void startCluster(String json) throws KaramelException; + void startCluster(String json) throws KaramelException; /** * In case user wants to pause the running cluster for inspection reasons. It implies that machines won't receive any @@ -158,7 +146,7 @@ public interface KaramelApi { * @param clusterName * @throws KaramelException */ - public void pauseCluster(String clusterName) throws KaramelException; + void pauseCluster(String clusterName) throws KaramelException; /** * It resumes an already paused cluster, machines will go on and run ssh commands. @@ -166,7 +154,7 @@ public interface KaramelApi { * @param clusterName * @throws KaramelException */ - public void resumeCluster(String clusterName) throws KaramelException; + void resumeCluster(String clusterName) throws KaramelException; /** * It stops sending new ssh command to machines, destroys the automatic allocated machines and disconnects ssh clients @@ -175,7 +163,7 @@ public interface KaramelApi { * @param clusterName * @throws KaramelException */ - public void terminateCluster(String clusterName) throws KaramelException; + void terminateCluster(String clusterName) throws KaramelException; /** * Returns a json containing all groups, machines, their status, tasks/commands and their status. @@ -184,7 +172,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public String getClusterStatus(String clusterName) throws KaramelException; + String getClusterStatus(String clusterName) throws KaramelException; /** * Returns installation flow DAG that each node is a task assigned to a certain machine with the current status of the @@ -194,7 +182,7 @@ public interface KaramelApi { * @return * @throws KaramelException */ - public String getInstallationDag(String clusterName) throws KaramelException; + String getInstallationDag(String clusterName) throws KaramelException; /** * Register password for Baremetal sudo account @@ -202,110 +190,21 @@ public interface KaramelApi { * @param password * @throws KaramelException */ - public void registerSudoPassword(String password) throws KaramelException; + void registerSudoPassword(String password) throws KaramelException; - /** - * Register username/password for github account - * - * @param user github account name - * @param password github password - * @return GithubUser Json object also containing primary github email address - * @throws KaramelException - */ - public GithubUser registerGithubAccount(String user, String password) throws KaramelException; + String loadGceCredentialsIfExist() throws KaramelException; - /** - * Load any existing credentials stored locally - * - * @return GithubUser object - * @throws KaramelException - */ - public GithubUser loadGithubCredentials() throws KaramelException; - - /** - * Lists the available repos in a github organization. - * - * @param organization - * @return List of available repos - * @throws KaramelException - */ - public List listGithubRepos(String organization) throws KaramelException; - - /** - * Lists the available organizations for a user in github. Must call 'registerGithubAccount' first. - * - * @return List of available orgs - * @throws KaramelException - */ - public List listGithubOrganizations() throws KaramelException; - -// /** -// * Create a new github repo in an organization -// * -// * @param organization if organization is empty or null, create the repo for the authenticated user -// * @param repo the name of the repo to create -// * @param description of what's in the repository -// * @throws KaramelException -// */ -// public void createGithubRepo(String organization, String repo, String description) throws KaramelException; - /** - * Add a file to an existing repo, commit it, and push it to github. - * - * @param experiment bash scripts and config files to add, commit, and push. - * @throws KaramelException - */ - public void commitAndPushExperiment(Experiment experiment) - throws KaramelException; - - /** - * Loads an experiment into the Designer, given its clone URL - * - * @param githubRepoUrl url for github repo - * @return Json object for the ExperimentContext - * @throws se.kth.karamel.common.exception.KaramelException - */ - public Experiment loadExperiment(String githubRepoUrl) throws KaramelException; - - /** - * - * @param org github org name - * @param repo github repo name - * @param description repo description - * @return RepoItem bean/json containing name, description of repo. - * @throws KaramelException - */ - public RepoItem createGithubRepo(String org, String repo, String description) throws KaramelException; - - /** - * - * @param owner - * @param repo - * @param experimentName - */ - public void removeFileFromExperiment(String owner, String repo, String experimentName); - - /** - * - * @param owner - * @param repo - * @param removeGitHub - * @param removeLocal - */ - public void removeRepo(String owner, String repo, boolean removeLocal, boolean removeGitHub) throws KaramelException; + boolean updateGceCredentialsIfValid(String jsonFilePath) throws KaramelException; - public String loadGceCredentialsIfExist() throws KaramelException; + NovaCredentials loadNovaCredentialsIfExist() throws KaramelException; - public boolean updateGceCredentialsIfValid(String jsonFilePath) throws KaramelException; + NovaCredentials loadNovaV3CredentialsIfExist() throws KaramelException; - public NovaCredentials loadNovaCredentialsIfExist() throws KaramelException; - - public NovaCredentials loadNovaV3CredentialsIfExist() throws KaramelException; + boolean updateNovaCredentialsIfValid(NovaCredentials credentials) throws KaramelException; - public boolean updateNovaCredentialsIfValid(NovaCredentials credentials) throws KaramelException; - - public boolean updateNovaV3CredentialsIfValid(NovaCredentials credentials) throws KaramelException; + boolean updateNovaV3CredentialsIfValid(NovaCredentials credentials) throws KaramelException; - public OcciCredentials loadOcciCredentialsIfExist() throws KaramelException; + OcciCredentials loadOcciCredentialsIfExist() throws KaramelException; - public boolean updateOcciCredentialsIfValid(OcciCredentials credentials) throws KaramelException; + boolean updateOcciCredentialsIfValid(OcciCredentials credentials) throws KaramelException; } diff --git a/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApiImpl.java b/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApiImpl.java index cb6f44f3..bc9069c6 100644 --- a/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApiImpl.java +++ b/karamel-core/src/main/java/se/kth/karamel/client/api/KaramelApiImpl.java @@ -2,21 +2,13 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.commons.io.FileUtils; import org.jclouds.ContextBuilder; import org.jclouds.domain.Credentials; import org.jclouds.openstack.nova.v2_0.NovaApiMetadata; import se.kth.karamel.backend.ClusterDefinitionService; import se.kth.karamel.backend.ClusterService; -import se.kth.karamel.backend.Experiment; import se.kth.karamel.backend.command.CommandResponse; import se.kth.karamel.backend.command.CommandService; -import se.kth.karamel.backend.github.GithubApi; -import se.kth.karamel.backend.github.GithubUser; -import se.kth.karamel.backend.github.OrgItem; -import se.kth.karamel.backend.github.RepoItem; -import se.kth.karamel.backend.github.util.ChefExperimentExtractor; -import se.kth.karamel.backend.github.util.GithubUrl; import se.kth.karamel.backend.launcher.amazon.Ec2Context; import se.kth.karamel.backend.launcher.amazon.Ec2Launcher; import se.kth.karamel.backend.launcher.google.GceContext; @@ -41,12 +33,6 @@ import se.kth.karamel.backend.running.model.tasks.RunRecipeTask; import se.kth.karamel.backend.running.model.tasks.ShellCommand; import se.kth.karamel.backend.running.model.tasks.VendorCookbookTask; -import se.kth.karamel.common.cookbookmeta.Berksfile; -import se.kth.karamel.common.cookbookmeta.DefaultRb; -import se.kth.karamel.common.cookbookmeta.ExperimentRecipe; -import se.kth.karamel.common.cookbookmeta.InstallRecipe; -import se.kth.karamel.common.cookbookmeta.KaramelFile; -import se.kth.karamel.common.cookbookmeta.KaramelFileYamlDeps; import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook; import se.kth.karamel.common.exception.InvalidNovaCredentialsException; import se.kth.karamel.common.exception.InvalidOcciCredentialsException; @@ -60,12 +46,6 @@ import se.kth.karamel.common.util.SshKeyService; import se.kth.karamel.common.util.settings.NovaSetting; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import se.kth.karamel.common.cookbookmeta.CookbookCache; /** @@ -90,19 +70,10 @@ public CommandResponse processCommand(String command, String... args) throws Kar } @Override - public String getCookbookDetails(String cookbookUrl, boolean refresh) throws KaramelException { - Set urls = new HashSet<>(); - urls.add(cookbookUrl); + public String getCookbookDetails(String cookbookName) throws KaramelException { CookbookCache cache = ClusterDefinitionService.CACHE; - if (refresh) { - cache.prepareNewParallel(urls); - KaramelizedCookbook cb = cache.get(cookbookUrl); - return cb.getInfoJson(); - } else { - cache.prepareParallel(urls); - KaramelizedCookbook cb = cache.get(cookbookUrl); - return cb.getInfoJson(); - } + KaramelizedCookbook cb = cache.get(cookbookName); + return cb.getInfoJson(); } @Override @@ -343,191 +314,4 @@ public SshKeyPair registerSshKeys(String clusterName, SshKeyPair keypair) throws public void registerSudoPassword(String password) { ClusterService.getInstance().getCommonContext().setSudoAccountPassword(password); } - - @Override - public List listGithubOrganizations() throws KaramelException { - return GithubApi.getOrganizations(); - } - - @Override - public List listGithubRepos(String organization) throws KaramelException { - return GithubApi.getRepos(organization); - } - - @Override - public GithubUser registerGithubAccount(String user, String password) throws KaramelException { - return GithubApi.registerCredentials(user, password); - } - - @Override - public GithubUser loadGithubCredentials() throws KaramelException { - return GithubApi.loadGithubCredentials(); - } - - private void initGithubRepo(String user, String owner, String repo, String description) throws KaramelException { - if (owner == null || owner.isEmpty() || owner.compareToIgnoreCase(user) == 0) { - GithubApi.createRepoForUser(repo, description); - } else { - GithubApi.createRepoForOrg(owner, repo, description); - } - } - - @Override - public void commitAndPushExperiment(Experiment experiment) throws KaramelException { - String owner = experiment.getGithubOwner(); - String repoName = experiment.getGithubRepo(); - File f = GithubApi.getRepoDirectory(repoName); - boolean repoExists = GithubApi.repoExists(owner, repoName); - if (repoExists) { - // local copy must exist, already pushed to GitHub - if (!f.exists()) { - throw new KaramelException("Remote repository already exists. Load the experiment if it already exists."); - } - } else // no repo on GitHub. Should not exist a local directory with same repo name. - { - if (f.exists()) { - throw new KaramelException("The remote repo does not exist, however a conflicting local directory was found. " - + "Remove the local directory in ~/.karamel/cookbook_designer first, then save again."); - } else { - // Create the repo if it doesn't exist and clone it to a local directory - // That way, the local directory will only ever exist if it the repo has been created first. - // Users should subsequently load a directory from GitHub. - initGithubRepo(GithubApi.getUser(), owner, repoName, experiment.getDescription()); - // Scaffold a new experiment project with Karamel/Chef - GithubApi.scaffoldRepo(repoName); - } - } - - // For all config and script files, compile them and generate Karamel/Chef files - ChefExperimentExtractor.parseAttributesAddToGit(owner, repoName, experiment); - - ChefExperimentExtractor.parseRecipesAddToGit(owner, repoName, experiment); - - // Commit and push all changes to github - GithubApi.commitPush(owner, repoName); - - } - - @Override - public Experiment loadExperiment(String githubRepoUrl) throws KaramelException { - Experiment ec = new Experiment(); - String repoName = GithubUrl.extractRepoName(githubRepoUrl); - ec.setGithubRepo(repoName); - String owner = GithubUrl.extractUserName(githubRepoUrl); - ec.setGithubOwner(owner); - if (repoName == null || owner == null || repoName.isEmpty() || owner.isEmpty()) { - throw new KaramelException("Misformed url repo/owner: " + githubRepoUrl); - } - - // Loading a repo involves wiping any existing local copy and replacing it with GitHub's copy. - File localPath = new File(Settings.COOKBOOKS_PATH + File.separator + repoName); - if (localPath.isDirectory() == true) { - try { - FileUtils.deleteDirectory(localPath); - } catch (IOException ex) { - logger.warn(ex.getMessage()); - if (localPath.isDirectory() == true) { - throw new KaramelException("Couldn't remove local copy of the repo at directory: " + localPath.getPath()); - } - } - } - // Download the latest copy from GitHub - GithubApi.cloneRepo(owner, repoName); - String strippedUrl = githubRepoUrl.replaceAll("\\.git", ""); - ec.setUrlGitClone(githubRepoUrl); - - KaramelizedCookbook kc = new KaramelizedCookbook(strippedUrl, true); - KaramelFile kf = kc.getKaramelFile(); - Berksfile bf = kc.getBerksFile(); - DefaultRb attributes = kc.getDefaultRb(); - List er = kc.getExperimentRecipes(); - InstallRecipe ir = kc.getInstallRecipe(); - - ec.setUser((String) attributes.getValue(repoName + "/user")); - ec.setGroup((String) attributes.getValue(repoName + "/group")); - ec.setUrlBinary((String) attributes.getValue(repoName + "/url")); - ec.setBerksfile(bf.toString()); - ec.setExperimentSetupCode(ir.getSetupCode()); - ec.setDefaultAttributes(attributes.getExperimentContextFormat()); - ArrayList deps = kf.getDependencies(); - Set localSet = new HashSet<>(); - Set globalSet = new HashSet<>(); - for (KaramelFileYamlDeps yd : deps) { - if (!yd.getRecipe().contains(Settings.COOKBOOK_DELIMITER + "install")) { - List locals = yd.getLocal(); - // remove duplicates from locals - localSet.addAll(locals); - List globals = yd.getGlobal(); - globalSet.addAll(globals); - - } - } - StringBuilder local = new StringBuilder(); - StringBuilder global = new StringBuilder(); - int i = 0; - for (String s : localSet) { - if (i == 0) { - local.append(s); - } else { - local.append(System.lineSeparator()).append(s); - } - i++; - } - i = 0; - for (String s : globalSet) { - if (i == 0) { - global.append(s); - } else { - global.append(System.lineSeparator()).append(s); - } - i++; - } - - ec.setLocalDependencies(local.toString()); - ec.setGlobalDependencies(global.toString()); - for (ExperimentRecipe r : er) { - Experiment.Code exp = new Experiment.Code(r.getRecipeName(), r.getScriptContents(), r.getConfigFileName(), - r.getConfigFileContents(), r.getScriptType()); - List exps = ec.getCode(); - exps.add(exp); - } - return ec; - } - - @Override - public RepoItem createGithubRepo(String org, String repo, String description) throws KaramelException { - return GithubApi.createRepoForOrg(org, repo, description); - } - - @Override - public void removeFileFromExperiment(String owner, String repo, String experimentName) { - try { - GithubApi.removeFile(owner, repo, experimentName); - } catch (KaramelException ex) { - // Do nothing - Repository hasn't been created yet. That's ok."); - } - } - - @Override - public void removeRepo(String owner, String repo, boolean removeLocal, boolean removeGitHub) throws KaramelException { - boolean failedLocal = false; - String failedMsg = ""; - - // if failure while removing locally, still try and remove remote repo (if requested) - if (removeLocal) { - try { - GithubApi.removeLocalRepo(owner, repo); - } catch (KaramelException ex) { - failedLocal = true; - failedMsg = ex.toString(); - } - } - if (removeGitHub) { - GithubApi.removeRepo(owner, repo); - } - if (failedLocal) { - throw new KaramelException(failedMsg); - } - - } } diff --git a/karamel-core/src/main/java/se/kth/karamel/common/CookbookScaffolder.java b/karamel-core/src/main/java/se/kth/karamel/common/CookbookScaffolder.java index e1378a59..ca5e6987 100644 --- a/karamel-core/src/main/java/se/kth/karamel/common/CookbookScaffolder.java +++ b/karamel-core/src/main/java/se/kth/karamel/common/CookbookScaffolder.java @@ -1,7 +1,9 @@ package se.kth.karamel.common; +import com.google.common.io.Resources; +import org.apache.commons.io.Charsets; import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.util.IoUtils; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -44,7 +46,7 @@ public static boolean deleteRecursive(File path) throws FileNotFoundException { * @throws IOException */ public static void createFile(String path, String template, String name) throws IOException { - String script = IoUtils.readContentFromClasspath(template); + String script = Resources.toString(Resources.getResource(template), Charsets.UTF_8); script = script.replaceAll("%%NAME%%", name); String uid = System.getProperty("user.name"); script = script.replaceAll("%%USER%%", uid); diff --git a/karamel-core/src/main/java/se/kth/karamel/common/TextTable.java b/karamel-core/src/main/java/se/kth/karamel/common/TextTable.java index 24e21286..b99fa6f8 100644 --- a/karamel-core/src/main/java/se/kth/karamel/common/TextTable.java +++ b/karamel-core/src/main/java/se/kth/karamel/common/TextTable.java @@ -5,9 +5,10 @@ */ package se.kth.karamel.common; +import org.apache.commons.lang.StringUtils; + import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; /** * diff --git a/karamel-core/src/main/java/se/kth/karamel/core/clusterdef/ClusterDefinitionValidator.java b/karamel-core/src/main/java/se/kth/karamel/core/clusterdef/ClusterDefinitionValidator.java index d3b48853..b21fef50 100644 --- a/karamel-core/src/main/java/se/kth/karamel/core/clusterdef/ClusterDefinitionValidator.java +++ b/karamel-core/src/main/java/se/kth/karamel/core/clusterdef/ClusterDefinitionValidator.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package se.kth.karamel.core.clusterdef; import com.google.common.collect.Lists; @@ -10,7 +5,6 @@ import se.kth.karamel.common.clusterdef.Baremetal; import se.kth.karamel.common.clusterdef.Provider; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.json.JsonGroup; import se.kth.karamel.common.clusterdef.json.JsonRecipe; import se.kth.karamel.common.exception.ValidationException; @@ -36,16 +30,14 @@ public static void validate(JsonCluster cluster) throws ValidationException { String.format("Number of ip addresses is not equal to the group size %d != %d", s1, group.getSize())); } } - for (JsonCookbook jc : group.getCookbooks()) { - ArrayList recs = Lists.newArrayList(jc.getRecipes()); - for (int i = 0; i < recs.size(); i++) { - for (int j = i + 1; j < recs.size(); j++) { - if (recs.get(i).getCanonicalName().equals(recs.get(j).getCanonicalName())) - throw new ValidationException( - String.format("More than one %s in the group %s", recs.get(i).getCanonicalName(), group.getName())); - } - } + ArrayList recs = Lists.newArrayList(group.getRecipes()); + for (int i = 0; i < recs.size(); i++) { + for (int j = i + 1; j < recs.size(); j++) { + if (recs.get(i).getCanonicalName().equals(recs.get(j).getCanonicalName())) + throw new ValidationException( + String.format("More than one %s in the group %s", recs.get(i).getCanonicalName(), group.getName())); + } } } } diff --git a/karamel-core/src/main/resources/karamel.properties b/karamel-core/src/main/resources/karamel.properties new file mode 100644 index 00000000..41350975 --- /dev/null +++ b/karamel-core/src/main/resources/karamel.properties @@ -0,0 +1 @@ +karamel.version=${project.version} diff --git a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/aptget_essentials.sc b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/aptget_essentials.sc index f6389062..1c104fa2 100644 --- a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/aptget_essentials.sc +++ b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/aptget_essentials.sc @@ -8,21 +8,15 @@ if [ %osfamily% == "redhat" ] ; then %sudo_command% yum install git -y %sudo_command% yum install make -y %sudo_command% yum install wget -y -git config --global user.name %github_username% -git config --global http.sslVerify false -git config --global http.postBuffer 524288000 elif [ %osfamily% == "ubuntu" ] ; then %sudo_command% apt-get update -y %sudo_command% apt-get update -y %sudo_command% apt-get install -f -y --force-yes git %sudo_command% apt-get install -f -y --force-yes curl %sudo_command% apt-get install -f -y --force-yes make -git config --global user.name %github_username% -git config --global http.sslVerify false -git config --global http.postBuffer 524288000 else echo "Unrecognized version of linux. Not ubuntu or redhat family." exit 1 fi echo '%task_id%' >> %succeedtasks_filepath% -' > aptget.sh ; chmod +x aptget.sh ; ./aptget.sh \ No newline at end of file +' > aptget.sh ; chmod +x aptget.sh ; ./aptget.sh diff --git a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/clone_vendor_cookbook.sb b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/clone_vendor_cookbook.sb index a4505331..ee475b58 100644 --- a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/clone_vendor_cookbook.sb +++ b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/clone_vendor_cookbook.sb @@ -1,14 +1,19 @@ set -e; mkdir -p %install_dir_path% ; cd %install_dir_path%; echo $$ > %pid_file%; echo '#!/bin/bash set -e -export LC_CTYPE=en_US.UTF-8 -# %sudo% chown -R $USER ~/.berkshelf -mkdir -p %cookbooks_home% -cd %cookbooks_home% -rm -fr %github_repo_name%* -git clone %github_repo_url% -cd %github_repo_name% -git checkout %branch_name% -berks vendor %vendor_path% -cd %install_dir_path% -echo '%task_id%' >> %succeedtasks_filepath% +IS_AIRGAP=%is_airgap% +%sudo_command% rm -rf %cookbooks_home%/nodes + +if [ "$IS_AIRGAP" == "false" ]; then + export LC_CTYPE=en_US.UTF-8 + # %sudo% chown -R $USER ~/.berkshelf + mkdir -p %cookbooks_home% + cd %cookbooks_home% + rm -fr %github_repo_name%* + git clone %github_repo_url% + cd %github_repo_name% + git checkout %branch_name% + berks vendor %vendor_path% + cd %install_dir_path% + echo '%task_id%' >> %succeedtasks_filepath% +fi ' > clone_%github_repo_name%.sh ; chmod +x clone_%github_repo_name%.sh ; ./clone_%github_repo_name%.sh \ No newline at end of file diff --git a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/install_chefdk.sh b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/install_chefdk.sh index ef89dea2..b8f8c293 100644 --- a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/install_chefdk.sh +++ b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/install_chefdk.sh @@ -8,7 +8,7 @@ if [ %osfamily% == "redhat" ] ; then chefdkfile='chefdk-%chefdk_version%-1.el7.x86_64.rpm' rm -f "$chefdkfile" - wget "http://snurran.sics.se/hops/$chefdkfile" + wget "https://hopsworks-distribution.s3-eu-west-1.amazonaws.com/$chefdkfile" %sudo_command% yum install -y "$chefdkfile" RES=$? @@ -26,7 +26,7 @@ elif [ %osfamily% == "ubuntu" ] ; then chefdkfile='chefdk_%chefdk_version%-1_amd64.deb' rm -f "$chefdkfile" - wget "http://snurran.sics.se/hops/$chefdkfile" + wget "https://hopsworks-distribution.s3-eu-west-1.amazonaws.com/$chefdkfile" %sudo_command% dpkg -i "$chefdkfile" RES=$? @@ -40,6 +40,8 @@ else exit 1 fi if [ $RES -eq 0 ] ; then + # Fix for expired Lets Encrypt CA + %sudo_command% sed -ie "/DST Root CA X3/,+19d" /opt/chefdk/embedded/ssl/certs/cacert.pem echo '%task_id%' >> %succeedtasks_filepath% fi exit $RES diff --git a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/make_solo_rb.sc b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/make_solo_rb.sc index 27c1abbf..ff5ba0a7 100644 --- a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/make_solo_rb.sc +++ b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/make_solo_rb.sc @@ -3,6 +3,7 @@ set -eo pipefail %sudo_command% touch solo.rb %sudo_command% chmod 777 solo.rb cat > solo.rb <<-'END_OF_FILE' -file_cache_path "/tmp/chef-solo" +file_cache_path "%file_cache_path%" cookbook_path [%cookbooks_path%] +%rubygems_url% END_OF_FILE' > make_solo_rb.sh ; chmod +x make_solo_rb.sh ; ./make_solo_rb.sh \ No newline at end of file diff --git a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/run_recipe.sc b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/run_recipe.sc index 47952d80..186556f1 100644 --- a/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/run_recipe.sc +++ b/karamel-core/src/main/resources/se/kth/karamel/backend/shellscripts/run_recipe.sc @@ -4,6 +4,6 @@ echo $(date '+%H:%M:%S'): '%json_file_name%' >> order cat > %json_file_name%.json <<-'END_OF_FILE' %chef_json% END_OF_FILE -%sudo_command% chef-solo -c %install_dir_path%/solo.rb -j %install_dir_path%/%json_file_name%.json 2>&1 | tee %log_file_name%.log +%sudo_command% chef-solo -c %install_dir_path%/solo.rb -j %install_dir_path%/%json_file_name%.json 2>&1 >> %log_file_name%.log echo '%task_id%' >> %succeedtasks_filepath% ' > %json_file_name%.sh ; chmod +x %json_file_name%.sh ; ./%json_file_name%.sh diff --git a/karamel-core/src/test/java/se/kth/karamel/backend/commad/CommandServiceTest.java b/karamel-core/src/test/java/se/kth/karamel/backend/commad/CommandServiceTest.java index 86145ff8..bc9dbd19 100644 --- a/karamel-core/src/test/java/se/kth/karamel/backend/commad/CommandServiceTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/backend/commad/CommandServiceTest.java @@ -6,17 +6,14 @@ package se.kth.karamel.backend.commad; import java.io.IOException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + import static org.junit.Assert.*; import org.junit.Test; import se.kth.karamel.backend.ClusterDefinitionService; import se.kth.karamel.backend.command.CommandResponse; import se.kth.karamel.backend.command.CommandService; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.Settings; import se.kth.karamel.common.exception.KaramelException; -import static se.kth.karamel.common.util.Settings.REPO_WITH_SUBCOOKBOOK_PATTERN; /** * diff --git a/karamel-core/src/test/java/se/kth/karamel/backend/converter/ChefJsonGeneratorTest.java b/karamel-core/src/test/java/se/kth/karamel/backend/converter/ChefJsonGeneratorTest.java index 3beee0df..8bf23a9f 100644 --- a/karamel-core/src/test/java/se/kth/karamel/backend/converter/ChefJsonGeneratorTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/backend/converter/ChefJsonGeneratorTest.java @@ -13,7 +13,6 @@ import se.kth.karamel.backend.ClusterDefinitionService; import se.kth.karamel.backend.running.model.ClusterRuntime; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.backend.mocking.MockingUtil; import se.kth.karamel.common.util.Settings; diff --git a/karamel-core/src/test/java/se/kth/karamel/backend/dag/DagTest.java b/karamel-core/src/test/java/se/kth/karamel/backend/dag/DagTest.java index c5310dd7..08949e94 100644 --- a/karamel-core/src/test/java/se/kth/karamel/backend/dag/DagTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/backend/dag/DagTest.java @@ -11,6 +11,7 @@ import java.util.Set; import org.junit.Assert; import org.junit.Test; +import se.kth.karamel.common.clusterdef.json.JsonCluster; import se.kth.karamel.common.exception.DagConstructionException; /** @@ -160,7 +161,7 @@ public void terminate() { dag.addTask(new DummyTask("task31")); dag.addTask(new DummyTask("task32")); //task33 doesnt have any task, expecting exception here - dag.start(); + dag.start(new JsonCluster()); } @Test @@ -232,7 +233,7 @@ public void terminate() { dag.addTask(new DummyTask("task31")); dag.addTask(new DummyTask("task32")); dag.addTask(new DummyTask("task33")); - dag.start(); + dag.start(new JsonCluster()); } @Test diff --git a/karamel-core/src/test/java/se/kth/karamel/backend/github/GithubUserTest.java b/karamel-core/src/test/java/se/kth/karamel/backend/github/GithubUserTest.java deleted file mode 100644 index 2654ef0b..00000000 --- a/karamel-core/src/test/java/se/kth/karamel/backend/github/GithubUserTest.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.backend.github; - -import com.google.common.base.Charsets; -import com.google.common.io.Files; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.constructor.Constructor; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; -import se.kth.karamel.backend.ClusterDefinitionService; -import se.kth.karamel.backend.Experiment; -import se.kth.karamel.backend.github.util.CookbookGenerator; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.client.api.KaramelApiImpl; -import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; -import se.kth.karamel.common.clusterdef.json.JsonGroup; -import se.kth.karamel.common.clusterdef.json.JsonRecipe; -import se.kth.karamel.common.util.Settings; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.common.cookbookmeta.KaramelFile; -import se.kth.karamel.common.cookbookmeta.KaramelFileYamlDeps; -import se.kth.karamel.common.cookbookmeta.KaramelFileYamlRep; - -public class GithubUserTest { - - String user = ""; - String password = ""; - KaramelApi api; - - public GithubUserTest() { - } - - @BeforeClass - public static void setUpClass() { - } - - @AfterClass - public static void tearDownClass() { - } - - @Before - public void setUp() { - api = new KaramelApiImpl(); - try { - GithubUser u2 = api.loadGithubCredentials(); - user = u2.getUser(); - password = u2.getPassword(); - } catch (KaramelException ex) { - fail(ex.getMessage()); - } - } - - @After - public void tearDown() { - } - - /** - * Test of getEmail method, of class GithubUser. - */ - @Test - public void testAccount() { - try { - api.registerGithubAccount(user, password); - GithubUser u2 = api.loadGithubCredentials(); - assertEquals(this.user, u2.getUser()); - assertEquals(password, u2.getPassword()); - // TODO review the generated test code and remove the default call to fail. - } catch (KaramelException ex) { - fail(ex.getMessage()); - } - } - - /** - * List Organizations in github - */ -// @Test - public void testListOrgs() { - try { - api.registerGithubAccount(user, password); - List orgs = api.listGithubOrganizations(); - for (OrgItem o : orgs) { - System.out.println("Organization: " + o.getName() + " : " + o.getGravitar()); - } - } catch (KaramelException ex) { - fail(ex.getMessage()); - } - } - -// @Test - public void testListRepos() { - try { - List orgs = api.listGithubRepos("hopshadoop"); - for (RepoItem o : orgs) { - System.out.println("Repo: " + o.getName() + " - " + o.getDescription() + " : " + o.getSshUrl()); - } - } catch (KaramelException ex) { - fail(ex.getMessage()); - } - } - -// @Test - public void testKaramelfile() { - - try { - StringBuilder karamelContents = CookbookGenerator.instantiateFromTemplate( - Settings.CB_TEMPLATE_KARAMELFILE, - "name", "jim", - "next_recipes", "" - ); - KaramelFile karamelFile = new KaramelFile(karamelContents.toString()); - - String ymlString = "name: MySqlCluster\n" - + "ec2:\n" - + " type: m3.medium\n" - + " region: eu-west-1\n" - + "\n" - + "cookbooks:\n" - + " ndb:\n" - + " github: \"hopshadoop/ndb-chef\"\n" - + " branch: \"master\"\n" - + " \n" - + "groups: \n" - + " nodes:\n" - + " size: 1 \n" - + " recipes: \n" - + " - ndb::mgmd\n" - + " - ndb::ndbd\n" - + " - ndb::mysqld\n" - + " - ndb::memcached"; - JsonCluster jsonCluster = ClusterDefinitionService.yamlToJsonObject(ymlString); - KaramelFileYamlDeps yd = new KaramelFileYamlDeps(); - List clusterDependencies = new ArrayList<>(); - for (JsonGroup g : jsonCluster.getGroups()) { - for (JsonCookbook cb : g.getCookbooks()) { - for (JsonRecipe r : cb.getRecipes()) { - clusterDependencies.add(r.getCanonicalName()); - } - } - } - yd.setRecipe("test::test"); - yd.setGlobal(clusterDependencies); - yd.setLocal(null); - List yds = karamelFile.getDependencies(); - yds.add(yd); - DumperOptions options = new DumperOptions(); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); - Representer r = new Representer(); - r.addClassTag(KaramelFile.class, Tag.MAP); - Yaml karamelYml = new Yaml(new Constructor(KaramelFileYamlRep.class), r, options); - String karamelFileContents = karamelYml.dump(karamelFile); - - File f = File.createTempFile("karamelfile", "out"); - try (PrintWriter out = new PrintWriter(f)) { - out.println(karamelFileContents); - } - String contents = Files.toString(f, Charsets.UTF_8); - KaramelFile karamelFile2 = new KaramelFile(contents); - } catch (KaramelException | IOException ex) { - fail(ex.getMessage()); - } - - } - -// @Test - public void testCreateAndDeleteRepo() { - try { - String owner = "karamelchef"; - Experiment ec = new Experiment(); - Experiment.Code exp = new Experiment.Code("experiment", "echo \"jim\"\n" - + "java -jar -D%%maxHeapSize%% prog.jar", "config.props", "%%maxHeapSize%%=128m\n%%log%%=true\n", - "bash"); - List exps = ec.getCode(); - exps.add(exp); - ec.setUser("blah"); - ec.setGroup("blah"); - ec.setGithubOwner(owner); - ec.setGithubRepo("test"); - ec.setDescription("Test experiment"); - ec.setClusterDefinition("name: MySqlCluster\n" - + "ec2:\n" - + " type: m3.medium\n" - + " region: eu-west-1\n" - + "\n" - + "cookbooks:\n" - + " ndb:\n" - + " github: \"hopshadoop/ndb-chef\"\n" - + " branch: \"master\"\n" - + " \n" - + "groups: \n" - + " nodes:\n" - + " size: 1 \n" - + " recipes: \n" - + " - ndb::mgmd\n" - + " - ndb::ndbd\n" - + " - ndb::mysqld\n" - + " - ndb::memcached" - ); - - api.commitAndPushExperiment(ec); - - try { - Thread.sleep(4000); - } catch (InterruptedException ex) { - fail(ex.getMessage()); - } - - api.removeRepo(owner, "test", true, true); - - } catch (KaramelException ex) { - fail(ex.getMessage()); - } - } - -} diff --git a/karamel-core/src/test/java/se/kth/karamel/backend/kandy/KandyRestClientTest.java b/karamel-core/src/test/java/se/kth/karamel/backend/kandy/KandyRestClientTest.java index bce6d07d..a8068deb 100644 --- a/karamel-core/src/test/java/se/kth/karamel/backend/kandy/KandyRestClientTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/backend/kandy/KandyRestClientTest.java @@ -12,7 +12,6 @@ import se.kth.karamel.common.stats.ClusterStats; import se.kth.karamel.common.stats.PhaseStat; import se.kth.karamel.common.stats.TaskStat; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.Settings; /** diff --git a/karamel-core/src/test/java/se/kth/karamel/client/api/KaramelApiTest.java b/karamel-core/src/test/java/se/kth/karamel/client/api/KaramelApiTest.java index 80d7afbe..569e6538 100644 --- a/karamel-core/src/test/java/se/kth/karamel/client/api/KaramelApiTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/client/api/KaramelApiTest.java @@ -4,7 +4,6 @@ import com.google.common.io.Resources; import org.junit.Test; import se.kth.karamel.backend.ClusterService; -import se.kth.karamel.backend.Experiment; import se.kth.karamel.backend.running.model.ClusterRuntime; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.util.Ec2Credentials; @@ -12,7 +11,6 @@ import se.kth.karamel.common.util.SshKeyPair; import java.io.IOException; -import java.util.ArrayList; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -31,32 +29,6 @@ public void dummyTest() { //just that we dont need to ignore this class all the time } - @Test - public void commitPushTest() throws KaramelException { - Experiment exp = new Experiment(); - - exp.setBerksfile("ark\njava"); - exp.setClusterDefinition(""); - Experiment.Code ec = new Experiment.Code("experiment", "echo 'jim'", "config/config.props", "jim=dow", "bash"); - Experiment.Code ecL = new Experiment.Code("linda", "python blah", "my.props", "lin=gron", "python"); - ArrayList code = new ArrayList<>(); - code.add(ec); - code.add(ecL); - exp.setCode(code); - - exp.setDescription("some repo"); - exp.setExperimentSetupCode("chef code"); - exp.setGithubOwner("karamelchef"); - exp.setGithubRepo("test"); - exp.setGlobalDependencies("hops::nn\nhops::dn"); - exp.setGroup("testG"); - exp.setLocalDependencies("hops::install"); - exp.setUser("testU"); -// ChefExperimentExtractor.parseAttributesAddToGit("karamelchef", "test", exp); -// ChefExperimentExtractor.parseRecipesAddToGit("karamelchef", "test", exp); -// KaramelizedCookbook kc = new KaramelizedCookbook("https://github.com/karamelchef/test", true); - } - // @Test public void testGetCookbookDetails() throws KaramelException { String json = api.getCookbookDetails("https://github.com/hopstart/hadoop-chef", false); diff --git a/karamel-core/src/test/java/se/kth/karamel/client/model/yaml/ClusterDefinitionTest.java b/karamel-core/src/test/java/se/kth/karamel/client/model/yaml/ClusterDefinitionTest.java index 1feb7958..df85af99 100644 --- a/karamel-core/src/test/java/se/kth/karamel/client/model/yaml/ClusterDefinitionTest.java +++ b/karamel-core/src/test/java/se/kth/karamel/client/model/yaml/ClusterDefinitionTest.java @@ -15,14 +15,12 @@ import se.kth.karamel.common.clusterdef.Gce; import se.kth.karamel.common.clusterdef.Nova; import se.kth.karamel.common.clusterdef.json.JsonCluster; -import se.kth.karamel.common.clusterdef.json.JsonCookbook; import se.kth.karamel.common.clusterdef.yaml.YamlCluster; import se.kth.karamel.common.clusterdef.yaml.YamlGroup; import se.kth.karamel.common.clusterdef.yaml.YamlScope; import se.kth.karamel.common.exception.KaramelException; import se.kth.karamel.common.exception.MetadataParseException; import se.kth.karamel.common.exception.ValidationException; -import se.kth.karamel.common.util.IoUtils; import se.kth.karamel.common.util.Settings; import se.kth.karamel.core.clusterdef.ClusterDefinitionValidator; diff --git a/karamel-ui/pom.xml b/karamel-ui/pom.xml index a804ffaf..61f2752b 100644 --- a/karamel-ui/pom.xml +++ b/karamel-ui/pom.xml @@ -6,7 +6,7 @@ se.kth.karamel karamel-parent - 0.5 + 0.10 se.kth.karamel @@ -271,7 +271,9 @@ conf true - -Xms128m + + -Xms128m -Xmx6g + unix diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/KaramelServiceApplication.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/KaramelServiceApplication.java index c058f9d3..c08d0c59 100644 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/KaramelServiceApplication.java +++ b/karamel-ui/src/main/java/se/kth/karamel/webservice/KaramelServiceApplication.java @@ -20,6 +20,7 @@ import java.net.URL; import java.net.UnknownHostException; import java.util.EnumSet; +import java.util.Properties; import javax.servlet.DispatcherType; import javax.servlet.FilterRegistration; import javax.swing.ImageIcon; @@ -50,16 +51,8 @@ import se.kth.karamel.webservice.calls.definition.YamlToJson; import se.kth.karamel.webservice.calls.ec2.LoadEc2Credentials; import se.kth.karamel.webservice.calls.ec2.ValidateEc2Credentials; -import se.kth.karamel.webservice.calls.experiment.LoadExperiment; -import se.kth.karamel.webservice.calls.experiment.PushExperiment; -import se.kth.karamel.webservice.calls.experiment.RemoveFileFromExperiment; import se.kth.karamel.webservice.calls.gce.LoadGceCredentials; import se.kth.karamel.webservice.calls.gce.ValidateGceCredentials; -import se.kth.karamel.webservice.calls.github.GetGithubCredentials; -import se.kth.karamel.webservice.calls.github.GetGithubOrgs; -import se.kth.karamel.webservice.calls.github.GetGithubRepos; -import se.kth.karamel.webservice.calls.github.RemoveRepository; -import se.kth.karamel.webservice.calls.github.SetGithubCredentials; import se.kth.karamel.webservice.calls.nova.LoadNovaCredentials; import se.kth.karamel.webservice.calls.nova.ValidateNovaCredentials; import se.kth.karamel.webservice.calls.occi.LoadOcciCredentials; @@ -164,6 +157,16 @@ public static void usage(int exitValue) { System.exit(exitValue); } + private String getKaramelVersion() { + final Properties props = new Properties(); + try { + props.load(getClass().getClassLoader().getResourceAsStream("karamel.properties")); + } catch (IOException ex) { + return "unknown"; + } + return props.getProperty("karamel.version", "unknown"); + } + public static void main(String[] args) throws Exception { System.setProperty("java.net.preferIPv4Stack", "true"); @@ -175,6 +178,8 @@ public static void main(String[] args) throws Exception { String sudoPasswd = ""; karamelApi = new KaramelApiImpl(); + KaramelServiceApplication karamelServiceApplication = new KaramelServiceApplication(); + System.out.println("Version: " + karamelServiceApplication.getKaramelVersion()); try { CommandLine line = parser.parse(options, args); @@ -215,7 +220,9 @@ public static void main(String[] args) throws Exception { // } // sudoPasswd = c.readLine("Enter your sudo password (just press 'enter' if you don't have one):"); // } - new KaramelServiceApplication().run(modifiedArgs); + karamelServiceApplication.run(modifiedArgs); + logger.info(String.format("Version: %s", karamelServiceApplication.getKaramelVersion())); + Thread.currentThread().sleep(2000); // Try to open and read the yaml file. @@ -250,7 +257,8 @@ public static void main(String[] args) throws Exception { } if (!cli) { - new KaramelServiceApplication().run(modifiedArgs); + karamelServiceApplication.run(modifiedArgs); + logger.info(String.format("Version: %s", karamelServiceApplication.getKaramelVersion())); } Runtime.getRuntime().addShutdownHook(new KaramelCleanupBeforeShutdownThread()); @@ -331,18 +339,6 @@ public void run(KaramelServiceConfiguration configuration, Environment environme environment.jersey().register(new ExitKaramel(karamelApi)); environment.jersey().register(new PingServer(karamelApi)); - //github - environment.jersey().register(new GetGithubCredentials(karamelApi)); - environment.jersey().register(new SetGithubCredentials(karamelApi)); - environment.jersey().register(new GetGithubOrgs(karamelApi)); - environment.jersey().register(new GetGithubRepos(karamelApi)); - environment.jersey().register(new RemoveRepository(karamelApi)); - - //experiment - environment.jersey().register(new LoadExperiment(karamelApi)); - environment.jersey().register(new PushExperiment(karamelApi)); - environment.jersey().register(new RemoveFileFromExperiment(karamelApi)); - //Openstack nova environment.jersey().register(new LoadNovaCredentials(karamelApi)); environment.jersey().register(new ValidateNovaCredentials(karamelApi)); diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/definition/FetchCookbook.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/definition/FetchCookbook.java index 67967030..984fa6fc 100644 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/definition/FetchCookbook.java +++ b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/definition/FetchCookbook.java @@ -33,7 +33,7 @@ public FetchCookbook(KaramelApi karamelApi) { public Response getCookbook(CookbookJSON cookbookJSON) { Response response = null; try { - String cookbookDetails = karamelApi.getCookbookDetails(cookbookJSON.getUrl(), cookbookJSON.isRefresh()); + String cookbookDetails = karamelApi.getCookbookDetails(cookbookJSON.getUrl()); response = Response.status(Response.Status.OK).entity(cookbookDetails).build(); } catch (KaramelException e) { diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/LoadExperiment.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/LoadExperiment.java deleted file mode 100644 index 03a5a916..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/LoadExperiment.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.experiment; - -import javax.ws.rs.Consumes; -import javax.ws.rs.FormParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.Experiment; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; - -/** - * - * @author kamal - */ -@Path("/experiment/load") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_FORM_URLENCODED) -public class LoadExperiment extends AbstractCall { - - public LoadExperiment(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response loadExperiment(@FormParam("experimentUrl") String experimentUrl) { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - Experiment ec = karamelApi.loadExperiment(experimentUrl); - response = Response.status(Response.Status.OK).entity(ec).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - - return response; - } -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/PushExperiment.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/PushExperiment.java deleted file mode 100644 index 2b99d231..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/PushExperiment.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.experiment; - -import javax.ws.rs.Consumes; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.Experiment; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; -import se.kth.karamel.webservicemodel.StatusResponseJSON; - -/** - * - * @author kamal - */ -@Path("/experiment/push") -@Consumes(MediaType.APPLICATION_JSON) -@Produces(MediaType.APPLICATION_JSON) -public class PushExperiment extends AbstractCall { - - public PushExperiment(KaramelApi karamelApi) { - super(karamelApi); - } - - @PUT - public Response pushExperiment(Experiment experiment) { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - karamelApi.commitAndPushExperiment(experiment); - response = Response.status(Response.Status.OK). - entity(new StatusResponseJSON(StatusResponseJSON.SUCCESS_STRING, "success")).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - - return response; - } -} - diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/RemoveFileFromExperiment.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/RemoveFileFromExperiment.java deleted file mode 100644 index 375070c9..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/experiment/RemoveFileFromExperiment.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.experiment; - -import javax.ws.rs.Consumes; -import javax.ws.rs.FormParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.webservice.calls.AbstractCall; -import se.kth.karamel.webservicemodel.StatusResponseJSON; - -/** - * - * @author kamal - */ -@Path("/experiment/removeFile") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_FORM_URLENCODED) -public class RemoveFileFromExperiment extends AbstractCall { - - public RemoveFileFromExperiment(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response removeFileFromExperiment(@FormParam("org") String org, @FormParam("repo") String repo, - @FormParam("filename") String filename) { - Response response = null; - try { - logger.debug(" Received request to set github credentials.... "); - karamelApi.removeFileFromExperiment(org, repo, filename); - response = Response.status(Response.Status.OK).entity( - new StatusResponseJSON(StatusResponseJSON.SUCCESS_STRING, "success")).build(); - } catch (Exception e) { - response = buildExceptionResponse(e); - } - return response; - } - -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubCredentials.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubCredentials.java deleted file mode 100644 index 2625dcbb..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubCredentials.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.github; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.github.GithubUser; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; - -/** - * - * @author kamal - */ -@Path("/github/getCredentials") -@Produces(MediaType.APPLICATION_JSON) -public class GetGithubCredentials extends AbstractCall { - - public GetGithubCredentials(KaramelApi karamelApi) { - super(karamelApi); - } - - @GET - public Response getGithubCredentials() { - Response response = null; - logger.debug(" Received request to get github credentials.... "); - try { - GithubUser credentials = karamelApi.loadGithubCredentials(); - response = Response.status(Response.Status.OK). - entity(credentials).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - return response; - } -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubOrgs.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubOrgs.java deleted file mode 100644 index 43d25b5a..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubOrgs.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.github; - -import java.util.List; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.github.OrgItem; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; - -/** - * - * @author kamal - */ -@Path("/github/getOrgs") -@Produces(MediaType.APPLICATION_JSON) -public class GetGithubOrgs extends AbstractCall { - - public GetGithubOrgs(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response getGithubOrgs() { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - List orgs = karamelApi.listGithubOrganizations(); - response = Response.status(Response.Status.OK). - entity(orgs).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - - return response; - } -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubRepos.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubRepos.java deleted file mode 100644 index 9dffd388..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/GetGithubRepos.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.github; - -import java.util.List; -import javax.ws.rs.Consumes; -import javax.ws.rs.FormParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.github.RepoItem; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; - -/** - * - * @author kamal - */ -@Path("/github/getRepos") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_FORM_URLENCODED) -public class GetGithubRepos extends AbstractCall { - - public GetGithubRepos(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response getGithubRepos(@FormParam("org") String org) { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - List repos = karamelApi.listGithubRepos(org); - response = Response.status(Response.Status.OK). - entity(repos).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - - return response; - } -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/RemoveRepository.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/RemoveRepository.java deleted file mode 100644 index 2bcf29ed..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/RemoveRepository.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.github; - -import javax.ws.rs.Consumes; -import javax.ws.rs.FormParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; -import se.kth.karamel.webservicemodel.StatusResponseJSON; - -/** - * - * @author kamal - */ -@Path("/github/removeRepository") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_FORM_URLENCODED) -public class RemoveRepository extends AbstractCall { - - public RemoveRepository(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response removeRepository(@FormParam("org") String org, @FormParam("repo") String repo, - @FormParam("local") boolean local, @FormParam("remote") boolean remote) { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - karamelApi.removeRepo(org, repo, local, remote); - response = Response.status(Response.Status.OK). - entity(new StatusResponseJSON(StatusResponseJSON.SUCCESS_STRING, "success")).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - return response; - } - -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/SetGithubCredentials.java b/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/SetGithubCredentials.java deleted file mode 100644 index 39508c70..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservice/calls/github/SetGithubCredentials.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservice.calls.github; - -import javax.ws.rs.Consumes; -import javax.ws.rs.FormParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import se.kth.karamel.backend.github.GithubUser; -import se.kth.karamel.client.api.KaramelApi; -import se.kth.karamel.common.exception.KaramelException; -import se.kth.karamel.webservice.calls.AbstractCall; - -/** - * - * @author kamal - */ -@Path("/github/setCredentials") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_FORM_URLENCODED) -public class SetGithubCredentials extends AbstractCall { - - public SetGithubCredentials(KaramelApi karamelApi) { - super(karamelApi); - } - - @POST - public Response setGithubCredentials(@FormParam("user") String user, @FormParam("password") String password) { - Response response = null; - logger.debug(" Received request to set github credentials.... "); - try { - GithubUser githubUser = karamelApi.registerGithubAccount(user, password); - - response = Response.status(Response.Status.OK). - entity(githubUser).build(); - } catch (KaramelException e) { - response = buildExceptionResponse(e); - } - - return response; - } -} diff --git a/karamel-ui/src/main/java/se/kth/karamel/webservicemodel/GithubOrgsJSON.java b/karamel-ui/src/main/java/se/kth/karamel/webservicemodel/GithubOrgsJSON.java deleted file mode 100644 index 0ed2803f..00000000 --- a/karamel-ui/src/main/java/se/kth/karamel/webservicemodel/GithubOrgsJSON.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package se.kth.karamel.webservicemodel; - -import se.kth.karamel.backend.github.OrgItem; -import java.util.List; - -public class GithubOrgsJSON { - - - - List orgs; - - public List getOrgs() { - return orgs; - } - - public void setOrgs(List orgs) { - this.orgs = orgs; - } - -} diff --git a/karamel-ui/src/main/resources/assets/karamel/active-cluster.service.js b/karamel-ui/src/main/resources/assets/karamel/active-cluster.service.js index d3667d95..829cbe17 100644 --- a/karamel-ui/src/main/resources/assets/karamel/active-cluster.service.js +++ b/karamel-ui/src/main/resources/assets/karamel/active-cluster.service.js @@ -10,12 +10,7 @@ angular.module('main.module') cacheService, alertService) { function _launchCluster() { - var coreFormatCluster = $rootScope.activeCluster.toCoreApiFormat(); - var data = { - json: angular.toJson(coreFormatCluster) - }; - $log.info(data); - coreService.startCluster(data) + coreService.startCluster($rootScope.activeCluster) .success(function(data, status, headers, config) { $log.info("Connection Successful."); alertService.addAlert({type: 'success', msg: 'Cluster Launch Successful.'}); @@ -28,7 +23,7 @@ angular.module('main.module') }); } return { - addNewRecipe: function(group) { + /*addNewRecipe: function(group) { var modalInstance = $modal.open({ templateUrl: 'karamel/board/groups/new-recipe.html', @@ -136,7 +131,7 @@ angular.module('main.module') SweetAlert.swal("Cancelled", "Phew, That was close :)", "error"); } }); - }, + }, */ sudoPassword: function(password) { coreService.sudoPassword(password) @@ -161,12 +156,12 @@ angular.module('main.module') }); modalInstance.result.then(function(newGroupInfo) { - if (newGroupInfo) { + /*if (newGroupInfo) { var group = new Group(); group.load(newGroupInfo); $rootScope.activeCluster.addGroup(group); cacheService.updateCache(); - } + }*/ }); }, updateGroupInfo: function(existingGroupInfo) { @@ -182,7 +177,7 @@ angular.module('main.module') }); modalInstance.result.then(function(updatedGroupInfo) { - if (updatedGroupInfo) { + /*if (updatedGroupInfo) { var cluster = $rootScope.activeCluster; var id = -1; for (var i = 0; i < cluster.groups.length; i++) { @@ -197,7 +192,7 @@ angular.module('main.module') } cacheService.updateCache(); - } + } */ }); }, configureGlobalAttributes: function() { @@ -216,10 +211,10 @@ angular.module('main.module') }); modalInstance.result.then(function(result) { - if (result) { + /*if (result) { $rootScope.activeCluster.cookbooks = result.cookbooks; cacheService.updateCache(); - } + } */ }); }, configureGlobalProvider: function() { @@ -238,11 +233,12 @@ angular.module('main.module') }); modalInstance.result.then(function(result) { - if (result) { + /*if (result) { $rootScope.activeCluster.ec2 = result.ec2; $rootScope.activeCluster.baremetal = result.baremetal; cacheService.updateCache(); - } + + } */ }); }, @@ -262,11 +258,11 @@ angular.module('main.module') }); modalInstance.result.then(function(result) { - if (result) { + /*if (result) { group.ec2 = result.ec2; group.baremetal = result.baremetal; cacheService.updateCache(); - } + }*/ }); }, configureGroupAttributes: function(group) { @@ -291,25 +287,22 @@ angular.module('main.module') }); }, hasEc2: function() { - return ($rootScope.activeCluster && $rootScope.activeCluster.hasEc2()); + return ($rootScope.activeCluster && $rootScope.activeCluster.ec2 != null); }, hasBaremetal: function() { - return ($rootScope.activeCluster && $rootScope.activeCluster.hasBaremetal()); + return ($rootScope.activeCluster && $rootScope.activeCluster.baremetal != null); }, hasGce: function() { - return ($rootScope.activeCluster && $rootScope.activeCluster.hasGce()); + return ($rootScope.activeCluster && $rootScope.activeCluster.gce != null); }, hasNova: function() { - return ($rootScope.activeCluster && $rootScope.activeCluster.hasNova()); + return ($rootScope.activeCluster && $rootScope.activeCluster.nova != null); }, hasOcci: function() { - return ($rootScope.activeCluster && $rootScope.activeCluster.hasOcci()); + return ($rootScope.activeCluster && $rootScope.activeCluster.occi != null); }, hasProvider: function() { - return ($rootScope.activeCluster && - ($rootScope.activeCluster.hasEc2() || $rootScope.activeCluster.hasBaremetal() || - $rootScope.activeCluster.hasGce() || $rootScope.activeCluster.hasNova() || - $rootScope.activeCluster.hasOcci())); + return (this.hasEc2() || this.hasBaremetal() || this.hasGce() || this.hasNova() || this.hasOcci()); }, name: function() { return $rootScope.activeCluster.name; @@ -321,19 +314,11 @@ angular.module('main.module') alertService.addAlert({type: 'warning', msg: 'No Active Cluster Found.'}); return; } - if (!$rootScope.activeCluster.areCredentialsSet()) { - this.setCredentials(true); - } - else { - _launchCluster(); - } + this.setCredentials(true); + _launchCluster(); }, getJsonForRest: function() { - var coreFormatCluster = $rootScope.activeCluster.toCoreApiFormat(); - var data = { - json: angular.toJson(coreFormatCluster) - }; - return data; + return $rootScope.activeCluster; }, setCredentials: function(isLaunch) { var modalInstance = $modal.open({ diff --git a/karamel-ui/src/main/resources/assets/karamel/board/board.contoller.js b/karamel-ui/src/main/resources/assets/karamel/board/board.contoller.js index 94f915b8..c65de6f1 100644 --- a/karamel-ui/src/main/resources/assets/karamel/board/board.contoller.js +++ b/karamel-ui/src/main/resources/assets/karamel/board/board.contoller.js @@ -15,8 +15,8 @@ angular.module('main.module') if (clusterObj !== null) { try { - var cluster = new Cluster(); - cluster.copy(clusterObj); + //var cluster = new Cluster(); + //cluster.copy(clusterObj); $rootScope.activeCluster = cluster; $rootScope.context = cluster.name; alertService.addAlert({type: 'success', msg: 'Model Loaded Successfully.'}); diff --git a/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js b/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js index 47b3d459..215950a2 100644 --- a/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js +++ b/karamel-ui/src/main/resources/assets/karamel/board/datamodel.js @@ -1,7 +1,7 @@ // ================================================ CLUSTER ====================================== // function Cluster() { this.name = null; - this.cookbooks = []; + this.rootCookbooks = []; this.groups = []; this.ec2 = null; this.gce = null; @@ -11,31 +11,31 @@ function Cluster() { this.sshKeyPair = null; this.addCookbook = function(cookbook) { - if (this.cookbooks == null) { - this.cookbooks = []; + if (this.rootCookbooks == null) { + this.rootCookbooks = []; } - this.cookbooks.push(cookbook); + this.rootCookbooks.push(cookbook); }; this.removeCookbook = function(cookbook) { var id = -1; - // In this we can also override the equals method in the cookbook object and then call the equals method TODO :============== - for (var i = 0; i < this.cookbooks.length; i++) { - if (this.cookbooks[i].id === cookbook.id) { + // In this we can also override the equals method in the cookbook object and then call the equals method TODO :============== + for (var i = 0; i < this.rootCookbooks.length; i++) { + if (this.rootCookbooks[i].id === cookbook.id) { id = i; } } // If any match found, then remove the entry. if (id != -1) { - this.cookbooks.splice(id, 1); + this.rootCookbooks.splice(id, 1); } }; this.containsCookbook = function(cookbook) { - for (var i = 0; i < this.cookbooks.length; i++) { - if (cookbook.id === this.cookbooks[i].id) { - return this.cookbooks[i]; + for (var i = 0; i < this.rootCookbooks.length; i++) { + if (cookbook.id === this.rootCookbooks[i].id) { + return this.rootCookbooks[i]; } } return null; @@ -47,7 +47,7 @@ function Cluster() { this.copyUpdatedClusterData = function(updatedClusterInfo) { this.name = updatedClusterInfo.name; - this.cookbooks = updatedClusterInfo.cookbooks; + this.cookbooks = updatedClusterInfo.rootCookbooks; this.groups = updatedClusterInfo.groups; }; @@ -61,7 +61,7 @@ function Cluster() { this.loadBaremetal(this, other["baremetal"]); this.loadSshKeyPair(this, null); this.loadGroups(this, other["groups"]); - this.loadCookbooks(this, other["cookbooks"]); + this.loadRootCookbook(this, other["rootCookbooks"]); }; this.copy = function(other) { @@ -101,7 +101,7 @@ function Cluster() { this.sshKeyPair.copy(other.sshKeyPair); } this.copyGroups(this, other["groups"]); - this.copyCookbooks(this, other["cookbooks"]); + this.copyCookbooks(this, other["rootCookbooks"]); }; this.containsCookbook = function(cookbookArray, cookbook) { @@ -176,12 +176,24 @@ function Cluster() { container.sshKeyPair = new SshKeyPair(); }; + this.laodRootCookbooks = function(container, cookbooks) { + for (var i = 0; i < cookbooks.length; i++) { + var cookbook = new Cookbook(); + cookbook.load(cookbooks[i]); + container.addCookbook(cookbook); + var recipes = cookbooks[i].metadataRb.recipes; + for (var j = 0; j < recipes.length; j++) { + cookbook.addRecipe(new Recipe(recipes[j]["name"])); + } + } + }; + this.loadCookbooks = function(container, cookbooks) { for (var i = 0; i < cookbooks.length; i++) { var cookbook = new Cookbook(); cookbook.load(cookbooks[i]); container.addCookbook(cookbook); - var recipes = cookbooks[i]["recipes"]; + var recipes = cookbooks[i].metadataRb.recipes; for (var j = 0; j < recipes.length; j++) { cookbook.addRecipe(new Recipe(recipes[j]["name"])); } @@ -203,7 +215,7 @@ function Cluster() { var cookbook = new Cookbook(); cookbook.copy(cookbooks[i]); container.addCookbook(cookbook); - var recipes = cookbooks[i]["recipes"]; + var recipes = cookbooks[i].metadataRb.recipes; for (var j = 0; j < recipes.length; j++) { cookbook.addRecipe(new Recipe(recipes[j]["title"])); } @@ -515,19 +527,9 @@ function Baremetal() { Baremetal.prototype = Object.create(Provider.prototype); // =========================================== COOKBOOKS ============================================== // -function Cookbook() { - this.id = null; - this.alias = null; - this.attributes = {}; - this.recipes = []; - - this.addPropertyToAttributes = function(key, value) { - this.attributes[key] = value; - }; - - this.removePropertyFromAttributes = function(key) { - delete this.attributes[key]; - }; +function RootCookbook() { + this.github = ""; + this.branch = ""; this.equals = function(other) { if (other != null) { @@ -540,75 +542,59 @@ function Cookbook() { // Load data into the cookbook. this.load = function(other) { - this.id = other.id; - this.alias = other.alias; - this.attributes = other.attrs; // FIX ME: Name discrepancy should not be there. - this.cookbookHomeUrl = other.cookbookHomeUrl; + this.github = other.github; + this.branch = other.branch; }; - this.copy = function(other) { - this.id = other.id; - this.alias = other.alias; - this.attributes = other.attributes; - this.cookbookHomeUrl = other.cookbookHomeUrl; - - // Load recipes instead of copying. + this.github = other.github; + this.branch = other.branch; }; +} - // Add recipe to the cookbook. - this.addRecipe = function(recipe) { - if (this.recipes == null) { - this.recipes = []; - } - this.recipes.push(recipe); - - }; +function KarmelFile() { - this.containsRecipe = function(recipe) { +} - if (this.recipes == null) { - return false; - } +// =========================================== Karamelized Cookbooks ============================================== // +function KaramelizedCookbooks() { + this.cookbookName; + this.metadataRb; + this.karamelFile; - for (var i = 0; i < this.recipes.length; i++) { - if (recipe.title === this.recipes[i].title) { - console.log(" Inside Comparison." + " Received: " + recipe.title + " Present: " + this.recipes[i].title); - return true; + this.equals = function(other) { + if (other != null) { + if (this.id === other.id) { + return true; + } } - } - return false; - }; - + return false; + }; - this.removeRecipe = function(receipe) { - var id = -1; - for (var i = 0; i < this.recipes.length; i++) { - if (this.recipes[i].title == receipe.title) { - id = i; - break; - } - } + // Load data into the cookbook. + this.load = function(other) { + this.github = other.github; + this.branch = other.branch; + }; - // If any recipe found. - if (id != -1) { - this.recipes.splice(id, 1); - } - }; + this.copy = function(other) { + this.github = other.github; + this.branch = other.branch; + }; } // ============================================================== GROUP =============================== // function Group() { - this.name = ""; - this.provider = ""; - this.attrs = []; - this.size = 0; - this.ec2 = {}; - this.gce = {}; - this.nova = {}; - this.occi = {}; - this.baremetal = {}; + this.name = ""; + this.provider = ""; + this.attrs = []; + this.size = 0; + this.ec2 = {}; + this.gce = {}; + this.nova = {}; + this.occi = {}; + this.baremetal = {}; this.cookbooks = []; this.addCookbook = function(cookbook) { diff --git a/karamel-ui/src/main/resources/assets/karamel/board/launch/launch.controller.js b/karamel-ui/src/main/resources/assets/karamel/board/launch/launch.controller.js index 36f35d2a..6362a0d7 100644 --- a/karamel-ui/src/main/resources/assets/karamel/board/launch/launch.controller.js +++ b/karamel-ui/src/main/resources/assets/karamel/board/launch/launch.controller.js @@ -8,16 +8,16 @@ angular.module('main.module') } function _setUpHolderMap(map) { - if (info.cluster.hasEc2()) { + if (info.cluster.ec2 != null) { map[info.cluster.ec2.getMapKey()] = info.cluster.ec2; } - if (info.cluster.hasGce()) { + if (info.cluster.gce != null) { map[info.cluster.gce.getMapKey()] = info.cluster.gce; } - if (info.cluster.hasNova()) { + if (info.cluster.nova != null) { map[info.cluster.nova.getMapKey()] = info.cluster.nova; } - if (info.cluster.hasOcci()) { + if (info.cluster.occi != null) { map[info.cluster.occi.getMapKey()] = info.cluster.occi; } map[info.cluster.sshKeyPair.getMapKey()] = info.cluster.sshKeyPair; diff --git a/karamel-ui/src/main/resources/assets/karamel/board/yaml-uploader.directive.js b/karamel-ui/src/main/resources/assets/karamel/board/yaml-uploader.directive.js index 435a3ad1..8dcd91a6 100644 --- a/karamel-ui/src/main/resources/assets/karamel/board/yaml-uploader.directive.js +++ b/karamel-ui/src/main/resources/assets/karamel/board/yaml-uploader.directive.js @@ -25,10 +25,10 @@ angular.module('main.module') try { $log.info(data); - var cluster = new Cluster(); - cluster.load(data); - $rootScope.activeCluster = cluster; - $rootScope.context = cluster.name; +// var cluster = new Cluster(); +// cluster.load(data); + $rootScope.activeCluster = data; + $rootScope.context = data.name; alertService.addAlert({type: 'success', msg: 'Model Created Successfully.'}); } catch (err) { diff --git a/karamel-ui/src/main/resources/assets/karamel/core-rest.service.js b/karamel-ui/src/main/resources/assets/karamel/core-rest.service.js index ef4f8e3d..175b07eb 100644 --- a/karamel-ui/src/main/resources/assets/karamel/core-rest.service.js +++ b/karamel-ui/src/main/resources/assets/karamel/core-rest.service.js @@ -19,8 +19,7 @@ angular.module('main.module') /* window.location.hostname for the webserver */ - - var _defaultHost = 'http://' + $location.host() + ':9090/api'; + var _defaultHost = 'http://' + $location.host() + ':' + $location.port() + '/api'; var _defaultContentType = 'application/json'; diff --git a/pom.xml b/pom.xml index bf58b043..686e0ec8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ se.kth.karamel karamel-parent pom - 0.5 + 0.10 karamel-parent http://maven.apache.org @@ -18,7 +18,7 @@ UTF-8 - 2.1.0 + 2.1.2 1.8 UTF-8