No remote with conf PR

README.md

@@ -14,10 +14,55 @@ Karamel introduces Karamelfile to orchestrate the execution of Chef recipes. Kar
 
 We leverage Berkshelf to transparently download and install transitive cookbook dependencies, so large systems can be defined in a few lines of code. Finally, the Karamel runtime builds and manages the execution of the DAG of Chef recipes by executing them with Chef Solo. 
 
+### Configuration
+#### Karamel working directory
+By default Karamel (local & remote) working directory is `$HOME/.karamel` If you want to override this directory
+use the environment variable `KARAMEL_WORKING_DIRECTORY` such as `export 
+KARAMEL_WORKING_DIRECTORY=/tmp/karamel_directory`
+
+#### Chef file cache
+Chef will download files to `/tmp/chef-solo` before using them. To change the staging location use the
+environment variable `CHEF_FILE_CACHE_PATH` such as `export CHEF_FILE_CACHE_PATH=/run/chef-solo`
+
+#### Elevated privileges program
+Karamel requires to run with elevated privileges. The default program for that is `sudo` but if your
+organization uses another program set it with the environment variable `CHEF_SUDO_BINARY` such as
+`export CHEF_SUDO_BINARY=dzdo`
+
+#### Airgapped installations
+If you are installing Hopsworks on an environment with no internet connectivity set `KARAMEL_AIRGAP=true`
+
+#### Ruby Gems server
+To point Chef to a local Ruby gems server instead of https://rubygems.org export 
+`CHEF_RUBYGEMS_URL=http://IP_ADDRESS:8808`
+
 ### Cluster Definition
 ---
 Cluster definition is an expressive DSL based on YAML as you can see in the following sample. Since Karamel can run several clusters simultaneously, name of the cluster must be unique in each Karamel-runtime.
 
+#### Recipes execution parallelism
+Karamel will execute recipes in parallel on all machines respecting any dependency stated in Karamelfile. Regardless 
+of the dependencies, the same recipes will be executed in parallel on all machines. For example if we run recipe 
+`ndb::ndbd` on three machines, they may be executed in parallel depending on how Karamel will schedule them.
+
+In some cases we want to guarantee only a portion of machines to execute recipes in parallel. For example in case of 
+a rolling restart/upgrade we want to execute `ndb::ndbd` serially. You can configure recipe parallelism under 
+`runtimeConfiguration/recipesParallelism` section in the cluster definition in the format `recipe_name: parallelism`.
+Multiple recipes can be configured. For example:
+
+```yaml
+runtimeConfiguration:                                                                                                                                                                                                                         
+  recipesParallelism:                                                                                                                                                                                                                         
+    consul::master: 1                                                                                                                                                                                                                         
+    ndb::ndbd: 1                                                                                                                                                                                                                         
+    ndb::mysqld: 2 
+    ndb::mysqld_tls: 2 
+    onlinefs::default: 1 
+    hops::nn: 1                                                                                                                                                                                                                               
+    hops::rm: 1
+```
+
+#### Cloud providers
 We support five cloud providers: Amazon EC2 (ec2), Google Compute Engine (gce), Openstack Nova (nova), OCCI and bare-metal (baremetal). You can define provider globally or per group. In the group scope, you can overwrite some attributes of the network/machines in the global scope or you can entirely choose another cloud provider, that's how we support multi-cloud deployment. Settings and properties for each provider is introduced in a following separate section. 
 
  Cookbooks section introduces github references to the used cookbooks, it is also possible to refer to a specific version or branch for each github repository.

karamel-common/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>se.kth.karamel</groupId>
     <artifactId>karamel-parent</artifactId>
-    <version>0.5</version>
+    <version>0.10</version>
   </parent>
   <groupId>se.kth.karamel</groupId>
   <artifactId>karamel-common</artifactId>

karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Cookbook.java

@@ -1,39 +1,15 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
 package se.kth.karamel.common.clusterdef;
 
-import se.kth.karamel.common.cookbookmeta.CookbookUrls;
-import se.kth.karamel.common.exception.CookbookUrlException;
-
-/**
- *
- * @author kamal
- */
 public class Cookbook {
 
   private String github;
-  private String version;
   private String branch;
-  private String cookbook;
 
   public Cookbook() {
   }
 
-  public Cookbook(Cookbook cookbook) {
-    this.github = cookbook.getGithub();
-    this.version = cookbook.getVersion();
-    this.branch = cookbook.getBranch();
-    this.cookbook = cookbook.cookbook;
-  }
-
-  public String getBranch() {
-    return branch;
-  }
-
-  public void setBranch(String branch) {
+  public Cookbook(String github, String branch) {
+    this.github = github;
     this.branch = branch;
   }
 
@@ -45,32 +21,11 @@ public void setGithub(String github) {
     this.github = github;
   }
 
-  public String getVersion() {
-    return version;
-  }
-
-  public void setVersion(String version) {
-    this.version = version;
-  }
-
-  public String getCookbook() {
-    return cookbook;
+  public String getBranch() {
+    return branch;
   }
 
-  public void setCookbook(String cookbook) {
-    this.cookbook = cookbook;
-  }
-
-  public CookbookUrls getUrls() throws CookbookUrlException {
-    CookbookUrls.Builder builder = new CookbookUrls.Builder();
-    builder.url(github);
-    if (branch != null)
-      builder.branchOrVersion(branch);
-    else if (version != null)
-      builder.branchOrVersion(version);
-    if (cookbook != null)
-      builder.cookbookRelPath(cookbook);
-    CookbookUrls urls = builder.build();
-    return urls;
+  public void setBranch(String branch) {
+    this.branch = branch;
   }
 }

karamel-common/src/main/java/se/kth/karamel/common/clusterdef/Scope.java

@@ -5,6 +5,7 @@
  */
 package se.kth.karamel.common.clusterdef;
 
+import se.kth.karamel.common.clusterdef.yaml.RuntimeConfiguration;
 import se.kth.karamel.common.exception.ValidationException;
 
 /**
@@ -16,6 +17,7 @@ public abstract class Scope {
   private Ec2 ec2;
   private Vagrant vagrant;
   private Baremetal baremetal;
+  private RuntimeConfiguration runtimeConfiguration = new RuntimeConfiguration();
   private Gce gce;
   private Nova nova;
   private Occi occi;
@@ -27,6 +29,7 @@ public Scope(Scope scope) {
     this.ec2 = scope.getEc2();
     this.vagrant = scope.getVagrant();
     this.baremetal = scope.getBaremetal();
+    this.runtimeConfiguration = scope.getRuntimeConfiguration();
     this.gce = scope.getGce();
     this.nova = scope.getNova();
     this.occi = scope.getOcci();
@@ -81,17 +84,17 @@ public Gce getGce() {
   public void setGce(Gce gce) {
     this.gce = gce;
   }
-  
+
   public Occi getOcci() {
     return occi;
   }
-  
+
   public void setOcci(Occi occi) {
     this.occi = occi;
   }
 
   public Nova getNova() {return nova;}
-  
+
   public void setNova(Nova nova) {
     this.nova = nova;
   }
@@ -116,5 +119,12 @@ public void validate() throws ValidationException {
       occi.validate();
     }
   }
-;
+
+  public RuntimeConfiguration getRuntimeConfiguration() {
+    return runtimeConfiguration;
+  }
+
+  public void setRuntimeConfiguration(RuntimeConfiguration runtimeConfiguration) {
+    this.runtimeConfiguration = runtimeConfiguration;
+  }
 }

karamel-common/src/main/java/se/kth/karamel/common/clusterdef/json/JsonCluster.java

@@ -1,40 +1,61 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
 package se.kth.karamel.common.clusterdef.json;
 
+import se.kth.karamel.common.clusterdef.Cookbook;
 import se.kth.karamel.common.clusterdef.yaml.YamlCluster;
 import se.kth.karamel.common.clusterdef.yaml.YamlGroup;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+
+import se.kth.karamel.common.cookbookmeta.Attribute;
+import se.kth.karamel.common.cookbookmeta.KaramelizedCookbook;
 import se.kth.karamel.common.exception.KaramelException;
 
-/**
- *
- * @author kamal
- */
 public class JsonCluster extends JsonScope {
 
   private String name;
-
+  private Map<String, Cookbook> rootCookbooks = new HashMap<>();
   private List<JsonGroup> groups = new ArrayList<>();
 
   public JsonCluster() {
   }
 
   public JsonCluster(YamlCluster cluster) throws KaramelException {
-    super(cluster, cluster);
-    this.name = cluster.getName();
+    super(cluster);
+    name = cluster.getName();
+    rootCookbooks = cluster.getCookbooks();
+    attributes = cluster.flattenAttrs();
+    Set<Attribute> validAttrs = new HashSet<>();
+
+    cookbooks.addAll(CACHE.loadAllKaramelizedCookbooks(cluster));
+
+    //filtering invalid(not defined in metadata.rb) attributes from yaml model
+    // Get all the valid attributes, also for transient dependency
+    for (KaramelizedCookbook kcb : cookbooks) {
+      validAttrs.addAll(kcb.getMetadataRb().getAttributes());
+    }
+
+    // TODO(Fabio): I think that this map should be <String, Attribute>. But I don't want to see
+    // what happen if I change it.
+    Map<String, Object> invalidAttrs = new HashMap<>();
+
+    for (String usedAttr: attributes.keySet()) {
+      if (!validAttrs.contains(new Attribute(usedAttr))) {
+        invalidAttrs.put(usedAttr, attributes.get(usedAttr));
+      }
+    }
+
+    if (!invalidAttrs.isEmpty()) {
+      throw new KaramelException(String.format("Invalid attributes, all used attributes must be defined in metadata.rb "
+          + "files: %s", invalidAttrs.keySet().toString()));
+    }
+
     Set<Map.Entry<String, YamlGroup>> entrySet = cluster.getGroups().entrySet();
     for (Map.Entry<String, YamlGroup> entry : entrySet) {
-      String gName = entry.getKey();
-      YamlGroup group = entry.getValue();
-      JsonGroup jsonGroup = new JsonGroup(cluster, group, gName);
-      this.groups.add(jsonGroup);
+      groups.add(new JsonGroup(entry.getValue(), entry.getKey(), cookbooks));
     }
 
   }
@@ -55,4 +76,11 @@ public void setGroups(List<JsonGroup> groups) {
     this.groups = groups;
   }
 
+  public Map<String, Cookbook> getRootCookbooks() {
+    return rootCookbooks;
+  }
+
+  public void setRootCookbooks(Map<String, Cookbook> rootCookbooks) {
+    this.rootCookbooks = rootCookbooks;
+  }
 }