From 87d49a704b13f3a80c9a46e00b9dd5cf61fc933d Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Wed, 7 Jul 2021 21:52:41 -0400 Subject: [PATCH 1/2] Add support for dynamic k8s labels --- .../kubernetes/KubernetesContext.java | 43 +++++++++++-------- .../scheduler/kubernetes/V1Controller.java | 16 ++++--- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java index f6359b82982..b87b26705a0 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java @@ -92,6 +92,11 @@ public enum KubernetesResourceRequestMode { "heron.kubernetes.pod.annotation."; public static final String HERON_KUBERNETES_SERVICE_ANNOTATION = "heron.kubernetes.service.annotation."; + public static final String HERON_KUBERNETES_POD_LABEL = + "heron.kubernetes.pod.annotation."; + public static final String HERON_KUBERNETES_SERVICE_LABEL = + "heron.kubernetes.service.annotation."; + private KubernetesContext() { } @@ -162,6 +167,22 @@ static String getContainerVolumeMountPath(Config config) { return config.getStringValue(HERON_KUBERNETES_CONTAINER_VOLUME_MOUNT_PATH); } + public static Map getPodLabels(Config config) { + return getConfigItemsByPrefix(config, HERON_KUBERNETES_POD_LABEL); + } + + public static Map getServiceLabels(Config config) { + return getConfigItemsByPrefix(config, HERON_KUBERNETES_SERVICE_LABEL); + } + + public static Map getPodAnnotations(Config config) { + return getConfigItemsByPrefix(config, HERON_KUBERNETES_POD_ANNOTATION); + } + + public static Map getServiceAnnotations(Config config) { + return getConfigItemsByPrefix(config, HERON_KUBERNETES_SERVICE_ANNOTATION); + } + static Set getConfigKeys(Config config, String keyPrefix) { Set annotations = new HashSet<>(); for (String s : config.getKeySet()) { @@ -172,26 +193,14 @@ static Set getConfigKeys(Config config, String keyPrefix) { return annotations; } - public static Map getPodAnnotations(Config config) { - final Map annotations = new HashMap<>(); - final Set keys = getConfigKeys(config, HERON_KUBERNETES_POD_ANNOTATION); - for (String s : keys) { - String value = config.getStringValue(s); - annotations.put(s.replaceFirst(KubernetesContext.HERON_KUBERNETES_POD_ANNOTATION, - ""), value); - } - return annotations; - } - - public static Map getServiceAnnotations(Config config) { - final Map annotations = new HashMap<>(); - final Set keys = getConfigKeys(config, HERON_KUBERNETES_SERVICE_ANNOTATION); + private static Map getConfigItemsByPrefix(Config config, String keyPrefix) { + final Map results = new HashMap<>(); + final Set keys = getConfigKeys(config, keyPrefix); for (String s : keys) { String value = config.getStringValue(s); - annotations.put(s.replaceFirst(KubernetesContext.HERON_KUBERNETES_SERVICE_ANNOTATION, - ""), value); + results.put(s.replaceFirst(keyPrefix, ""), value); } - return annotations; + return results; } public static boolean hasContainerVolume(Config config) { diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java index 59399b64ec8..2bf815b623f 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/V1Controller.java @@ -327,12 +327,13 @@ private V1Service createTopologyService() { final V1ObjectMeta objectMeta = new V1ObjectMeta(); objectMeta.name(topologyName); objectMeta.annotations(getServiceAnnotations()); + objectMeta.setLabels(getServiceLabels()); service.setMetadata(objectMeta); // create the headless service final V1ServiceSpec serviceSpec = new V1ServiceSpec(); serviceSpec.clusterIP("None"); - serviceSpec.setSelector(getMatchLabels(topologyName)); + serviceSpec.setSelector(getPodMatchLabels(topologyName)); service.setSpec(serviceSpec); @@ -363,14 +364,14 @@ private V1StatefulSet createStatefulSet(Resource containerResource, int numberOf // add selector match labels "app=heron" and "topology=topology-name" // so the we know which pods to manage final V1LabelSelector selector = new V1LabelSelector(); - selector.matchLabels(getMatchLabels(topologyName)); + selector.matchLabels(getPodMatchLabels(topologyName)); statefulSetSpec.selector(selector); // create a pod template final V1PodTemplateSpec podTemplateSpec = new V1PodTemplateSpec(); // set up pod meta - final V1ObjectMeta templateMetaData = new V1ObjectMeta().labels(getLabels(topologyName)); + final V1ObjectMeta templateMetaData = new V1ObjectMeta().labels(getPodLabels(topologyName)); Map annotations = new HashMap<>(); annotations.putAll(getPodAnnotations()); annotations.putAll(getPrometheusAnnotations()); @@ -408,20 +409,25 @@ private Map getPrometheusAnnotations() { return annotations; } - private Map getMatchLabels(String topologyName) { + private Map getPodMatchLabels(String topologyName) { final Map labels = new HashMap<>(); labels.put(KubernetesConstants.LABEL_APP, KubernetesConstants.LABEL_APP_VALUE); labels.put(KubernetesConstants.LABEL_TOPOLOGY, topologyName); return labels; } - private Map getLabels(String topologyName) { + private Map getPodLabels(String topologyName) { final Map labels = new HashMap<>(); labels.put(KubernetesConstants.LABEL_APP, KubernetesConstants.LABEL_APP_VALUE); labels.put(KubernetesConstants.LABEL_TOPOLOGY, topologyName); + labels.putAll(KubernetesContext.getPodLabels(getConfiguration())); return labels; } + private Map getServiceLabels() { + return KubernetesContext.getServiceLabels(getConfiguration()); + } + private V1PodSpec getPodSpec(List executorCommand, Resource resource, int numberOfInstances) { final V1PodSpec podSpec = new V1PodSpec(); From 96670538c84d078cb3ed416547a8c2da3583ef91 Mon Sep 17 00:00:00 2001 From: Nicholas Nezis Date: Thu, 8 Jul 2021 09:26:38 -0400 Subject: [PATCH 2/2] Typo fix --- .../apache/heron/scheduler/kubernetes/KubernetesContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java index b87b26705a0..4074fbc833d 100644 --- a/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java +++ b/heron/schedulers/src/java/org/apache/heron/scheduler/kubernetes/KubernetesContext.java @@ -93,9 +93,9 @@ public enum KubernetesResourceRequestMode { public static final String HERON_KUBERNETES_SERVICE_ANNOTATION = "heron.kubernetes.service.annotation."; public static final String HERON_KUBERNETES_POD_LABEL = - "heron.kubernetes.pod.annotation."; + "heron.kubernetes.pod.label."; public static final String HERON_KUBERNETES_SERVICE_LABEL = - "heron.kubernetes.service.annotation."; + "heron.kubernetes.service.label."; private KubernetesContext() {