Skip to content
This repository has been archived by the owner on Jan 3, 2023. It is now read-only.

Commit

Permalink
Refine storage utilization page & node info page (#2050)
Browse files Browse the repository at this point in the history
  • Loading branch information
archichen authored and PHILO-HE committed Apr 1, 2019
1 parent e6f6c47 commit 5ca7804
Show file tree
Hide file tree
Showing 16 changed files with 427 additions and 37 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@ smart-zeppelin/zeppelin-web/.flattened-pom.xml
smart-zeppelin/zeppelin-web/.tmp
smart-zeppelin/zeppelin-web/bower_components
smart-zeppelin/zeppelin-web/dist
smart-zeppelin/zeppelin-web/src/assets/resp.json
smart-zeppelin/zeppelin-web/node
smart-zeppelin/zeppelin-web/node_modules
smart-zeppelin/zeppelin-web/reports
smart-zeppelin/zeppelin-web/src/fonts
smart-zeppelin/zeppelin-web/yarn-error.log
smart-zeppelin/zeppelin-web/yarn.lock
smart-zeppelin/zeppelin-web/package-lock.json
smart-zeppelin/zeppelin-zengine/.flattened-pom.xml
smart-zeppelin/zeppelin-distribution/.flattened-pom.xml
92 changes: 92 additions & 0 deletions smart-common/src/main/java/org/smartdata/NodeHosts.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.smartdata;

import org.smartdata.conf.SmartConf;
import org.smartdata.conf.SmartConfKeys;

import java.io.File;
import java.io.FileNotFoundException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;

public class NodeHosts {

private SmartConf conf;
public Set<String> agentHosts;
public Set<String> serverHosts;

public NodeHosts(SmartConf conf) {
this.conf = conf;
this.serverHosts = init("server");
this.agentHosts = init("agent");
}

public Set<String> init(String role) {
String fileName = "/agents";
switch (role) {
case "agent":
fileName = "/agents";
break;
case "server":
fileName = "/servers";
break;
}
String hostName = "";
try {
InetAddress address = InetAddress.getLocalHost();
hostName = address.getHostName();
} catch (UnknownHostException e) {
e.printStackTrace();
}

String agentConfFile = conf.get(SmartConfKeys.SMART_CONF_DIR_KEY,
SmartConfKeys.SMART_CONF_DIR_DEFAULT) + fileName;
Scanner sc = null;
HashSet<String> hosts = new HashSet<>();
try {
sc = new Scanner(new File(agentConfFile));
} catch (FileNotFoundException ex) {
ex.printStackTrace();
}

while (sc != null && sc.hasNextLine()) {
String host = sc.nextLine().trim();
if (!host.startsWith("#") && !host.isEmpty()) {
if (host.equals("localhost")) {
hosts.add(hostName);
} else {
hosts.add(host);
}
}
}

return hosts;
}

public Set<String> getServerHosts() {
return serverHosts;
}

public Set<String> getAgentHosts() {
return agentHosts;
}
}
13 changes: 13 additions & 0 deletions smart-engine/src/main/java/org/smartdata/server/SmartEngine.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.smartdata.AbstractService;
import org.smartdata.NodeHosts;
import org.smartdata.conf.SmartConf;
import org.smartdata.model.StorageCapacity;
import org.smartdata.model.Utilization;
Expand All @@ -36,11 +37,13 @@
import org.smartdata.server.engine.cmdlet.agent.AgentInfo;

import java.io.IOException;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.Set;

public class SmartEngine extends AbstractService {
private ConfManager confMgr;
Expand All @@ -50,6 +53,7 @@ public class SmartEngine extends AbstractService {
private RuleManager ruleMgr;
private CmdletManager cmdletManager;
private AgentExecutorService agentService;
private NodeHosts nodeHosts;
private HazelcastExecutorService hazelcastService;
private List<AbstractService> services = new ArrayList<>();
public static final Logger LOG = LoggerFactory.getLogger(SmartEngine.class);
Expand All @@ -67,6 +71,7 @@ public void init() throws IOException {
cmdletManager = new CmdletManager(serverContext);
services.add(cmdletManager);
agentService = new AgentExecutorService(conf, cmdletManager);
nodeHosts = new NodeHosts(conf);
hazelcastService = new HazelcastExecutorService(cmdletManager);
cmdletManager.registerExecutorService(agentService);
cmdletManager.registerExecutorService(hazelcastService);
Expand Down Expand Up @@ -120,6 +125,14 @@ public List<StandbyServerInfo> getStandbyServers() {
return hazelcastService.getStandbyServers();
}

public Set<String> getAgentHosts() {
return nodeHosts.getAgentHosts();
}

public Set<String> getServerHosts() {
return nodeHosts.getServerHosts();
}

public List<AgentInfo> getAgents() {
return agentService.getAgentInfos();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,16 @@ public Response agents() {
// return list of agents and their states
return new JsonResponse<>(Response.Status.OK, smartEngine.getAgents()).build();
}

@GET
@Path("/allAgentHosts")
public Response allAgentHosts() {
return new JsonResponse<>(Response.Status.OK, smartEngine.getAgentHosts()).build();
}

@GET
@Path("/allServerHosts")
public Response allMasterHosts() {
return new JsonResponse<>(Response.Status.OK, smartEngine.getServerHosts()).build();
}
}
8 changes: 7 additions & 1 deletion smart-zeppelin/zeppelin-web/src/app/app.js
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,13 @@ var zeppelinWebApp = angular.module('zeppelinWebApp', [
resolve: {
nodes0: ['models', function (models) {
return models.$get.nodes();
}]
}],
serverHosts: ['models', function (models) {
return models.$get.serverHosts();
}],
agentHosts: ['models', function (models) {
return models.$get.agentHosts();
}],
}
})
.when('/jobmanager', {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ angular.module('org.apache.hadoop.ssm.models', [])

.factory('models', ['$timeout', 'conf', 'restapi', 'locator',
function ($timeout, conf, restapi, locator) {
'use strict';

var util = {
usage: function (current, total) {
Expand Down Expand Up @@ -263,6 +262,12 @@ angular.module('org.apache.hadoop.ssm.models', [])
nodes: function () {
return get('cluster/primary/ssmnodescmdletmetrics', decoder.nodes);
},
agentHosts: function () {
return get('system/allAgentHosts', decoder.nodes);
},
serverHosts: function () {
return get('system/allServerHosts', decoder.nodes);
},
notebookInfo: function () {
return get('note/info', decoder.default)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
.statusPoint {
width: 15px;
height: 15px;
border-radius: 15px;
}

.statusPoint-container {
display: flex;
justify-content: center;
align-items: center;
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,37 @@
specific language governing permissions and limitations
under the License.
-->
<div class="col-md-12">
<div class="col-md-2"></div>
<div class="col-md-8">
<br/>
<table class="table table-bordered">
<caption class="text-center"><h4>Node Info</h4></caption>
<table class="table table-bordered table-hover table-condensed" style="margin-bottom: 40px;margin-top: 40px">
<!--<caption class="text-center" style="margin-bottom: 20px; float: left"><h3><strong>Node Info</strong></h3></caption>-->
<tr>
<td class="col-md-5">Live Nodes</td>
<td>{{liveNumber}}</td>
</tr>
<tr class="{{deadNumber !== 0 ? 'danger' : ''}}">
<td>Dead Nodes</td>
<td>{{deadNumber}}</td>
</tr>
<tr>
<td>Pending Cmdlets for Scheduling</td>
<td>{{(nodes[0].nodeInfo.executorType == "LOCAL" ? nodes[0].numPendingSchedule : "")}}</td>
</tr>
<tr>
<td>Pending Cmdlets for Dispatching</td>
<td>{{(nodes[0].nodeInfo.executorType == "LOCAL" ? nodes[0].numPendingDispatch : "")}}</td>
</tr>
</table>
<table class="table table-bordered table-hover table-condensed">
<thead ng-show="nodes.length > 0">
<tr>
<th>Id</th>
<th>Regist Time</th>
<th>Number of Executors</th>
<th>ID</th>
<th>Register Time</th>
<th>Executors</th>
<th>Cmdlets in Execution</th>
<th>Cmdlets Executed</th>
<th>Cmdlets In Execution</th>
<th>Status</th>
</tr>
</thead>
<thead ng-if="nodes.length <= 0">
Expand All @@ -35,21 +55,25 @@
<div class="table-no-data" style="width: 100%;">
<h2 class="glyphicon glyphicon-bullhorn"></h2>
<h4>No node is running</h4>

<p>Please start a node first.</p>
</div>
</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="node in nodes">
<td>{{node.nodeInfo.id + (node.nodeInfo.executorType == "LOCAL" ? " (PendingSchedule:" + node.numPendingSchedule + ", PendingDispatch:" + node.numPendingDispatch + ")" : "")}}</td>
<td>{{node.registTime}}</td>
<td>{{node.numExecutors}}</td>
<td>{{node.cmdletsExecuted}}</td>
<td>{{node.cmdletsInExecution}}</td>
<tbody ng-init="index=-1">
<tr ng-repeat="node in nodes" ng-init="isLive=node.isLive">
<td>{{
isLive
? (node.nodeInfo.id)
: node.nodeInfo.host + (node.type === "agent" ? "@agent" : "@server")
}}</td>
<td>{{isLive ? node.registTime : '-'}}</td>
<td>{{isLive ? node.numExecutors : '-'}}</td>
<td>{{isLive ? node.cmdletsInExecution : '-'}}</td>
<td>{{isLive ? node.cmdletsExecuted : '-'}}</td>
<td class="statusPoint-container"><div class="statusPoint {{isLive ? 'btn-success' : 'btn-danger'}}"></div></td>
</tr>
</tbody>
</table>
</div>

<div class="col-md-2"></div>
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,59 @@
angular.module('zeppelinWebApp')

.controller('NodesCtrl', NodesCtrl);
NodesCtrl.$inject = ['$scope', '$filter', 'nodes0'];
function NodesCtrl($scope, $filter, nodes0) {
NodesCtrl.$inject = ['$scope', '$filter', 'nodes0', 'serverHosts', 'agentHosts'];
function NodesCtrl($scope, $filter, nodes0, serverHosts, agentHosts) {
$scope.nodes = nodes0.body;
$scope.serverHosts = serverHosts.body;
$scope.agentHosts = agentHosts.body;

angular.forEach($scope.nodes, function (data, index) {
if ('maxInExecution' in data) {
let index = $scope.serverHosts.indexOf(data.nodeInfo.host);
if (index >= 0) {
data.isLive = true;
data.type = 'server';
$scope.serverHosts.splice(index, 1);
}
} else {
let index = $scope.agentHosts.indexOf(data.nodeInfo.host);
if (index >= 0) {
data.isLive = true;
data.type = 'agent';
$scope.agentHosts.splice(index, 1);
}
}
});

$scope.liveNumber = $scope.nodes.length;
$scope.deadNumber = $scope.serverHosts.length + $scope.agentHosts.length;

angular.forEach($scope.serverHosts, function (host, index) {
$scope.nodes.push(
{
nodeInfo: {
host: host
},
isLive: false,
type: 'server'
}
);
});

angular.forEach($scope.agentHosts, function (host, index) {
$scope.nodes.push(
{
nodeInfo: {
host: host
},
isLive: false,
type: 'agent'
}
);
});

angular.forEach($scope.nodes, function (data,index) {
data.registTime = data.registTime === 0 ? "-" :
data.registTime = data.registTime === 0 ? '-' :
$filter('date')(data.registTime,'yyyy-MM-dd HH:mm:ss');
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
.thumbnail-thumbnail-margin-right {
margin-right: 10px;
}

.storage-thumbnail {
position: relative;
transition: all 0.4s;
cursor: pointer;
}

.storage-thumbnail:hover {
top: -5px;
box-shadow: 0 5px 25px -6px #8f8f8f ;
}

.nvtooltip {
z-index: 10003;
}
Loading

0 comments on commit 5ca7804

Please sign in to comment.