-
Notifications
You must be signed in to change notification settings - Fork 2
/
spa_2016.cmd
152 lines (127 loc) · 4.98 KB
/
spa_2016.cmd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
:: Name: spa_2016.cmd
:: Purpose: Windows script to start / stop Big Data services
:: Author: Nick Rozanski
:: Syntax: spa_2016.cmd start | stop | status | init_metastore | hadoop command / hadoop_browser | pyspark | beeline
@ECHO OFF
SETLOCAL ENABLEEXTENSIONS
SET script=%~n0
SET command=%~1
for /F "tokens=2 delims=:" %%i in ('"ipconfig | findstr IPv4"') do SET IP_ADDRESS=%%i
SET IP_ADDRESS=%IP_ADDRESS: =%
REM see http://superuser.com/questions/230233/how-to-get-lan-ip-to-a-variable-in-a-windows-batch-file
REM if the above doesn't work do SET IP_ADDRESS=nnn.nnn.nnn.nnn
ECHO IP address is %IP_ADDRESS%.
CALL env.cmd
IF "%command%" == "start" (CALL :start_all && EXIT /B 0)
IF "%command%" == "stop" (CALL :stop_all && EXIT /B 0)
IF "%command%" == "status" (CALL :status_all && EXIT /B 0)
IF "%command%" == "init_metastore" (CALL :init_metastore && EXIT /B 0)
IF "%command%" == "hadoop" (CALL :hadoop && EXIT /B 0)
IF "%command%" == "hadoop_browser" (CALL :hadoop_browser && EXIT /B 0)
IF "%command%" == "pyspark" (CALL :pyspark && EXIT /B 0)
IF "%command%" == "beeline" (CALL :beeline && EXIT /B 0)
ECHO Syntax: spa_2016.cmd start / stop / status / init_metastore / hadoop command / hadoop_browser / pyspark / beeline
ENDLOCAL
ECHO ON
@EXIT /B 0
:: ======================
:: Start Functions
:: ======================
:start_all
CALL :start_hadoop || GOTO :start_failed
CALL :start_spark || GOTO :start_failed
CALL :start_hive || GOTO :start_failed
ECHO Java processes:
%JAVA_HOME%\bin\jps.exe
EXIT /B 0
:start_failed
EXIT /B 1
:: ----------------------
:start_hadoop
ECHO %script%: Starting Hadoop
CALL %SPA_2016%\hadoop\sbin\start-dfs.cmd || (ECHO %script%: Failed to start Hadoop HDFS, error=%ERRORLEVEL%)
ECHO %script%: Started HDFS
CALL :sleep 5
CALL %SPA_2016%\hadoop\sbin\start-yarn.cmd || (ECHO %script%: Failed to start Hadoop YARN, error=%ERRORLEVEL%)
ECHO %script%: Started YARN
EXIT /B 0
:: ----------------------
:start_spark
ECHO %script%: Starting Spark
START %SPA_2016%\spark\bin\spark-class.cmd org.apache.spark.deploy.master.Master || (ECHO %script%: Failed to start Spark Master, error=%ERRORLEVEL%)
ECHO %script%: Started Spark Master on %IP_ADDRESS%
CALL :sleep 5
START %SPA_2016%\spark\bin\spark-class.cmd org.apache.spark.deploy.worker.Worker spark://%IP_ADDRESS%:7077 || (ECHO %script%: Failed to start Spark Worker, error=%ERRORLEVEL%)
ECHO %script%: Started Spark Worker on %IP_ADDRESS%
EXIT /B 0
:: ----------------------
:start_hive
ECHO %script%: Starting Hive Server
START %HIVE_HOME%\bin\hive.cmd --service hiveserver2 1> %SPA_2016%\logs\hive.log 2>&1 || (ECHO %script%: Failed to start Hive server, error=%ERRORLEVEL%)
ECHO %script%: Started Hive Server
EXIT /B 0
:: ======================
:: Stop Functions
:: ======================
:stop_all
ECHO %script%: Stopping Hadoop
CALL %SPA_2016%\hadoop\sbin\stop-dfs.cmd || (ECHO %script%: Failed to stop Hadoop DFS, error=%ERRORLEVEL%)
CALL %SPA_2016%\hadoop\sbin\stop-yarn.cmd || (ECHO %script%: Failed to stop Hadoop YARN, error=%ERRORLEVEL%)
ECHO Java processes:
%JAVA_HOME%\bin\jps.exe
ECHO PLEASE CLOSE THE ABOVE SPARK AND HIVE WINDOWS USING THE COMMAND:
ECHO 'taskkill /f /pid processid'
EXIT /B 0
:: ======================
:: Status Functions
:: ======================
:status_all
ECHO Java processes:
%JAVA_HOME%\bin\jps.exe
EXIT /B 0
:: ======================
:: Client Functions
:: ======================
:hadoop
ECHO %script%: Running Hadoop command %*
START %SPA_2016%\hadoop\sbin\hadoop.cmd %*
ECHO %script%: Completed Hadoop command
EXIT /B 0
:hadoop_browser
ECHO %script%: Opening Hadoop browser
SET CLIENT_URL='http://localhost:50070/explorer.html#/'
START %CLIENT_URL%
EXIT /B 0
:pyspark
ECHO %script%: Running Spark client
ECHO Press Control-D to quit
CALL %SPARK_HOME%/bin/pyspark --master spark://%IP_ADDRESS%:7077
REM START %SPARK_HOME%\bin\spark-shell.cmd
EXIT /B 0
:beeline
ECHO %script%: Running Hive client (Spark version)
ECHO Type '!quit' to quit
%JAVA_HOME%\bin\java -cp "%SPARK_HOME%\conf\;%SPARK_HOME%\lib\spark-assembly-1.6.1-hadoop2.6.0.jar;%SPARK_HOME%\lib\datanucleus-api-jdo-3.2.6.jar;%SPARK_HOME%\lib\datanucleus-core-3.2.10.jar;%SPARK_HOME%\lib\datanucleus-rdbms-3.2.9.jar" -Djline.terminal=jline.UnsupportedTerminal -Xms1g -Xmx1g org.apache.hive.beeline.BeeLine -u jdbc:hive2://
EXIT /B 0
:: ======================
:: Miscellaneous Functions
:: ======================
:init_metastore
REM SET metastore_dir=%SPA_2016%\data\hive
ECHO Does not work on Windows - please extract the TAR file in the downloads directory
REM MKDIR %metastore_dir%
REM PUSHD %metastore_dir%
REM ECHO Creating Hive metastore at %metastore_dir%\metastore_db
REM SET HADOOP=%HADOOP_HOME%\bin\hadoop.cmd
REM SET HIVE_LIB=%HIVE_HOME%\lib
REM SET HIVE_BIN_PATH=%HIVE_HOME%\bin
REM SET HIVEARGS=-initSchema -dbType derby
REM %HIVE_HOME%\bin\ext\schemaTool.cmd
REM ECHO Created Hive metastore at %metastore_dir%\metastore_db
REM DIR %metastore_dir%
REM POPD
EXIT /B 0
:sleep
ECHO %script%: Sleeping for %1 seconds
PING.EXE -N %~1 127.0.0.1 > NULL
EXIT /B 0