forked from ThinkBigAnalytics/scalding-workshop
-
Notifications
You must be signed in to change notification settings - Fork 6
/
run.rb
executable file
·70 lines (60 loc) · 2.05 KB
/
run.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
#!/usr/bin/env ruby
#-------------------------------------------
# run.rb - Simple driver for the Scalding workshop.
# usage: run.rb scaldingscript.scala [options]
#
# Run.rb is an "approximate" port of the Scala "run" script, suitable for
# users with Ruby installed Run.rb verifies that you passed the name of
# a Scalding script (a Scala source file) as an argument, compiles it,
# and invokes Scalding in "local" mode. Scalding comes with a more
# sophisticated driver script called "scald.rb". For example, scald.rb
# handles invoking Scalding scripts as Hadoop jobs. This script is simpler
# and avoids some issues using "scald.rb".
# TODO: The VERSION and SCALA values should be detected automatically.
VERSION = "0.4.1"
SCALA = "2.11"
# Increase (or decrease) this heap size value if necessary.
HEAP = "-Xmx1g"
ASSEMBLY = "target/scala-#{VERSION}/scalding-workshop-#{VERSION}.jar"
LIBS = "lib/*"
$LOAD_PATH << File.join(File.expand_path(File.dirname(File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__)), 'lib')
require 'fileutils'
def run_command(command)
puts "Running: #{command}"
`#{command}`
unless $?.success?
puts "Failed!"
exit(1)
end
end
# TODO: Use a real option processor.
if ARGV.length == 0
puts "Must specify a Scalding script!"
exit(1)
end
script = ARGV[0]
unless script =~ /\//
script="scripts/#{script}"
end
ARGV.shift
classfile = File.basename(script, ".scala")
now = Time.now.strftime("%Y%m%d-%H%S%M")
tmpnow = "tmp/#{now}"
status = 0
begin
unless File.exists?(ASSEMBLY)
puts "You must build the all-inclusive 'assembly' (#{ASSEMBLY}) first."
puts "See the README for instructions."
exit(1)
end
FileUtils.mkdir_p(tmpnow)
puts "Compiling script \"#{script}\""
run_command("scalac -deprecation -cp 'classes:#{ASSEMBLY}:#{LIBS}' -d #{tmpnow} #{script}")
run_command("java #{HEAP} -cp 'classes:#{ASSEMBLY}:#{LIBS}:#{tmpnow}' com.twitter.scalding.Tool #{classfile} --local #{ARGV.join(" ")}")
rescue Exception => e
puts "Exception #{e} raised!"
status = 1
ensure
FileUtils.remove_dir(tmpnow) if File.exists?(tmpnow)
end
exit(status)