Modified scripts to fix minor bug

openzipkin · Jul 20, 2012 · d7a71a5 · d7a71a5
1 parent 2031a53
commit d7a71a5
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 16 deletions.
diff --git a/zipkin-hadoop/src/main/scala/com/twitter/zipkin/hadoop/sources/SpanSource.scala b/zipkin-hadoop/src/main/scala/com/twitter/zipkin/hadoop/sources/SpanSource.scala
@@ -83,31 +83,29 @@ trait LzoTsv extends DelimitedScheme {
  */
 case class SpanSource(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[Span]("/logs/zipkin/", dateRange)
 
-case class SpanSource1(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[Span]("good_data", dateRange)
-
 case class FixedSpanSource(p : String) extends FixedPathSource(p) with LzoThrift[Span] {
   def column = classOf[Span]
 }
 
 /**
  * This is the source for trace data that has been merged. Directories are like in SpanSource
  */
-case class PrepNoNamesSpanSource(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[Span]("test", dateRange)
+case class PrepNoNamesSpanSource(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[Span]("Preprocessed", dateRange)
 
 /**
  * This is the source for trace data that has been merged and for which we've found
  * the best possible client side and service names. Directories are like in SpanSource
  */
-case class PreprocessedSpanSource(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[SpanServiceName]("testagain", dateRange)
+case class PreprocessedSpanSource(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[SpanServiceName]("FindNames", dateRange)
 
-case class PreprocessedSpanSourceTest(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[SpanServiceName]("testalpha", dateRange)
+case class PreprocessedSpanSourceTest(implicit dateRange: DateRange) extends HourlySuffixLzoThrift[SpanServiceName]("FindIDtoNames", dateRange)
 
 /**
  * This is the source for data of the form (id, service name)
  */
 
 case class PrepTsvSource()(implicit dateRange : DateRange)
-  extends DailySuffixSource("id_names", dateRange)
+  extends HourlySuffixSource("id_names", dateRange)
   with LzoTsv
   with Mappable[(Long, String)]
   with SuccessFileSource {

diff --git a/zipkin-hadoop/src/scripts/run.sh → zipkin-hadoop/src/scripts/run_all_jobs.sh b/zipkin-hadoop/src/scripts/run.sh → zipkin-hadoop/src/scripts/run_all_jobs.sh
@@ -17,7 +17,7 @@ TIMES=
 STARTTIME=
 ENDTIME=
 OUTPUT=
-while getopts “hn:d:o:” OPTION
+while getopts "hd:o:" OPTION
 do
      case $OPTION in
          h)
@@ -56,15 +56,25 @@ echo "Start and end time: $STARTTIME - $ENDTIME"
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-echo $DIR
+echo "From run.sh $DIR"
+
+#ENDTIME="2012-07-19T01:00"
+
+$DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.sources.Preprocessed --date $ENDTIME $ENDTIME
+$DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.WorstRuntimes --date $ENDTIME $ENDTIME --output $OUTPUT/WorstRuntimes
 
-$DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.sources.Preprocessed --date $STARTTIME $ENDTIME
 $DIR/run_job.sh -j WorstRuntimes -d $ENDTIME -o $OUTPUT/WorstRuntimes &
-$DIR/run_job.sh -j MemcacheRequest -d $ENDTIME -o $OUTPUT/MemcacheRequest &
-$DIR/run_job.sh -j FindNames -p -d $ENDTIME
+$DIR/run_job.sh -j MemcacheRequest -d $ENDTIME -o $OUTPUT/MemcacheRequest &#$DIR/run_job.sh -j FindNames -p -d $ENDTIME
 $DIR/run_job.sh -j PopularKeys  -d $ENDTIME -o $OUTPUT/PopularKeys &
+
+$DIR/run_job.sh -j DependencyTree  -d $ENDTIME -o $OUTPUT/DependencyTree &
+
 $DIR/run_job.sh -j PopularAnnotations  -d $ENDTIME -o $OUTPUT/PopularAnnotations  &
 $DIR/run_job.sh -j FindIDtoName -p  -d $ENDTIME
 $DIR/run_job.sh -j DependencyTree -d $ENDTIME -o $OUTPUT/DependencyTree  &
 $DIR/run_job.sh -j Timeouts -s "--error_type finagle.timeout" -o $OUTPUT/Timeouts -d $ENDTIME &
 $DIR/run_job.sh -j Timeouts -s "--error_type finagle.retry" -o $OUTPUT/Retries -d $ENDTIME &
+
+wait
+
+echo "Finished all jobs!"
diff --git a/zipkin-hadoop/src/scripts/run_job.sh b/zipkin-hadoop/src/scripts/run_job.sh
@@ -21,11 +21,11 @@ ISPREPROCESSOR=
 DATE=
 SETTINGS=
 OUTPUTDIR=
-while getopts “hj:ps:d:n:o:” OPTION
+while getopts "hj:ps:d:o:" OPTION
 do
     case $OPTION in
 	h)
-	    usage()
+	    usage
 	    exit 1
 	    ;;
         j)
@@ -53,19 +53,20 @@ echo "Date: $DATE"
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+echo "From run_job.sh $DIR"
+
 if [ "$ISPREPROCESSOR" == "1" ]; then
     $DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.sources.$JOBNAME $SETTINGS --date $DATE
     if [ "$?" != 0 ]; then
         echo "Job $JOBNAME failed; exiting"
         exit 1
     fi
 else
-    $DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.$JOBNAME $SETTINGS --date $DATE --output $OUTPUTDIR
+    $DIR/scald.rb --hdfs com.twitter.zipkin.hadoop.$JOBNAME $SETTINGS --date $DATE $DATE --output $OUTPUTDIR
     if [ "$?" == "0" ]; then
         echo "Job $JOBNAME succesfully completed"
     else
         echo "Job $JOBNAME failed; exiting"
         exit 1
     fi
-fi
-
+fi