Merge pull request #1 from amplab/master

merge

.gitignore

@@ -5,7 +5,11 @@
 .settings/
 conf/tachyon-env.sh
 data/
+docs/_site
+docs/api
+docs/serve/
 journal/
 logs/
+script/
 tachyon.iml
 target/

README.md

@@ -1,17 +1,13 @@
 Tachyon
 =======
 
-The master branch is in version 0.4.0-SNAPSHOT:
+The master branch is in version 0.5.0-SNAPSHOT:
 
-Project Homepage: http://www.tachyonproject.org
-
-Previous Releases: https://github.com/amplab/tachyon/tags
-
-Documentation: https://github.com/amplab/tachyon/wiki
-
-JIRA: https://spark-project.atlassian.net/browse/TACHYON
-
-User Mailing List: https://groups.google.com/forum/?fromgroups#!forum/tachyon-users
+- [Project Homepage](http://www.tachyonproject.org)
+- [Previous Releases](https://github.com/amplab/tachyon/tags)
+- [Master Branch Document](http://tachyon-project.org/master/)
+- [Tachyon Issue Tracker](https://spark-project.atlassian.net/browse/TACHYON)
+- [User Mailing List](https://groups.google.com/forum/?fromgroups#!forum/tachyon-users)
 
 ## Dependency Information
 
@@ -20,18 +16,18 @@ User Mailing List: https://groups.google.com/forum/?fromgroups#!forum/tachyon-us
 <dependency>
   <groupId>org.tachyonproject</groupId>
   <artifactId>tachyon</artifactId>
-  <version>0.3.0</version>
+  <version>0.4.1</version>
 </dependency>
 ```
 
 ### Apache Ant
 ```xml
-<dependency org="org.tachyonproject" name="tachyon" rev="0.3.0">
+<dependency org="org.tachyonproject" name="tachyon" rev="0.4.1">
   <artifact name="tachyon" type="jar" />
 </dependency>
 ```
 
 ### SBT
 ```
-libraryDependencies += "org.tachyonproject" % "tachyon" % "0.3.0"
+libraryDependencies += "org.tachyonproject" % "tachyon" % "0.4.1"
 ```

bin/tachyon

@@ -3,15 +3,17 @@
 function printUsage {
   echo "Usage: tachyon COMMAND "
   echo "where COMMAND is one of:"
-  echo -e "  format        \t Format Tachyon"
+  echo -e "  format [-s]   \t Format Tachyon (if -s specified, only format if underfs doesn't exist)"
+  echo -e "  bootstrap-conf\t Generate a config file if one doesn't exist"
   echo -e "  tfs           \t Command line input for generic filesystem user client."
   echo -e "  loadufs       \t Load existing files in underlayer filesystem into Tachyon."
   echo -e "  runTest       \t Run a end-to-end test on a Tachyon cluster."
   echo -e "  runTests      \t Run all end-to-end tests on a Tachyon cluster."
-  echo -e "  killAll <WORD>\t Kill processes containing the WORD"
-  echo -e "  copyDir <PATH>\t Copy the PATH to all worker nodes"
+  echo -e "  killAll <WORD>\t Kill processes containing the WORD."
+  echo -e "  copyDir <PATH>\t Copy the PATH to all worker nodes."
+  echo -e "  clearCache    \t Clear OS buffer cache of the machine."
   echo -e "  thriftGen     \t Generate all thrift code."
-  echo -e "  clearCache    \t Clear OS buffer cache of the machine"
+  echo -e "  version       \t Print Tachyon version and exit."
   echo "Commands print help when invoked without parameters."
 }
 
@@ -30,7 +32,7 @@ TACHYON_LIBEXEC_DIR=${TACHYON_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
 . $TACHYON_LIBEXEC_DIR/tachyon-config.sh
 
 function runTest {
-  Usage="Usage: tachyon runTest <Basic|BasicRawTable> <MUST_CACHE|TRY_CACHE|CACHE_THROUGH|THROUGH>"
+  Usage="Usage: tachyon runTest <Basic|BasicRawTable> <MUST_CACHE|TRY_CACHE|CACHE_THROUGH|THROUGH|ASYNC_THROUGH>"
 
   if [ "$#" -ne 2 ]; then
     echo $Usage
@@ -43,12 +45,16 @@ function runTest {
   fi
 
   if [[ "$1" == "Basic" ]]; then
-    $bin/tachyon tfs rm /Basic_File_$2
-    $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicOperations $MASTER_ADDRESS:19998 /Basic_File_$2 $2
+    $bin/tachyon tfs rm /BasicFile_$2
+    $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicOperations tachyon://$MASTER_ADDRESS:19998 /BasicFile_$2 $2
     exit 0
   elif [[ "$1" == "BasicRawTable" ]]; then
-    $bin/tachyon tfs rm /Basic_Raw_Table_$2
-    $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicRawTableOperations $MASTER_ADDRESS:19998 /Basic_Raw_Table_$2 $2
+    $bin/tachyon tfs rm /BasicRawTable_$2
+    $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicRawTableOperations tachyon://$MASTER_ADDRESS:19998 /BasicRawTable_$2 $2
+    exit 0
+  elif [[ "$1" == "BasicCheckpoint" ]]; then
+    $bin/tachyon tfs rm /BasicCheckpoint
+    $JAVA -cp $TACHYON_CONF_DIR/:$TACHYON_JAR tachyon.examples.BasicCheckpoint tachyon://$MASTER_ADDRESS:19998 /BasicCheckpoint 10
     exit 0
   fi
 
@@ -94,12 +100,58 @@ function copyDir {
   wait
 }
 
+function bootstrapConf {
+  if [ $# -ne 1 ]; then
+    echo "Usage $0 bootstrap-conf TACHYON_MASTER_HOSTNAME"
+    exit 1
+  fi
+
+  if [ ! -e "$TACHYON_CONF_DIR/tachyon-env.sh" ]; then
+    # Create a default config that can be overridden later
+    cp "$TACHYON_CONF_DIR/tachyon-env.sh.template" "$TACHYON_CONF_DIR/tachyon-env.sh"
+    sed -i "s/TACHYON_MASTER_ADDRESS=localhost/TACHYON_MASTER_ADDRESS=$1/" "$TACHYON_CONF_DIR/tachyon-env.sh"
+    TOTAL_MEM=`awk '/MemTotal/{print $2}' /proc/meminfo`
+    TOTAL_MEM=$[TOTAL_MEM * 2 / 3]
+    sed -i "s/TACHYON_WORKER_MEMORY_SIZE=1GB/TACHYON_WORKER_MEMORY_SIZE=${TOTAL_MEM}KB/" "$TACHYON_CONF_DIR/tachyon-env.sh"
+  fi
+}
+
+PARAMETER=""
+
 if [ "$COMMAND" == "format" ]; then
+  if [ $# -eq 1 ]; then
+    if [ "$1" == "-s" ]; then
+      if [ -e $TACHYON_UNDERFS_ADDRESS ] || [[ $TACHYON_UNDERFS_ADDRESS == hdfs://* ]] || [[ $TACHYON_UNDERFS_ADDRESS == s3://* ]]; then
+        # already exists, hdfs, or s3, don't format
+        exit 0
+      else
+        shift # remove -s param
+      fi
+    else
+      echo "Usage $0 format [-s]"
+      exit 2
+    fi
+  elif [ $# -gt 1 ]; then
+    echo "Usage $0 format [-s]"
+    exit 2
+  fi
+
   if [ -z $TACHYON_MASTER_ADDRESS ] ; then
     TACHYON_MASTER_ADDRESS=localhost
   fi
-  echo "Formatting Tachyon @ $TACHYON_MASTER_ADDRESS"
+
+  $bin/tachyon-slaves.sh $bin/tachyon formatWorker
+
+  echo "Formatting Tachyon Master @ $TACHYON_MASTER_ADDRESS"
+  CLASS=tachyon.Format
+  PARAMETER=master
+elif [ "$COMMAND" == "bootstrap-conf" ]; then
+  bootstrapConf "$@"
+  exit 0
+elif [ "$COMMAND" == "formatWorker" ]; then
+  echo "Formatting Tachyon Worker @ `hostname`"
   CLASS=tachyon.Format
+  PARAMETER=worker
 elif [ "$COMMAND" == "tfs" ]; then
   CLASS=tachyon.command.TFsShell
 elif [ "$COMMAND" == "loadufs" ]; then
@@ -108,7 +160,7 @@ elif [ "$COMMAND" == "runTest" ]; then
   runTest "$@"
   exit 0
 elif [ "$COMMAND" == "runTests" ]; then
-  declare -a opArr=(MUST_CACHE TRY_CACHE CACHE_THROUGH THROUGH)
+  declare -a opArr=(MUST_CACHE TRY_CACHE CACHE_THROUGH THROUGH ASYNC_THROUGH)
 
   for op in ${opArr[@]}
   do
@@ -132,9 +184,11 @@ elif [ "$COMMAND" == "thriftGen" ]; then
 elif [ "$COMMAND" == "clearCache" ]; then
   sync; echo 3 > /proc/sys/vm/drop_caches ;
   exit 0
+elif [ "$COMMAND" == "version" ]; then
+  CLASS=tachyon.Version
 else
   printUsage
   exit 1
 fi
 
-$JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS -Dtachyon.logger.type=USER_LOGGER $TACHYON_JAVA_OPTS $CLASS $@
+$JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS -Dtachyon.logger.type=USER_LOGGER $TACHYON_JAVA_OPTS $CLASS $PARAMETER $@

bin/tachyon-start.sh

@@ -101,7 +101,7 @@ start_master() {
   fi
 
   echo "Starting master @ $MASTER_ADDRESS"
-  (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.logger.type="MASTER_LOGGER" -Dlog4j.configuration=file:$TACHYON_CONF_DIR/log4j.properties $TACHYON_JAVA_OPTS tachyon.Master > /dev/null 2>&1) &
+  (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.logger.type="MASTER_LOGGER" -Dlog4j.configuration=file:$TACHYON_CONF_DIR/log4j.properties $TACHYON_JAVA_OPTS tachyon.master.TachyonMaster > /dev/null 2>&1) &
 }
 
 start_worker() {
@@ -111,21 +111,21 @@ start_worker() {
     exit 1
   fi
   echo "Starting worker @ `hostname`"
-  (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.logger.type="WORKER_LOGGER" -Dlog4j.configuration=file:$TACHYON_CONF_DIR/log4j.properties $TACHYON_JAVA_OPTS tachyon.Worker `hostname` > /dev/null 2>&1 ) &
+  (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.logger.type="WORKER_LOGGER" -Dlog4j.configuration=file:$TACHYON_CONF_DIR/log4j.properties $TACHYON_JAVA_OPTS tachyon.worker.TachyonWorker `hostname` > /dev/null 2>&1 ) &
 }
 
 restart_worker() {
-  RUN=`ps -ef | grep "tachyon.Worker" | grep "java" | wc | cut -d" " -f7`
+  RUN=`ps -ef | grep "tachyon.worker.TachyonWorker" | grep "java" | wc | cut -d" " -f7`
   if [[ $RUN -eq 0 ]] ; then
     echo "Restarting worker @ `hostname`"
-    (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.is.system=true -Dtachyon.logger.type="WORKER_LOGGER" $TACHYON_JAVA_OPTS tachyon.Worker `hostname` > /dev/null 2>&1) &
+    (nohup $JAVA -cp $TACHYON_JAR -Dtachyon.home=$TACHYON_HOME -Dtachyon.is.system=true -Dtachyon.logger.type="WORKER_LOGGER" $TACHYON_JAVA_OPTS tachyon.worker.TachyonWorker `hostname` > /dev/null 2>&1) &
   fi
 }
 
 run_safe() {
   while [ 1 ]
   do
-    RUN=`ps -ef | grep "tachyon.Master" | grep "java" | wc | cut -d" " -f7`
+    RUN=`ps -ef | grep "tachyon.master.TachyonMaster" | grep "java" | wc | cut -d" " -f7`
     if [[ $RUN -eq 0 ]] ; then
       echo "Restarting the system master..."
       start_master
@@ -197,7 +197,7 @@ case "${WHAT}" in
     ;;
   workers)
     check_mount_mode $2
-    $bin/tachyon-slaves.sh $bin/tachyon-start.sh worker $2
+    $bin/tachyon-slaves.sh $bin/tachyon-start.sh worker $2 $TACHYON_MASTER_ADDRESS
     ;;
   restart_worker)
     restart_worker

bin/tachyon-stop.sh

@@ -26,7 +26,7 @@ fi
 
 bin=`cd "$( dirname "$0" )"; pwd`
 
-$bin/tachyon killAll tachyon.Master
-$bin/tachyon killAll tachyon.Worker
+$bin/tachyon killAll tachyon.master.TachyonMaster
+$bin/tachyon killAll tachyon.worker.TachyonWorker
 
-$bin/tachyon-slaves.sh $bin/tachyon killAll tachyon.Worker
+$bin/tachyon-slaves.sh $bin/tachyon killAll tachyon.worker.TachyonWorker

conf/tachyon-env.sh.template

@@ -9,26 +9,30 @@
 # - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address.
 # - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker
 # - TACHYON_RAM_FOLDER, to set where worker stores in memory data
-# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem, 
-#   org.apache.hadoop.hdfs.DistributedFileSystem) 
+# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem,
+#   org.apache.hadoop.hdfs.DistributedFileSystem)
 
 # The following gives an example:
 
 if [[ `uname -a` == Darwin* ]]; then
   # Assuming Mac OS X
+  export JAVA_HOME=$(/usr/libexec/java_home)
   export TACHYON_RAM_FOLDER=/Volumes/ramdisk
   export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
-  export JAVA_HOME=$(/usr/libexec/java_home)
 else
   # Assuming Linux
+  if [ -z "$JAVA_HOME" ]; then
+    export JAVA_HOME=/usr/lib/jvm/java-7-oracle
+  fi
   export TACHYON_RAM_FOLDER=/mnt/ramdisk
-  export JAVA_HOME=/usr/lib/jvm/java-7-oracle
 fi
 
 export JAVA="$JAVA_HOME/bin/java"
 export TACHYON_MASTER_ADDRESS=localhost
-export TACHYON_UNDERFS_ADDRESS=hdfs://localhost:8020
+export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underfs
+#export TACHYON_UNDERFS_ADDRESS=hdfs://localhost:9000
 export TACHYON_WORKER_MEMORY_SIZE=1GB
+export TACHYON_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem
 
 CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
@@ -37,10 +41,13 @@ export TACHYON_JAVA_OPTS+="
   -Dtachyon.debug=false
   -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS
   -Dtachyon.underfs.hdfs.impl=$TACHYON_UNDERFS_HDFS_IMPL
+  -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data
+  -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers
   -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE
   -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/
   -Dtachyon.master.worker.timeout.ms=60000
   -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS
   -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/
   -Dtachyon.master.pinlist=/pinfiles;/pindata
+  -Dorg.apache.jasper.compiler.disablejsr199=true
 "

docs/Building-Tachyon-Master-Branch.md

@@ -0,0 +1,59 @@
+---
+layout: global
+title: Building Tachyon Master Branch
+---
+
+This guide describes how to compile Tachyon from the beginning.
+
+The prerequisite for this guide is that you have [Java 6 (or above)](Java-Setup.html),
+[Maven](Maven.html), and [Thrift 0.9 (Optional)](Thrift.html) installed.
+
+Checkout Tachyon master branch from Github, and package:
+
+    $ git clone git://github.com/amplab/tachyon.git
+    $ cd tachyon
+    $ mvn package
+
+If you want to build a particular version of Tachyon, for example {{site.TACHYON_RELEASED_VERSION}},
+please do `git checkout v{{site.TACHYON_RELEASED_VERSION}}` after `cd tachyon`.
+
+The Maven build system fetches its dependencies, compiles, runs system's unit tests, and package the
+system. If this is the first time you are building the project, it can take a while to download all
+the dependencies. Subsequent builds, however, will be much faster.
+
+Once it is built, you can start the Tachyon:
+
+    $ cp conf/tachyon-env.sh.template conf/tachyon-env.sh
+    $ ./bin/tachyon format
+    $ ./bin/tachyon-start.sh local
+
+To verify that Tachyon is running, you can visit [http://localhost:19999](http://localhost:19999),
+or see the log in the folder tachyon/logs. You can also run a simple program:
+
+    $ ./bin/tachyon runTest Basic CACHE_THROUGH
+
+You should be able to see results similar to the following:
+
+    /Basic_File_CACHE_THROUGH has been removed
+    2014-02-02 09:32:02,760 INFO   (TachyonFS.java:connect) - Trying to connect master @ localhost/127.0.0.1:19998
+    2014-02-02 09:32:02,791 INFO   (MasterClient.java:getUserId) - User registered at the master localhost/127.0.0.1:19998 got UserId 10
+    2014-02-02 09:32:02,792 INFO   (TachyonFS.java:connect) - Trying to get local worker host : hy-ubuntu
+    2014-02-02 09:32:02,800 INFO   (TachyonFS.java:connect) - Connecting local worker @ hy-ubuntu/127.0.1.1:29998
+    2014-02-02 09:32:02,819 INFO   (CommonUtils.java:printTimeTakenMs) - createFile with fileId 18 took 60 ms.
+    2014-02-02 09:32:03,194 INFO   (TachyonFS.java:createAndGetUserTempFolder) - Folder /mnt/ramdisk/tachyonworker/users/10 was created!
+    2014-02-02 09:32:03,198 INFO   (BlockOutStream.java:<init>) - /mnt/ramdisk/tachyonworker/users/10/19327352832 was created!
+    Passed the test!
+
+You can also stop the system by using:
+
+    $ ./bin/tachyon-stop.sh
+
+# Unit tests
+
+To run all unit tests:
+
+    $ mvn test
+
+To run all the unit tests with under filesystem other than local filesystem:
+
+    $ mvn test [ -Dhadoop.version=x.x.x ] -Dintegration [ -Dufs=tachyon.LocalMiniDFSCluster ]