diff --git a/tigon-docs/developer-guide/build.sh b/tigon-docs/developer-guide/build.sh index 3981f252..15484376 100755 --- a/tigon-docs/developer-guide/build.sh +++ b/tigon-docs/developer-guide/build.sh @@ -29,6 +29,7 @@ SCRIPT=`basename $0` SOURCE="source" BUILD="build" BUILD_PDF="build-pdf" +BUILD_TEMP="build-temp" HTML="html" INCLUDES="_includes" @@ -96,6 +97,7 @@ function usage() { echo "Usage: $SCRIPT < option > [source]" echo "" echo " Options (select one)" + echo " all Clean build of everything: HTML docs and Javadocs, GitHub and Web versions" echo " build Clean build of javadocs and HTML docs, copy javadocs into place, zip results" echo " build-includes Clean conversion of example/README.md to _includes directory reST files" echo " build-quick Clean build of HTML docs, copy existing javadocs into place" @@ -186,6 +188,20 @@ function make_zip_localized() { zip -r $ZIP_DIR_NAME.zip $PROJECT_VERSION/* } +function build_all() { + echo "Building GitHub Docs." + ./build.sh build-github + echo "Stashing GitHub Docs." + cd $SCRIPT_PATH + mkdir -p $SCRIPT_PATH/$BUILD_TEMP + mv $SCRIPT_PATH/$BUILD/*.zip $SCRIPT_PATH/$BUILD_TEMP + echo "Building Web Docs." + ./build.sh build-web + echo "Moving GitHub Docs." + mv $SCRIPT_PATH/$BUILD_TEMP/*.zip $SCRIPT_PATH/$BUILD + rm -rf $SCRIPT_PATH/$BUILD_TEMP +} + function build() { build_docs build_javadocs @@ -242,6 +258,21 @@ function pandoc_includes() { pandoc -t rst $PROJECT_PATH/$TIGON_EXAMPLES/SQLJoinFlow/README.md -o $INCLUDES_DIR/sql-join-flow.rst pandoc -t rst $PROJECT_PATH/$TIGON_EXAMPLES/tigon-sql/README.md -o $INCLUDES_DIR/tigon-sql.rst pandoc -t rst $PROJECT_PATH/$TIGON_EXAMPLES/TwitterAnalytics/README.md -o $INCLUDES_DIR/twitter-analytics.rst + + # Fix version + version_rewrite $SCRIPT_PATH/$SOURCE/getting-started.txt $INCLUDES_DIR/getting-started-versioned.rst + +} + +function version_rewrite() { + version + cd $SCRIPT_PATH + echo "Re-writing $1 to $2" + # Re-writes the version in an RST-snippet file to have the current version. + REWRITE_SOURCE=$1 + REWRITE_TARGET=$2 + + sed -e "s||$PROJECT_VERSION|g" $REWRITE_SOURCE > $REWRITE_TARGET } function build_quick() { @@ -311,6 +342,7 @@ if [ $# -lt 1 ]; then fi case "$1" in + all ) build_all; exit 1;; build ) build; exit 1;; build-includes ) build_includes; exit 1;; build-quick ) build_quick; exit 1;; diff --git a/tigon-docs/developer-guide/source/_themes/tigon/static/tigon.css_t b/tigon-docs/developer-guide/source/_themes/tigon/static/tigon.css_t index b28e78e5..65816bc4 100644 --- a/tigon-docs/developer-guide/source/_themes/tigon/static/tigon.css_t +++ b/tigon-docs/developer-guide/source/_themes/tigon/static/tigon.css_t @@ -255,6 +255,16 @@ p.admonition-title:after { content: ":"; } +p.rubric { + background-color: #E0E0E0; + font-size: 120%; + font-weight: normal; + color: #111; + margin: 30px 0px 10px -15px; + padding: 5px 0 5px 15px; + text-shadow: 0px 1px 0 white +} + em { font-style: italic; } diff --git a/tigon-docs/developer-guide/source/admin.rst b/tigon-docs/developer-guide/source/admin.rst index 620e1f1b..4b2d2896 100644 --- a/tigon-docs/developer-guide/source/admin.rst +++ b/tigon-docs/developer-guide/source/admin.rst @@ -68,4 +68,4 @@ Where to Go Next Now that you're familiar with how to run Tigon, take a look at: -- `Licenses and Copyrights `__, which covers how Tigon has been released. +- :doc:`licenses`, which covers how Tigon has been released. diff --git a/tigon-docs/developer-guide/source/apis/index.rst b/tigon-docs/developer-guide/source/apis/index.rst index 3fa85d97..007d4a53 100644 --- a/tigon-docs/developer-guide/source/apis/index.rst +++ b/tigon-docs/developer-guide/source/apis/index.rst @@ -29,7 +29,7 @@ Where to Go Next Now that you're familiar with the APIs for Tigon, take a look at: -- `Tools <../tools.html>`__, which describes the command-line interface for running a Tigon Application. +- :doc:`/tools`, which describes the command-line interface for running a Tigon Application. .. downloads: diff --git a/tigon-docs/developer-guide/source/concepts.rst b/tigon-docs/developer-guide/source/concepts.rst index bb845d2b..ba6d2e15 100644 --- a/tigon-docs/developer-guide/source/concepts.rst +++ b/tigon-docs/developer-guide/source/concepts.rst @@ -61,4 +61,4 @@ Where to Go Next Now that you're familiar with the basic concept and architecture of Tigon, take a look at: -- `Developer `__, which covers writing applications using Tigon. +- :doc:`developer`, which covers writing applications using Tigon. diff --git a/tigon-docs/developer-guide/source/developer.rst b/tigon-docs/developer-guide/source/developer.rst index 427ea649..d29f64da 100644 --- a/tigon-docs/developer-guide/source/developer.rst +++ b/tigon-docs/developer-guide/source/developer.rst @@ -780,6 +780,6 @@ Where to Go Next Now that you're familiar with the components and concepts of Tigon, take a look at: -- `Examples `__, with a series of examples demonstrating Tigon. +- :doc:`/examples/index`, with a series of examples demonstrating Tigon. diff --git a/tigon-docs/developer-guide/source/examples/index.rst b/tigon-docs/developer-guide/source/examples/index.rst index ac7a1ef8..5594bbba 100644 --- a/tigon-docs/developer-guide/source/examples/index.rst +++ b/tigon-docs/developer-guide/source/examples/index.rst @@ -49,10 +49,14 @@ Reference applications built with Tigon are included in the - `AdNetworkFlow: `__ Demonstrates using Tigon to write a realtime bidding (RTB) advertisement framework. +- `ClickStreamFlow: `__ + Demonstrates using Tigon SQL to join 2 data streams. In this example, a view event data + stream and a click event data stream are joined to generate meta information for each + click event. Click events are filtered on the basis of conditions specified in a SQL query. Where to Go Next ================ Now that you're seen an example demonstrating Tigon, take a look at: -- `APIs <../apis/index.html>`__, which includes the Java and TigonSQL APIs of Tigon. +- :doc:`/apis/index`, which includes the Java and TigonSQL APIs of Tigon. diff --git a/tigon-docs/developer-guide/source/faq.rst b/tigon-docs/developer-guide/source/faq.rst index 74a5c5fa..0ea313a9 100644 --- a/tigon-docs/developer-guide/source/faq.rst +++ b/tigon-docs/developer-guide/source/faq.rst @@ -9,71 +9,71 @@ Tigon FAQ General ======= -**What is Tigon?** - +What is Tigon? +-------------------------- **Tigon** is an open-source, real-time stream processing framework built on top of Apache |(TM)| Hadoop |(R)| and HBase. -**Who is Tigon intended for?** - +Who is Tigon intended for? +-------------------------- Developers who are interested in creating powerful, yet simple-to-develop stream processing applications that can handle large volumes of data. -**What are some of the applications that can be built on Tigon?** +What are some of the applications that can be built on Tigon? +------------------------------------------------------------------------------ - Processing stream sources such as Twitter, Webserver Logs - Rapid Joining, Filtering, and Aggregating of Streams -**How does Tigon work?** - +How does Tigon work? +------------------------------------------------------------------------------ Tigon is built on top of Hadoop/HBase. It uses the `Tephra Transaction Engine `__ and `Twill `__ for spinning up YARN applications. For in-memory stream processing, it uses the `TigonSQL `__ in-memory stream processing engine developed by AT&T. -**What is the difference between Tigon and Cask's other projects, such as CDAP?** - +What is the difference between Tigon and Cask's other projects, such as CDAP? +------------------------------------------------------------------------------ Tigon is focused on solving the issues faced in real-time stream processing. The `Cask Data Application Platform (CDAP) `__ is a generalized 'Big Data' application platform with additional features such as dataset abstractions, batch job integration, and security. -**What's the vision for Tigon?** - +What's the vision for Tigon? +------------------------------------------------------------------------------ Enable every Java developer to create powerful, real-time stream processing applications. Running Tigon ============= -**How do I get started with Tigon?** - +How do I get started with Tigon? +------------------------------------------------------------------------------ Download the zip, unzip it and checkout the examples! -See our `Getting Started With Tigon `__ guide. - -**What are the prerequisites for running Tigon?** - -See our `Getting Started With Tigon `__ guide -for a list of prerequisites. +See our :doc:`getting-started` guide. -**What platforms and Java version does Tigon run on?** +What are the prerequisites for running Tigon? +------------------------------------------------------------------------------ +See our :doc:`getting-started` guide for a :ref:`list of prerequisites. ` +What platforms and Java version does Tigon run on? +------------------------------------------------------------------------------ Tigon runs on \*NIX systems such as Linux and Macintosh OS X. A Java Development Kit such as JDK 6 or JDK 7 is required to run Tigon. .. **Are there Tigon RPM or Debian packages available for download?** -**Does Tigon run on Windows?** - +Does Tigon run on Windows? +------------------------------------------------------------------------------ Currently, Tigon does not run on Windows. -**What hardware do I need for Tigon?** - +What hardware do I need for Tigon? +------------------------------------------------------------------------------ Tigon runs on the same hardware that would support Hadoop/HBase. -**What programming languages are supported by Tigon?** - +What programming languages are supported by Tigon? +------------------------------------------------------------------------------ Applications that use Tigon currently need to be written in Java. If you are using TigonSQL, commands are written in an SQL dialect. @@ -83,42 +83,42 @@ certain elements are written in Perl and Python. Tigon Support ========================= -**Where can I find more information about Tigon?** - +Where can I find more information about Tigon? +------------------------------------------------------------------------------ Our resources include this website, our parent website (`cask.co `__), two mailing lists, an IRC channel and a `GitHub repository. `__ -**What mailing lists are available for additional help?** - +What mailing lists are available for additional help? +------------------------------------------------------------------------------ Tigon User Group and Development Discussions: - Focused on Tigon usage questions, discussions, and solutions: `tigon-user@googlegroups.com `__ - For developers modifying Tigon itself: `tigon-dev@googlegroups.com `__ -**Is there an IRC Channel?** - -Tigon IRC Channel #tigon on irc.freenode.net - -**Where are Bug Reports and Feature Requests kept?** +Is there an IRC Channel? +------------------------------------------------------------------------------ +Tigon IRC Channel #tigon on `irc.freenode.net. `__ -Bugs and tasks are tracked in a public JIRA issue tracker. Details on access will be forthcoming. - -**Is commercial support available for Tigon?** +Where are Bug Reports and Feature Requests kept? +------------------------------------------------------------------------------ +We have a `JIRA for filing issues. `__ +Is commercial support available for Tigon? +------------------------------------------------------------------------------ Contact Cask Data for information on `commercial Tigon support. `__ Contributing to Tigon ========================= -**How can I help make Tigon better?** - +How can I help make Tigon better? +------------------------------------------------------------------------------ We welcome all contributions, whether in filing detailed bug reports, submitting pull requests for code changes and improvements, or by asking questions and assisting others on the mailing list. -**How can I contribute?** - +How can I contribute? +------------------------------------------------------------------------------ Are you interested in making Tigon better? We have a simple pull-based development model with a consensus-building phase, similar to Apache's voting process. If you’d like to help make Tigon better by adding new features, enhancing existing features, or fixing bugs, @@ -136,6 +136,7 @@ here's how to do it: #. After we review and accept your request, we’ll commit your code to the `caskdata/tigon `__ repository. + .. |(TM)| unicode:: U+2122 .. trademark sign :ltrim: diff --git a/tigon-docs/developer-guide/source/getting-started.rst b/tigon-docs/developer-guide/source/getting-started.rst index f7fe43e7..736a6e02 100644 --- a/tigon-docs/developer-guide/source/getting-started.rst +++ b/tigon-docs/developer-guide/source/getting-started.rst @@ -2,152 +2,5 @@ :description: Getting started guide :copyright: Copyright © 2014 Cask Data, Inc. -============================================ -Getting Started with Tigon -============================================ - -We recommend you follow these steps to get started with Tigon: - -1. Check that you have the `prerequisites`_ and install if necessary. -#. `Download`_ the pre-compiled sources and related files. -#. `Configure`_ your system, if necessary. -#. Run the `examples`_ to test your installation and learn about Tigon. - - -Prerequisites -============= - -Tigon is supported on \*NIX systems such as Linux and Macintosh OS X. -It is not supported on Microsoft Windows. - -To install and use Tigon and its included examples, there are a few prerequisites: - -1. JDK 6 or JDK 7 (required to run Tigon; note that $JAVA_HOME should be set) -#. GCC -#. G++ -#. Apache Maven 3.0+ (required to build the example applications) - -**Note:** To run the TigonSQL Stream Engine outside of Tigon, ``libz``, ``Perl 5.x``, and ``Python 3.x`` -are required. - - -Download -======== - -Pre-compiled sources and related files can be downloaded in a zip file: -`tigon-developer-release-0.2.0.zip. -`__ - - -Install -======= - -Once the download has completed, unzip the file to a suitable location. - - -Creating an Application -======================= - -.. highlight:: console - -The best way to start developing a Tigon application is by using the Maven archetype:: - - $ mvn archetype:generate \ - -DarchetypeGroupId=co.cask.tigon \ - -DarchetypeArtifactId=tigon-app-archetype - -DarchetypeVersion=0.2.0 - -This creates a Maven project with all required dependencies, Maven plugins, and a simple -application template for the development of your application. You can import this Maven -project into your preferred IDE—such as Eclipse or IntelliJ—and start developing your -first Tigon application. - - -Building Tigon from Source -========================== - -.. highlight:: console - -You can also build Tigon directly from the latest source code:: - - git clone https://github.com/caskdata/tigon.git - cd tigon - mvn clean package -DskipTests -Pdist - -After the build completes, you will have a distribution of Tigon under the -`tigon-distribution/target/` directory. - -Take the `tigon-sdk-.zip` file and unzip it into a suitable location. - - -Is It Building? ---------------- -These links allow you to check the status of the automated builds of the source code: - -- `Bamboo Build `__ -- `GitHub Version `__ - -.. _configure: - -Configuration -============= - -Macintosh OS X --------------- - -.. highlight:: console - -TigonSQL will not run with the default shared memory settings of the Macintosh, as -OS X does not provide sufficient shared memory regions by default. -To run TigonSQL, you'll need to revise these settings by executing these commands:: - - sudo sysctl -w kern.sysv.shmall=102400 - - sudo sysctl -w kern.sysv.shmseg=100 - - sudo sysctl -w kern.sysv.shmmax=419430400 - - -Problems with the Downloaded Package? ---------------------------------------- - -If the downloaded package doesn't work, try to build a tarball for your system -as described above. - -If you face problems using TigonSQL, you can build the TigonSQL Streaming library from the GitHub repo source:: - - mvn clean install -DskipTests -P sql-lib,dist - -The TigonSQL jar will be installed locally and will be used by maven to create Tigon -applications when they are created on the same machine. - -Examples -======== - -Examples of using Tigon are described in the `Tigon Examples Guide: `__ - -- `Hello World Example `__ - - .. include:: examples/hello-world.rst - :start-line: 5 - :end-before: Building the JAR - -- `Twitter Analytics Example `__ - - .. include:: examples/twitter-analytics.rst - :start-line: 5 - :end-before: Twitter Configuration - -- `SQL Join Flow Example `__ - - .. include:: examples/sql-join-flow.rst - :start-line: 5 - :end-before: Flow Runtime Arguments - - -Where to Go Next -================ - -Now that you've gotten started with Tigon, take a look at: - -- `Concepts and Architecture `__, which covers the basic design behind Tigon. +.. include:: ../build/_includes/getting-started-versioned.rst + :start-line: 4 \ No newline at end of file diff --git a/tigon-docs/developer-guide/source/getting-started.txt b/tigon-docs/developer-guide/source/getting-started.txt new file mode 100644 index 00000000..585f2f23 --- /dev/null +++ b/tigon-docs/developer-guide/source/getting-started.txt @@ -0,0 +1,155 @@ +.. :author: Cask Data, Inc. + :description: Getting started guide + :copyright: Copyright © 2014 Cask Data, Inc. + +============================================ +Getting Started with Tigon +============================================ + +We recommend you follow these steps to get started with Tigon: + +1. Check that you have the `prerequisites`_ and install if necessary. +#. `Download`_ the pre-compiled sources and related files. +#. `Configure`_ your system, if necessary. +#. Run the :doc:`/examples/index` to test your installation and learn about Tigon. + +.. _getting-started-prerequisites: + +Prerequisites +============= + +Tigon is supported on \*NIX systems such as Linux and Macintosh OS X. +It is not supported on Microsoft Windows. + +To install and use Tigon and its included examples, there are a few prerequisites: + +1. JDK 6 or JDK 7 (required to run Tigon; note that $JAVA_HOME should be set) +#. GCC +#. G++ +#. Apache Maven 3.0+ (required to build the example applications) + +**Note:** To run the TigonSQL Stream Engine outside of Tigon, ``libz``, ``Perl 5.x``, and ``Python 3.x`` +are required. + + +Download +======== + +Pre-compiled sources and related files can be downloaded in a zip file: +`tigon-developer-release-.zip. +/tigon-developer-release-.zip>`__ + + +Install +======= + +Once the download has completed, unzip the file to a suitable location. + + +Creating an Application +======================= + +.. highlight:: console + +The best way to start developing a Tigon application is by using the Maven archetype:: + + $ mvn archetype:generate \ + -DarchetypeGroupId=co.cask.tigon \ + -DarchetypeArtifactId=tigon-app-archetype + -DarchetypeVersion= + +This creates a Maven project with all required dependencies, Maven plugins, and a simple +application template for the development of your application. You can import this Maven +project into your preferred IDE—such as Eclipse or IntelliJ—and start developing your +first Tigon application. + + +Building Tigon from Source +========================== + +.. highlight:: console + +You can also build Tigon directly from the latest source code:: + + git clone https://github.com/caskdata/tigon.git + cd tigon + mvn clean package -DskipTests -Pdist + +After the build completes, you will have a distribution of Tigon under the +`tigon-distribution/target/` directory. + +Take the `tigon-sdk-.zip` file and unzip it into a suitable location. + + +Is It Building? +--------------- +These links allow you to check the status of the automated builds of the source code: + +- `Bamboo Build `__ +- `GitHub Version `__ + +.. _configure: + +Configuration +============= + +Macintosh OS X +-------------- + +.. highlight:: console + +TigonSQL will not run with the default shared memory settings of the Macintosh, as +OS X does not provide sufficient shared memory regions by default. +To run TigonSQL, you'll need to revise these settings by executing these commands:: + + sudo sysctl -w kern.sysv.shmall=102400 + + sudo sysctl -w kern.sysv.shmseg=100 + + sudo sysctl -w kern.sysv.shmmax=419430400 + + +Problems with the Downloaded Package? +--------------------------------------- + +If the downloaded package doesn't work, try to build a tarball for your system +as described above. + +If you face problems using TigonSQL, you can build the TigonSQL Streaming library from the GitHub repo source:: + + mvn clean install -DskipTests -P sql-lib,dist + +The TigonSQL jar will be installed locally and will be used by maven to create Tigon +applications when they are created on the same machine. + + +Examples and Applications +========================= + +Examples of using Tigon are described in :doc:`/examples/index`, such as: + +- :doc:`/examples/hello-world` + + .. include:: examples/hello-world.rst + :start-line: 5 + :end-before: Building the JAR + +- :doc:`/examples/twitter-analytics` + + .. include:: examples/twitter-analytics.rst + :start-line: 5 + :end-before: Twitter Configuration + +- :doc:`/examples/sql-join-flow` + + .. include:: examples/sql-join-flow.rst + :start-line: 5 + :end-before: Flow Runtime Arguments + + +Where to Go Next +================ + +Now that you've gotten started with Tigon, take a look at: + +- :doc:`concepts`, which covers the basic design behind Tigon. diff --git a/tigon-docs/developer-guide/source/index.rst b/tigon-docs/developer-guide/source/index.rst index e9499b23..437c5b85 100644 --- a/tigon-docs/developer-guide/source/index.rst +++ b/tigon-docs/developer-guide/source/index.rst @@ -94,7 +94,7 @@ Where to Go Next .. |remaining| replace:: **Remaining Sections:** -- |remaining| the final documentation sections cover the :doc:`Licenses and Copyrights, +- |remaining| the final sections cover the :doc:`Licenses and Copyrights, ` :doc:`Releases Notes, ` and a :doc:`FAQ. ` diff --git a/tigon-docs/developer-guide/source/release-notes.rst b/tigon-docs/developer-guide/source/release-notes.rst index dedf3e8f..76624ff1 100644 --- a/tigon-docs/developer-guide/source/release-notes.rst +++ b/tigon-docs/developer-guide/source/release-notes.rst @@ -6,6 +6,18 @@ Tigon Release Notes ============================================ +Release 0.2.1 +============= + +Bug Fix +--------- + +- Fixed a problem in TigonSQL which mapped all schemas to a default interface set, + resulting in incorrect outputs. Instead, we now have a single interface per input schema + with no default interface set. Modified Flows that use TigonSQL to adhere to this + approach. + + Release 0.2.0 ============= diff --git a/tigon-docs/developer-guide/source/tools.rst b/tigon-docs/developer-guide/source/tools.rst index 9b0934b8..68aba626 100644 --- a/tigon-docs/developer-guide/source/tools.rst +++ b/tigon-docs/developer-guide/source/tools.rst @@ -70,5 +70,5 @@ Where to Go Next Now that you're seen the command line tool for Tigon, take a look at: -- `Admin `__, which describes administrating a Tigon application. +- :doc:`admin`, which describes administrating a Tigon application.