- {if (hasDrivers)
-
@@ -121,13 +123,15 @@ private[spark] class IndexPage(parent: MasterWebUI) {
-
+ {if (hasDrivers) {
+
-
- Running Drivers
- {activeDriversTable} -
+
+ }
}
+
+ Running Drivers
+ {activeDriversTable} +
- {if (hasDrivers)
-
;
@@ -175,7 +179,8 @@ private[spark] class IndexPage(parent: MasterWebUI) {
-
+ {
+ if (hasDrivers) {
+
-
- Completed Drivers
- {completedDriversTable} -
+
+ }
}
+
+ Completed Drivers
+ {completedDriversTable} +
{if (hasDrivers)
;
-
+ // scalastyle:on
UIUtils.basicSparkPage(content, "Spark Worker at %s:%s".format(
workerState.host, workerState.port))
}
@@ -133,10 +133,10 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
@@ -113,7 +113,7 @@ private[spark] class IndexPage(parent: WorkerWebUI) {
}
- ![]({prependBaseUri("/static/spark-logo-77x50px-hd.png")})
+
+
+
+
+
+
-
{jobs}
{storage}
@@ -93,7 +97,8 @@ private[spark] object UIUtils {
-
+
-
+
{title}
-
@@ -96,7 +95,7 @@ private[spark] class StagePage(parent: JobProgressUI) {
}
Heading
+ + + + diff --git a/docs/bagel-programming-guide.md b/docs/bagel-programming-guide.md index cffa55ee952b0..b070d8e73a38b 100644 --- a/docs/bagel-programming-guide.md +++ b/docs/bagel-programming-guide.md @@ -16,7 +16,7 @@ This guide shows the programming model and features of Bagel by walking through To use Bagel in your program, add the following SBT or Maven dependency: groupId = org.apache.spark - artifactId = spark-bagel_{{site.SCALA_VERSION}} + artifactId = spark-bagel_{{site.SCALA_BINARY_VERSION}} version = {{site.SPARK_VERSION}} # Programming Model diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 6a9a8d681742f..ded12926885b9 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -17,10 +17,10 @@ You'll need to configure Maven to use more memory than usual by setting `MAVEN_O If you don't run this, you may see errors like the following: - [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_VERSION}}/classes... + [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_BINARY_VERSION}}/classes... [ERROR] PermGen space -> [Help 1] - [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_VERSION}}/classes... + [INFO] Compiling 203 Scala sources and 9 Java sources to /Users/me/Development/spark/core/target/scala-{{site.SCALA_BINARY_VERSION}}/classes... [ERROR] Java heap space -> [Help 1] You can fix this by setting the `MAVEN_OPTS` variable as discussed before. diff --git a/docs/css/main.css b/docs/css/main.css index 8566400f071c9..f6fe7d5f07da1 100755 --- a/docs/css/main.css +++ b/docs/css/main.css @@ -138,3 +138,11 @@ ul.nav li.dropdown ul.dropdown-menu li.dropdown-submenu ul.dropdown-menu { .nav-tabs > li > a, .nav-tabs > li > a:hover { color: #333; } + +/** + * MathJax (embedded latex formulas) + */ +.MathJax .mo { color: inherit } +.MathJax .mi { color: inherit } +.MathJax .mf { color: inherit } +.MathJax .mh { color: inherit } diff --git a/docs/index.md b/docs/index.md index 7fea73024a8a0..aa9c8666e7d75 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Spark uses [Simple Build Tool](http://www.scala-sbt.org), which is bundled with sbt/sbt assembly -For its Scala API, Spark {{site.SPARK_VERSION}} depends on Scala {{site.SCALA_VERSION}}. If you write applications in Scala, you will need to use this same version of Scala in your own program -- newer major versions may not work. You can get the right version of Scala from [scala-lang.org](http://www.scala-lang.org/download/). +For its Scala API, Spark {{site.SPARK_VERSION}} depends on Scala {{site.SCALA_BINARY_VERSION}}. If you write applications in Scala, you will need to use a compatible Scala version (e.g. {{site.SCALA_BINARY_VERSION}}.X) -- newer major versions may not work. You can get the right version of Scala from [scala-lang.org](http://www.scala-lang.org/download/). # Running the Examples and Shell diff --git a/docs/mllib-classification-regression.md b/docs/mllib-classification-regression.md new file mode 100644 index 0000000000000..18a3e8e075086 --- /dev/null +++ b/docs/mllib-classification-regression.md @@ -0,0 +1,422 @@ +--- +layout: global +title: MLlib - Classification and Regression +--- + +* Table of contents +{:toc} + + +`\[ +\newcommand{\R}{\mathbb{R}} +\newcommand{\E}{\mathbb{E}} +\newcommand{\x}{\mathbf{x}} +\newcommand{\y}{\mathbf{y}} +\newcommand{\wv}{\mathbf{w}} +\newcommand{\av}{\mathbf{\alpha}} +\newcommand{\bv}{\mathbf{b}} +\newcommand{\N}{\mathbb{N}} +\newcommand{\id}{\mathbf{I}} +\newcommand{\ind}{\mathbf{1}} +\newcommand{\0}{\mathbf{0}} +\newcommand{\unit}{\mathbf{e}} +\newcommand{\one}{\mathbf{1}} +\newcommand{\zero}{\mathbf{0}} +\]` + + +# Supervised Machine Learning +Supervised machine learning is the setting where we are given a set of training data examples +`$\{\x_i\}$`, each example `$\x_i$` coming with a corresponding label `$y_i$`. +Given the training data `$\{(\x_i,y_i)\}$`, we want to learn a function to predict these labels. +The two most well known classes of methods are +[classification](http://en.wikipedia.org/wiki/Statistical_classification), and +[regression](http://en.wikipedia.org/wiki/Regression_analysis). +In classification, the label is a category (e.g. whether or not emails are spam), whereas in +regression, the label is real value, and we want our prediction to be as close to the true value +as possible. + +Supervised Learning involves executing a learning *Algorithm* on a set of *labeled* training +examples. The algorithm returns a trained *Model* (such as for example a linear function) that +can predict the label for new data examples for which the label is unknown. + + +## Mathematical Formulation +Many standard *machine learning* methods can be formulated as a convex optimization problem, i.e. +the task of finding a minimizer of a convex function `$f$` that depends on a variable vector +`$\wv$` (called `weights` in the code), which has `$d$` entries. +Formally, we can write this as the optimization problem `$\min_{\wv \in\R^d} \; f(\wv)$`, where +the objective function is of the form +`\begin{equation} + f(\wv) := + \lambda\, R(\wv) + + \frac1n \sum_{i=1}^n L(\wv;\x_i,y_i) + \label{eq:regPrimal} + \ . +\end{equation}` +Here the vectors `$\x_i\in\R^d$` are the training data examples, for `$1\le i\le n$`, and +`$y_i\in\R$` are their corresponding labels, which we want to predict. + +The objective function `$f$` has two parts: +The *loss-function* measures the error of the model on the training data. The loss-function +`$L(\wv;.)$` must be a convex function in `$\wv$`. +The purpose of the [regularizer](http://en.wikipedia.org/wiki/Regularization_(mathematics)) is to +encourage simple models, by punishing the complexity of the model `$\wv$`, in order to e.g. avoid +over-fitting. +Usually, the regularizer `$R(.)$` is chosen as either the standard (Euclidean) L2-norm, `$R(\wv) +:= \frac{1}{2}\|\wv\|^2$`, or the L1-norm, `$R(\wv) := \|\wv\|_1$`, see +[below](#using-different-regularizers) for more details. + +The fixed regularization parameter `$\lambda\ge0$` (`regParam` in the code) defines the trade-off +between the two goals of small loss and small model complexity. + + +## Binary Classification + +**Input:** Datapoints `$\x_i\in\R^{d}$`, labels `$y_i\in\{+1,-1\}$`, for `$1\le i\le n$`. + +**Distributed Datasets.** +For all currently implemented optimization methods for classification, the data must be +distributed between the worker machines *by examples*. Every machine holds a consecutive block of +the `$n$` example/label pairs `$(\x_i,y_i)$`. +In other words, the input distributed dataset +([RDD](scala-programming-guide.html#resilient-distributed-datasets-rdds)) must be the set of +vectors `$\x_i\in\R^d$`. + +### Support Vector Machine +The linear [Support Vector Machine (SVM)](http://en.wikipedia.org/wiki/Support_vector_machine) +has become a standard choice for classification tasks. +Here the loss function in formulation `$\eqref{eq:regPrimal}$` is given by the hinge-loss +`\[ +L(\wv;\x_i,y_i) := \max \{0, 1-y_i \wv^T \x_i \} \ . +\]` + +By default, SVMs are trained with an L2 regularization, which gives rise to the large-margin +interpretation if these classifiers. We also support alternative L1 regularization. In this case, +the primal optimization problem becomes an [LP](http://en.wikipedia.org/wiki/Linear_programming). + +### Logistic Regression +Despite its name, [Logistic Regression](http://en.wikipedia.org/wiki/Logistic_regression) is a +binary classification method, again when the labels are given by binary values +`$y_i\in\{+1,-1\}$`. The logistic loss function in formulation `$\eqref{eq:regPrimal}$` is +defined as +`\[ +L(\wv;\x_i,y_i) := \log(1+\exp( -y_i \wv^T \x_i)) \ . +\]` + + +## Linear Regression (Least Squares, Lasso and Ridge Regression) + +**Input:** Data matrix `$A\in\R^{n\times d}$`, right hand side vector `$\y\in\R^n$`. + +**Distributed Datasets.** +For all currently implemented optimization methods for regression, the data matrix +`$A\in\R^{n\times d}$` must be distributed between the worker machines *by rows* of `$A$`. In +other words, the input distributed dataset +([RDD](scala-programming-guide.html#resilient-distributed-datasets-rdds)) must be the set of the +`$n$` rows `$A_{i:}$` of `$A$`. + +Least Squares Regression refers to the setting where we try to fit a vector `$\y\in\R^n$` by +linear combination of our observed data `$A\in\R^{n\times d}$`, which is given as a matrix. + +It comes in 3 flavors: + +### Least Squares +Plain old [least squares](http://en.wikipedia.org/wiki/Least_squares) linear regression is the +problem of minimizing + `\[ f_{\text{LS}}(\wv) := \frac1n \|A\wv-\y\|_2^2 \ . \]` + +### Lasso +The popular [Lasso](http://en.wikipedia.org/wiki/Lasso_(statistics)#Lasso_method) (alternatively +also known as `$L_1$`-regularized least squares regression) is given by + `\[ f_{\text{Lasso}}(\wv) := \frac1n \|A\wv-\y\|_2^2 + \lambda \|\wv\|_1 \ . \]` + +### Ridge Regression +[Ridge regression](http://en.wikipedia.org/wiki/Ridge_regression) uses the same loss function but +with a L2 regularizer term: + `\[ f_{\text{Ridge}}(\wv) := \frac1n \|A\wv-\y\|_2^2 + \frac{\lambda}{2}\|\wv\|^2 \ . \]` + +**Loss Function.** +For all 3, the loss function (i.e. the measure of model fit) is given by the squared deviations +from the right hand side `$\y$`. +`\[ +\frac1n \|A\wv-\y\|_2^2 += \frac1n \sum_{i=1}^n (A_{i:} \wv - y_i )^2 += \frac1n \sum_{i=1}^n L(\wv;\x_i,y_i) +\]` +This is also known as the [mean squared error](http://en.wikipedia.org/wiki/Mean_squared_error). +In our generic problem formulation `$\eqref{eq:regPrimal}$`, this means the loss function is +`$L(\wv;\x_i,y_i) := (A_{i:} \wv - y_i )^2$`, each depending only on a single row `$A_{i:}$` of +the data matrix `$A$`. + + +## Using Different Regularizers + +As we have mentioned above, the purpose of *regularizer* in `$\eqref{eq:regPrimal}$` is to +encourage simple models, by punishing the complexity of the model `$\wv$`, in order to e.g. avoid +over-fitting. +All machine learning methods for classification and regression that we have mentioned above are +of interest for different types of regularization, the 3 most common ones being + +* **L2-Regularization.** +`$R(\wv) := \frac{1}{2}\|\wv\|^2$`. +This regularizer is most commonly used for SVMs, logistic regression and ridge regression. + +* **L1-Regularization.** +`$R(\wv) := \|\wv\|_1$`. The L1 norm `$\|\wv\|_1$` is the sum of the absolut values of the +entries of a vector `$\wv$`. +This regularizer is most commonly used for sparse methods, and feature selection, such as the +Lasso. + +* **Non-Regularized.** +`$R(\wv):=0$`. +Of course we can also train the models without any regularization, or equivalently by setting the +regularization parameter `$\lambda:=0$`. + +The optimization problems of the form `$\eqref{eq:regPrimal}$` with convex regularizers such as +the 3 mentioned here can be conveniently optimized with gradient descent type methods (such as +SGD) which is implemented in `MLlib` currently, and explained in the next section. + + +# Optimization Methods Working on the Primal Formulation + +**Stochastic subGradient Descent (SGD).** +For optimization objectives `$f$` written as a sum, *stochastic subgradient descent (SGD)* can be +an efficient choice of optimization method, as we describe in the optimization section in more detail. +Because all methods considered here fit into the optimization formulation +`$\eqref{eq:regPrimal}$`, this is especially natural, because the loss is written as an average +of the individual losses coming from each datapoint. + +Picking one datapoint `$i\in[1..n]$` uniformly at random, we obtain a stochastic subgradient of +`$\eqref{eq:regPrimal}$`, with respect to `$\wv$` as follows: +`\[ +f'_{\wv,i} := L'_{\wv,i} + \lambda\, R'_\wv \ , +\]` +where `$L'_{\wv,i} \in \R^d$` is a subgradient of the part of the loss function determined by the +`$i$`-th datapoint, that is `$L'_{\wv,i} \in \frac{\partial}{\partial \wv} L(\wv;\x_i,y_i)$`. +Furthermore, `$R'_\wv$` is a subgradient of the regularizer `$R(\wv)$`, i.e. `$R'_\wv \in +\frac{\partial}{\partial \wv} R(\wv)$`. The term `$R'_\wv$` does not depend on which random +datapoint is picked. + + + +**Gradients.** +The following table summarizes the gradients (or subgradients) of all loss functions and +regularizers that we currently support: + +| Function | Stochastic (Sub)Gradient | |
|---|---|---|
| SVM Hinge Loss | $L(\wv;\x_i,y_i) := \max \{0, 1-y_i \wv^T \x_i \}$ | +$L'_{\wv,i} = \begin{cases}-y_i \x_i & \text{if $y_i \wv^T \x_i <1$}, \\ 0 & +\text{otherwise}.\end{cases}$ | +
| Logistic Loss | $L(\wv;\x_i,y_i) := \log(1+\exp( -y_i \wv^T \x_i))$ | +$L'_{\wv,i} = -y_i \x_i \left(1-\frac1{1+\exp(-y_i \wv^T \x_i)} \right)$ | +
| Least Squares Loss | $L(\wv;\x_i,y_i) := (A_{i:} \wv - y_i)^2$ | +$L'_{\wv,i} = 2 A_{i:}^T (A_{i:} \wv - y_i)$ | +
| Non-Regularized | $R(\wv) := 0$ | $R'_\wv = \0$ | +
| L2 Regularizer | $R(\wv) := \frac{1}{2}\|\wv\|^2$ | $R'_\wv = \wv$ | +
| L1 Regularizer | $R(\wv) := \|\wv\|_1$ | $R'_\wv = \mathop{sign}(\wv)$ | +
| Source | Artifact |
|---|---|
| Kafka | spark-streaming-kafka_{{site.SCALA_VERSION}} |
| Flume | spark-streaming-flume_{{site.SCALA_VERSION}} |
| spark-streaming-twitter_{{site.SCALA_VERSION}} | |
| ZeroMQ | spark-streaming-zeromq_{{site.SCALA_VERSION}} |
| MQTT | spark-streaming-mqtt_{{site.SCALA_VERSION}} |
| Kafka | spark-streaming-kafka_{{site.SCALA_BINARY_VERSION}} |
| Flume | spark-streaming-flume_{{site.SCALA_BINARY_VERSION}} |
| spark-streaming-twitter_{{site.SCALA_BINARY_VERSION}} | |
| ZeroMQ | spark-streaming-zeromq_{{site.SCALA_BINARY_VERSION}} |
| MQTT | spark-streaming-mqtt_{{site.SCALA_BINARY_VERSION}} |
@@ -428,9 +428,9 @@ KafkaUtils.createStream(javaStreamingContext, kafkaParams, ...);
-For more details on these additional sources, see the corresponding [API documentation]
-(#where-to-go-from-here). Furthermore, you can also implement your own custom receiver
-for your sources. See the [Custom Receiver Guide](streaming-custom-receivers.html).
+For more details on these additional sources, see the corresponding [API documentation](#where-to-go-from-here).
+Furthermore, you can also implement your own custom receiver for your sources. See the
+[Custom Receiver Guide](streaming-custom-receivers.html).
## Operations
There are two kinds of DStream operations - _transformations_ and _output operations_. Similar to
@@ -520,9 +520,8 @@ The last two transformations are worth highlighting again.
UpdateStateByKey Operation
-The `updateStateByKey` operation allows -you to main arbitrary stateful computation, where you want to maintain some state data and -continuously update it with new information. To use this, you will have to do two steps. +The `updateStateByKey` operation allows you to maintain arbitrary state while continuously updating +it with new information. To use this, you will have to do two steps. 1. Define the state - The state can be of arbitrary data type. 1. Define the state update function - Specify with a function how to update the state using the @@ -925,7 +924,7 @@ exception saying so. ## Monitoring Besides Spark's in-built [monitoring capabilities](monitoring.html), the progress of a Spark Streaming program can also be monitored using the [StreamingListener] -(streaming/index.html#org.apache.spark.scheduler.StreamingListener) interface, +(api/streaming/index.html#org.apache.spark.scheduler.StreamingListener) interface, which allows you to get statistics of batch processing times, queueing delays, and total end-to-end delays. Note that this is still an experimental API and it is likely to be improved upon (i.e., more information reported) in the future. @@ -1000,11 +999,11 @@ Since all data is modeled as RDDs with their lineage of deterministic operations for output operations. ## Failure of the Driver Node -To allows a streaming application to operate 24/7, Spark Streaming allows a streaming computation +For a streaming application to operate 24/7, Spark Streaming allows a streaming computation to be resumed even after the failure of the driver node. Spark Streaming periodically writes the metadata information of the DStreams setup through the `StreamingContext` to a HDFS directory (can be any Hadoop-compatible filesystem). This periodic -*checkpointing* can be enabled by setting a the checkpoint +*checkpointing* can be enabled by setting the checkpoint directory using `ssc.checkpoint(*
- * - * This blog post has a good overview of the Count-Min Sketch (CMS). The CMS is a data structure - * for approximate frequency estimation in data streams (e.g. Top-K elements, frequency of any given element, etc), - * that uses space sub-linear in the number of elements in the stream. Once elements are added to the CMS, the - * estimated count of an element can be computed, as well as "heavy-hitters" that occur more than a threshold - * percentage of the overall total count. + * + * This blog post has a good overview of the Count-Min Sketch (CMS). The CMS is a data + * structure for approximate frequency estimation in data streams (e.g. Top-K elements, frequency + * of any given element, etc), that uses space sub-linear in the number of elements in the + * stream. Once elements are added to the CMS, the estimated count of an element can be computed, + * as well as "heavy-hitters" that occur more than a threshold percentage of the overall total + * count. *
- * Algebird's implementation is a monoid, so we can succinctly merge two CMS instances in the reduce operation. + * Algebird's implementation is a monoid, so we can succinctly merge two CMS instances in the + * reduce operation. */ +// scalastyle:on object TwitterAlgebirdCMS { def main(args: Array[String]) { if (args.length < 1) { diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala index a0094d460feec..45771d7050eeb 100644 --- a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala +++ b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala @@ -23,7 +23,7 @@ import com.twitter.algebird.HyperLogLog._ import org.apache.spark.storage.StorageLevel import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.streaming.twitter._ - +// scalastyle:off /** * Illustrates the use of the HyperLogLog algorithm, from Twitter's Algebird library, to compute * a windowed and global estimate of the unique user IDs occurring in a Twitter stream. @@ -31,12 +31,15 @@ import org.apache.spark.streaming.twitter._ *
* This * blog post and this - * blog post - * have good overviews of HyperLogLog (HLL). HLL is a memory-efficient datastructure for estimating - * the cardinality of a data stream, i.e. the number of unique elements. + * + * blog post + * have good overviews of HyperLogLog (HLL). HLL is a memory-efficient datastructure for + * estimating the cardinality of a data stream, i.e. the number of unique elements. *
- * Algebird's implementation is a monoid, so we can succinctly merge two HLL instances in the reduce operation.
+ * Algebird's implementation is a monoid, so we can succinctly merge two HLL instances in the
+ * reduce operation.
*/
+// scalastyle:on
object TwitterAlgebirdHLL {
def main(args: Array[String]) {
if (args.length < 1) {
@@ -82,7 +85,8 @@ object TwitterAlgebirdHLL {
userSet ++= partial
println("Exact distinct users this batch: %d".format(partial.size))
println("Exact distinct users overall: %d".format(userSet.size))
- println("Error rate: %2.5f%%".format(((globalHll.estimatedSize / userSet.size.toDouble) - 1) * 100))
+ println("Error rate: %2.5f%%".format(((globalHll.estimatedSize / userSet.size.toDouble) - 1
+ ) * 100))
}
})
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
index 85b4ce5e81950..35be7ffa1e872 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
@@ -53,11 +53,13 @@ object SimpleZeroMQPublisher {
}
}
+// scalastyle:off
/**
* A sample wordcount with ZeroMQStream stream
*
* To work with zeroMQ, some native libraries have to be installed.
- * Install zeroMQ (release 2.1) core libraries. [ZeroMQ Install guide](http://www.zeromq.org/intro:get-the-software)
+ * Install zeroMQ (release 2.1) core libraries. [ZeroMQ Install guide]
+ * (http://www.zeromq.org/intro:get-the-software)
*
* Usage: ZeroMQWordCount