[ML-42] [PIP] Misc improvements and refactor code

.github/workflows/oap-mllib-ci.yml

@@ -27,5 +27,5 @@ jobs:
         run: |          
           source ${{github.workspace}}/dev/setup-all.sh
       - name: Build and Test
-        run: |          
+        run: |
           ${{github.workspace}}/dev/ci-test.sh

dev/ci-test.sh

@@ -54,8 +54,9 @@ for SparkVer in ${SupportedSparkVersions[*]}; do
 
     mvn --no-transfer-progress -P$SparkVer -Dtest=none -DwildcardSuites=org.apache.spark.ml.clustering.IntelKMeansSuite test
     mvn --no-transfer-progress -P$SparkVer -Dtest=none -DwildcardSuites=org.apache.spark.ml.feature.IntelPCASuite test
-    # mvn -P$SparkVer -Dtest=none -DwildcardSuites=org.apache.spark.ml.recommendation.IntelALSSuite test
+    # mvn --no-transfer-progress -P$SparkVer -Dtest=none -DwildcardSuites=org.apache.spark.ml.recommendation.IntelALSSuite test
 done
 
 # Yarn cluster test without profile
-$GITHUB_WORKSPACE/dev/test-cluster/ci-test-cluster.sh
+$GITHUB_WORKSPACE/dev/ci-build.sh
+$GITHUB_WORKSPACE/dev/test-cluster/ci-test-cluster.sh

dev/codestyle/lint-scala.sh

@@ -23,7 +23,7 @@ if [ -z $MVN ]; then
     exit 1
 fi
 
-ERRORS=$($MVN scalastyle:check | grep error)
+ERRORS=$($MVN scalastyle:check | grep "error file")
 
 if test ! -z "$ERRORS"; then
     echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS"

mllib-dal/pom.xml

@@ -1,5 +1,5 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
@@ -273,7 +273,7 @@
                     <testSourceDirectories>
                         <directory>${basedir}/src/test/java</directory>
                     </testSourceDirectories>
-                        <configLocation>${basedir}/../dev/codestyle/checkstyle.xml</configLocation>
+                    <configLocation>${basedir}/../dev/codestyle/checkstyle.xml</configLocation>
                     <outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
                 </configuration>
                 <dependencies>
@@ -303,7 +303,7 @@
                     <failOnViolation>true</failOnViolation>
                     <includeTestSourceDirectory>false</includeTestSourceDirectory>
                     <failOnWarning>false</failOnWarning>
-                    <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+                    <sourceDirectory>${basedir}/src/main/scala/org/apache/spark</sourceDirectory>
                     <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
                     <configLocation>scalastyle-config.xml</configLocation>
                     <outputFile>${basedir}/target/scalastyle-output.xml</outputFile>
@@ -344,7 +344,7 @@
                         </goals>
                     </execution>
                 </executions>
-            </plugin>           
+            </plugin>
 
             <plugin>
                 <artifactId>maven-antrun-plugin</artifactId>
@@ -354,8 +354,9 @@
                         <phase>process-classes</phase>
                         <configuration>
                             <target name="build-natives" description="Build native code">
-                                <echo>Building native code</echo>                                
-                                <exec dir="${project.basedir}/src/main/native" executable="./build.sh" failonerror="true">
+                                <echo>Building native code</echo>
+                                <exec dir="${project.basedir}/src/main/native" executable="./build.sh"
+                                      failonerror="true">
                                 </exec>
                             </target>
                         </configuration>
@@ -375,11 +376,11 @@
                         <resource>
                             <directory>${env.CCL_ROOT}/lib</directory>
                             <includes>
-				                <include>${ccl.lib}</include>
+                                <include>${ccl.lib}</include>
                                 <include>${ccl.mpi.lib}</include>
                                 <include>${ccl.fabric.lib}</include>
                             </includes>
-                        </resource>                        
+                        </resource>
                         <resource>
                             <directory>${env.CCL_ROOT}/lib/prov</directory>
                             <includes>
@@ -418,18 +419,21 @@
                         <configuration>
                             <fileSets>
                                 <!-- oneDAL Java API doesn't load correct libtbb version for oneAPI Beta 10,
-                                rename to workaround. See https://github.com/oneapi-src/oneDAL/issues/1254 -->                                
+                                rename to workaround. See https://github.com/oneapi-src/oneDAL/issues/1254 -->
                                 <fileSet>
                                     <sourceFile>${project.build.testOutputDirectory}/lib/${tbb.lib}</sourceFile>
-                                    <destinationFile>${project.build.testOutputDirectory}/lib/libtbb.so.2</destinationFile>
+                                    <destinationFile>${project.build.testOutputDirectory}/lib/libtbb.so.2
+                                    </destinationFile>
                                 </fileSet>
                                 <fileSet>
                                     <sourceFile>${project.build.testOutputDirectory}/lib/${tbb.malloc.lib}</sourceFile>
-                                    <destinationFile>${project.build.testOutputDirectory}/lib/libtbbmalloc.so.2</destinationFile>
+                                    <destinationFile>${project.build.testOutputDirectory}/lib/libtbbmalloc.so.2
+                                    </destinationFile>
                                 </fileSet>
                                 <fileSet>
                                     <sourceFile>${project.build.testOutputDirectory}/lib/${ccl.mpi.lib}</sourceFile>
-                                    <destinationFile>${project.build.testOutputDirectory}/lib/libmpi.so.12</destinationFile>
+                                    <destinationFile>${project.build.testOutputDirectory}/lib/libmpi.so.12
+                                    </destinationFile>
                                 </fileSet>
                             </fileSets>
                         </configuration>

mllib-dal/scalastyle-config.xml

@@ -49,14 +49,13 @@ This file is divided into 3 sections:
   <check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
     <parameters>
        <parameter name="header"><![CDATA[/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
+ * Copyright 2020 Intel Corporation
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -142,7 +141,8 @@ This file is divided into 3 sections:
   </check>
 
   <!-- As of SPARK-7977 all printlns need to be wrapped in '// scalastyle:off/on println' -->
-  <check customId="println" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
+  <!-- Disabled for OAP MLlib -->
+  <check customId="println" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="false">
     <parameters><parameter name="regex">^println$</parameter></parameters>
     <customMessage><![CDATA[Are you sure you want to println? If yes, wrap the code block with
       // scalastyle:off println

mllib-dal/src/main/java/org/apache/spark/ml/clustering/KMeansResult.java

@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*
  * Copyright 2020 Intel Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,12 +12,11 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *******************************************************************************/
+ */
 
 package org.apache.spark.ml.clustering;
 
 public class KMeansResult {
-    public long cNumericTable;
-    public int iterationNum;
-    public double totalCost;
+  public int iterationNum;
+  public double totalCost;
 }

mllib-dal/src/main/java/org/apache/spark/ml/feature/PCAResult.java

@@ -1,6 +1,22 @@
+/*
+ * Copyright 2020 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.ml.feature;
 
 public class PCAResult {
-    public long pcNumericTable;
-    public long explainedVarianceNumericTable;
+  public long pcNumericTable;
+  public long explainedVarianceNumericTable;
 }

mllib-dal/src/main/java/org/apache/spark/ml/recommendation/ALSPartitionInfo.java

@@ -1,6 +1,22 @@
+/*
+ * Copyright 2020 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.ml.recommendation;
 
 public class ALSPartitionInfo {
-    public int ratingsNum;
-    public int csrRowNum;
+  public int ratingsNum;
+  public int csrRowNum;
 }

mllib-dal/src/main/java/org/apache/spark/ml/recommendation/ALSResult.java

@@ -1,9 +1,25 @@
+/*
+ * Copyright 2020 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.ml.recommendation;
 
 public class ALSResult {
-    public long rankId = -1;
-    public long cUsersFactorsNumTab;
-    public long cItemsFactorsNumTab;
-    public long cUserOffset;
-    public long cItemOffset;
+  public long rankId = -1;
+  public long cUsersFactorsNumTab;
+  public long cItemsFactorsNumTab;
+  public long cUserOffset;
+  public long cItemOffset;
 }

mllib-dal/src/main/java/org/apache/spark/ml/util/CCLParam.java

@@ -1,4 +1,4 @@
-/*******************************************************************************
+/*
  * Copyright 2020 Intel Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,11 +12,11 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- *******************************************************************************/
+ */
 
 package org.apache.spark.ml.util;
 
 class CCLParam {
-    long commSize;
-    long rankId;
-}
+  long commSize;
+  long rankId;
+}