From 414039c3797a832ae4729bde7fca163e4947fd62 Mon Sep 17 00:00:00 2001 From: "Wu, Xiaochang" Date: Sat, 8 May 2021 12:26:40 +0800 Subject: [PATCH 1/3] Add codestyle scripts for java/scala/cpp --- dev/codestyle/.clang-format | 136 +++++++ dev/codestyle/eclipse-java-google-style.xml | 337 +++++++++++++++++ dev/codestyle/format-cpp.sh | 29 ++ dev/codestyle/format-java.sh | 26 ++ dev/codestyle/format-scala.sh | 27 ++ dev/codestyle/gen-clang-format.sh | 29 ++ dev/codestyle/lint-cpp.sh | 36 ++ dev/codestyle/lint-java.sh | 33 ++ dev/codestyle/lint-scala.sh | 33 ++ mllib-dal/.scalafmt.conf | 28 ++ mllib-dal/pom.xml | 82 +++- mllib-dal/scalastyle-config.xml | 400 ++++++++++++++++++++ 12 files changed, 1195 insertions(+), 1 deletion(-) create mode 100644 dev/codestyle/.clang-format create mode 100644 dev/codestyle/eclipse-java-google-style.xml create mode 100755 dev/codestyle/format-cpp.sh create mode 100755 dev/codestyle/format-java.sh create mode 100755 dev/codestyle/format-scala.sh create mode 100755 dev/codestyle/gen-clang-format.sh create mode 100755 dev/codestyle/lint-cpp.sh create mode 100755 dev/codestyle/lint-java.sh create mode 100755 dev/codestyle/lint-scala.sh create mode 100644 mllib-dal/.scalafmt.conf create mode 100644 mllib-dal/scalastyle-config.xml diff --git a/dev/codestyle/.clang-format b/dev/codestyle/.clang-format new file mode 100644 index 000000000..0a9ea9178 --- /dev/null +++ b/dev/codestyle/.clang-format @@ -0,0 +1,136 @@ +--- +Language: Cpp +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Right +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + - Regex: '.*' + Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PointerAlignment: Right +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Latest +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Never +... + diff --git a/dev/codestyle/eclipse-java-google-style.xml b/dev/codestyle/eclipse-java-google-style.xml new file mode 100644 index 000000000..7bb6804eb --- /dev/null +++ b/dev/codestyle/eclipse-java-google-style.xml @@ -0,0 +1,337 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dev/codestyle/format-cpp.sh b/dev/codestyle/format-cpp.sh new file mode 100755 index 000000000..f81331202 --- /dev/null +++ b/dev/codestyle/format-cpp.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CLANG_FORMAT=$(which clang-format) + +if [ -z $CLANG_FORMAT ]; then + echo clang-format not found, please install it. + exit 1 +fi + +if [ -f .clang-format ]; then + echo .clang-format is not found in current directory, please generate it. + exit 1 +fi + +$CLANG_FORMAT -i --style=file *.cpp *.h diff --git a/dev/codestyle/format-java.sh b/dev/codestyle/format-java.sh new file mode 100755 index 000000000..0b869a1d6 --- /dev/null +++ b/dev/codestyle/format-java.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +MVN=$(which mvn) + +if [ -z $MVN ]; then + echo Apache Maven not found, please install it. + exit 1 +fi + +$MVN formatter:format diff --git a/dev/codestyle/format-scala.sh b/dev/codestyle/format-scala.sh new file mode 100755 index 000000000..9a7654e58 --- /dev/null +++ b/dev/codestyle/format-scala.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +MVN=$(which mvn) + +if [ -z $MVN ]; then + echo Apache Maven not found, please install it. + exit 1 +fi + +VERSION="${@:-2.12}" +$MVN mvn-scalafmt_$VERSION:format \ No newline at end of file diff --git a/dev/codestyle/gen-clang-format.sh b/dev/codestyle/gen-clang-format.sh new file mode 100755 index 000000000..d211fb84f --- /dev/null +++ b/dev/codestyle/gen-clang-format.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CLANG_FORMAT=$(which clang-format) + +if [ -z $CLANG_FORMAT ]; then + echo clang-format not found, please install it. + exit 1 +fi + +$CLANG_FORMAT -style="{ BasedOnStyle: LLVM, \ + UseTab: Never, \ + IndentWidth: 4, \ + TabWidth: 4 \ + }" \ + -dump-config > .clang-format diff --git a/dev/codestyle/lint-cpp.sh b/dev/codestyle/lint-cpp.sh new file mode 100755 index 000000000..a5fa94d30 --- /dev/null +++ b/dev/codestyle/lint-cpp.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CLANG_FORMAT=$(which clang-format) + +if [ -z $CLANG_FORMAT ]; then + echo clang-format not found, please install it. + exit 1 +fi + +if [ -f .clang-format ]; then + echo .clang-format is not found in current directory, please generate it. + exit 1 +fi + +ERRORS=$($CLANG_FORMAT --style=file -Werror -n *.cpp *.h 2> /dev/stdout 1> /dev/null | grep error) + +if test ! -z "$ERRORS"; then + echo -e "clang-format style checks failed at following occurrences:\n$ERRORS" + exit 1 +else + echo -e "clang-format style checks passed." +fi \ No newline at end of file diff --git a/dev/codestyle/lint-java.sh b/dev/codestyle/lint-java.sh new file mode 100755 index 000000000..e081e3929 --- /dev/null +++ b/dev/codestyle/lint-java.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +MVN=$(which mvn) + +if [ -z $MVN ]; then + echo Apache Maven not found, please install it. + exit 1 +fi + +ERRORS=$($MVN checkstyle:check | grep ERROR) + +if test ! -z "$ERRORS"; then + echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS" + exit 1 +else + echo -e "Checkstyle checks passed." +fi diff --git a/dev/codestyle/lint-scala.sh b/dev/codestyle/lint-scala.sh new file mode 100755 index 000000000..e18a731b4 --- /dev/null +++ b/dev/codestyle/lint-scala.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Copyright 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +MVN=$(which mvn) + +if [ -z $MVN ]; then + echo Apache Maven not found, please install it. + exit 1 +fi + +ERRORS=$($MVN scalastyle:check | grep error) + +if test ! -z "$ERRORS"; then + echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS" + exit 1 +else + echo -e "Scalastyle checks passed." +fi \ No newline at end of file diff --git a/mllib-dal/.scalafmt.conf b/mllib-dal/.scalafmt.conf new file mode 100644 index 000000000..9a8813e3b --- /dev/null +++ b/mllib-dal/.scalafmt.conf @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +align = none +align.openParenDefnSite = false +align.openParenCallSite = false +align.tokens = [] +optIn = { + configStyleArguments = false +} +danglingParentheses = false +docstrings = JavaDoc +maxColumn = 98 + diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml index cc2316502..b008484ee 100644 --- a/mllib-dal/pom.xml +++ b/mllib-dal/pom.xml @@ -259,7 +259,87 @@ - + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.0 + + false + true + + ${basedir}/src/main/java + + + ${basedir}/src/test/java + + google_checks.xml + ${basedir}/target/checkstyle-output.xml + + + + com.puppycrawl.tools + checkstyle + 8.29 + + + + + + check + + + + + + + net.revelc.code.formatter + formatter-maven-plugin + 2.15.0 + + ${basedir}/../dev/codestyle/eclipse-java-google-style.xml + + + + + org.scalastyle + scalastyle-maven-plugin + 1.0.0 + + false + true + false + false + ${basedir}/src/main/scala + ${basedir}/src/test/scala + scalastyle-config.xml + ${basedir}/target/scalastyle-output.xml + + + + + check + + + + + + + org.antipathy + mvn-scalafmt_${scala.binary.version} + 1.0.3 + + .scalafmt.conf + + + + validate + + format + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/mllib-dal/scalastyle-config.xml b/mllib-dal/scalastyle-config.xml new file mode 100644 index 000000000..c1dc57be5 --- /dev/null +++ b/mllib-dal/scalastyle-config.xml @@ -0,0 +1,400 @@ + + + + + Scalastyle standard configuration + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ARROW, EQUALS, ELSE, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + ARROW, EQUALS, COMMA, COLON, IF, ELSE, DO, WHILE, FOR, MATCH, TRY, CATCH, FINALLY, LARROW, RARROW + + + + + + + + + ^FunSuite[A-Za-z]*$ + Tests must extend org.apache.spark.SparkFunSuite instead. + + + + + ^println$ + + + + + spark(.sqlContext)?.sparkContext.hadoopConfiguration + + + + + @VisibleForTesting + + + + + Runtime\.getRuntime\.addShutdownHook + + + + + mutable\.SynchronizedBuffer + + + + + Class\.forName + + + + + Await\.result + + + + + Await\.ready + + + + + (\.toUpperCase|\.toLowerCase)(?!(\(|\(Locale.ROOT\))) + + + + + throw new \w+Error\( + + + + + + JavaConversions + Instead of importing implicits in scala.collection.JavaConversions._, import + scala.collection.JavaConverters._ and use .asScala / .asJava methods + + + + org\.apache\.commons\.lang\. + Use Commons Lang 3 classes (package org.apache.commons.lang3.*) instead + of Commons Lang 2 (package org.apache.commons.lang.*) + + + + FileSystem.get\([a-zA-Z_$][a-zA-Z_$0-9]*\) + + + + + extractOpt + Use jsonOption(x).map(.extract[T]) instead of .extractOpt[T], as the latter + is slower. + + + + + java,scala,3rdParty,spark + javax?\..* + scala\..* + (?!org\.apache\.spark\.).* + org\.apache\.spark\..* + + + + + + COMMA + + + + + + \)\{ + + + + + (?m)^(\s*)/[*][*].*$(\r|)\n^\1 [*] + Use Javadoc style indentation for multiline comments + + + + case[^\n>]*=>\s*\{ + Omit braces in case clauses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 800> + + + + + 30 + + + + + 10 + + + + + 50 + + + + + + + + + + + -1,0,1,2,3 + + + From a1fe2f9a69901f9159e590c926cf782f865da0f0 Mon Sep 17 00:00:00 2001 From: "Wu, Xiaochang" Date: Sat, 8 May 2021 17:56:47 +0800 Subject: [PATCH 2/3] Use Spark checkstyle & checkstyle-suppressions --- dev/codestyle/checkstyle-suppressions.xml | 32 ++++ dev/codestyle/checkstyle.xml | 190 ++++++++++++++++++++++ mllib-dal/pom.xml | 12 +- 3 files changed, 224 insertions(+), 10 deletions(-) create mode 100644 dev/codestyle/checkstyle-suppressions.xml create mode 100644 dev/codestyle/checkstyle.xml diff --git a/dev/codestyle/checkstyle-suppressions.xml b/dev/codestyle/checkstyle-suppressions.xml new file mode 100644 index 000000000..985f48f29 --- /dev/null +++ b/dev/codestyle/checkstyle-suppressions.xml @@ -0,0 +1,32 @@ + + + + + + + + + diff --git a/dev/codestyle/checkstyle.xml b/dev/codestyle/checkstyle.xml new file mode 100644 index 000000000..fdb673b0e --- /dev/null +++ b/dev/codestyle/checkstyle.xml @@ -0,0 +1,190 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml index b008484ee..b816a9722 100644 --- a/mllib-dal/pom.xml +++ b/mllib-dal/pom.xml @@ -273,7 +273,7 @@ ${basedir}/src/test/java - google_checks.xml + ${basedir}/../dev/codestyle/checkstyle.xml ${basedir}/target/checkstyle-output.xml @@ -329,16 +329,8 @@ mvn-scalafmt_${scala.binary.version} 1.0.3 - .scalafmt.conf + ${basedir}/.scalafmt.conf - - - validate - - format - - - org.apache.maven.plugins From d82bb1898d7c71dae04214b8c68deb999c6d630f Mon Sep 17 00:00:00 2001 From: "Wu, Xiaochang" Date: Sat, 8 May 2021 18:02:50 +0800 Subject: [PATCH 3/3] nit --- mllib-dal/pom.xml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml index b816a9722..055308cd6 100644 --- a/mllib-dal/pom.xml +++ b/mllib-dal/pom.xml @@ -283,13 +283,6 @@ 8.29 - - - - check - - - @@ -315,13 +308,6 @@ scalastyle-config.xml ${basedir}/target/scalastyle-output.xml - - - - check - - -