Merge pull request #81 from ChrisCummins/fix/bettercode

bettercodehub.com fixes
ChrisCummins · Jan 24, 2020 · 3c70a3a · 3c70a3a
2 parents d779310 + 64d8177
commit 3c70a3a
Show file tree

Hide file tree

Showing 42 changed files with 1,998 additions and 367 deletions.
diff --git a/BUILD b/BUILD
@@ -75,16 +75,6 @@ python_grpc_library(
     deps = ["//:config_pb"],
 )
 
-python_string_genrule(
-    name = "config_pbtxt_py",
-    src = ":config",
-)
-
-filegroup(
-    name = "configure_py",
-    srcs = ["configure"],
-)
-
 py_library(
     name = "conftest",
     testonly = 1,

diff --git a/compilers/llvm/BUILD b/compilers/llvm/BUILD
@@ -146,7 +146,10 @@ py_library(
 py_binary(
     name = "opt",
     srcs = ["opt.py"],
-    data = ["//third_party/llvm:opt"],
+    data = [
+        ":opt_transform_passes",
+        "//third_party/llvm:opt",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         ":llvm",
@@ -166,6 +169,11 @@ py_test(
     ],
 )
 
+filegroup(
+    name = "opt_transform_passes",
+    srcs = ["opt_transform_passes.json"],
+)
+
 py_library(
     name = "opt_util",
     srcs = ["opt_util.py"],

diff --git a/datasets/github/scrape_repos/README.md b/datasets/github/scrape_repos/README.md
@@ -5,7 +5,7 @@ GitHub.
 
 ## Pre-requisites
 
-See [//datasets/github](/datasets/github) for instructions on how to 
+See [//datasets/github](/datasets/github) for instructions on how to
 set up your GitHub credentials.
 
 

diff --git a/datasets/github/scrape_repos/preprocessors/JavaMethodsBatchedExtractor.java b/datasets/github/scrape_repos/preprocessors/JavaMethodsBatchedExtractor.java
@@ -23,8 +23,8 @@
 package datasets.github.scrape_repos.preprocessors;
 
 import com.google.common.io.ByteStreams;
-import datasets.github.scrape_repos.ScrapeReposProtos.ListOfListOfStrings;
-import datasets.github.scrape_repos.ScrapeReposProtos.ListOfStrings;
+import com.scrape_repos.ListOfListOfStrings;
+import com.scrape_repos.ListOfStrings;
 import java.io.IOException;
 import java.lang.reflect.Modifier;
 import org.eclipse.jdt.core.dom.AST;

diff --git a/datasets/github/scrape_repos/preprocessors/JavaMethodsExtractor.java b/datasets/github/scrape_repos/preprocessors/JavaMethodsExtractor.java
@@ -24,7 +24,7 @@
 package datasets.github.scrape_repos.preprocessors;
 
 import com.google.common.io.ByteStreams;
-import datasets.github.scrape_repos.ScrapeReposProtos.ListOfStrings;
+import com.scrape_repos.ListOfStrings;
 import java.io.IOException;
 import java.lang.reflect.Modifier;
 import org.eclipse.jdt.core.dom.AST;

diff --git a/datasets/github/scrape_repos/proto/scrape_repos.proto b/datasets/github/scrape_repos/proto/scrape_repos.proto
@@ -13,11 +13,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 syntax = "proto2";
+
 package scrape_repos;
 
-option java_package = "datasets.github.scrape_repos";
-option java_outer_classname = "ScrapeReposProtos";
+option go_package = "scrape_repospb";
+option java_multiple_files = true;
+option java_outer_classname = "ScrapeReposProto";
+option java_package = "com.scrape_repos";
 
 // The schema for "clone lists", which are used to determine the GitHub queries
 // to run and repos to clone.

diff --git a/deeplearning/clgen/corpuses/BUILD b/deeplearning/clgen/corpuses/BUILD
@@ -30,10 +30,12 @@ py_library(
 py_test(
     name = "atomizers_test",
     srcs = ["atomizers_test.py"],
+    data = [":token_lists"],
     deps = [
         ":atomizers",
         "//deeplearning/clgen/tests:fixtures",
         "//labm8/py:app",
+        "//labm8/py:bazelutil",
         "//labm8/py:test",
     ],
 )
@@ -132,3 +134,9 @@ py_library(
         "//third_party/py/sqlalchemy",
     ],
 )
+
+filegroup(
+    name = "token_lists",
+    srcs = ["token_lists.json"],
+    visibility = ["//visibility:public"],
+)
diff --git a/deeplearning/clgen/corpuses/atomizers_test.py b/deeplearning/clgen/corpuses/atomizers_test.py
@@ -13,132 +13,27 @@
 # You should have received a copy of the GNU General Public License
 # along with clgen.  If not, see <https://www.gnu.org/licenses/>.
 """Unit tests for //deeplearning/clgen/corpuses:atomizers."""
+import json
 import pathlib
 import tempfile
 
 import deeplearning.clgen.errors
 from deeplearning.clgen.corpuses import atomizers
 from labm8.py import app
+from labm8.py import bazelutil
 from labm8.py import test
 
 FLAGS = app.FLAGS
 
 pytest_plugins = ["deeplearning.clgen.tests.fixtures"]
 
 # The set of multichar tokens for the OpenCL programming language.
-OPENCL_ATOMS = set(
-  [
-    "  ",
-    "__assert",
-    "__attribute",
-    "__builtin_astype",
-    "__clc_fabs",
-    "__clc_fma",
-    "__constant",
-    "__global",
-    "__inline",
-    "__kernel",
-    "__local",
-    "__private",
-    "__read_only",
-    "__read_write",
-    "__write_only",
-    "*/",
-    "/*",
-    "//",
-    "abs",
-    "alignas",
-    "alignof",
-    "atomic_add",
-    "auto",
-    "barrier",
-    "bool",
-    "break",
-    "case",
-    "char",
-    "clamp",
-    "complex",
-    "const",
-    "constant",
-    "continue",
-    "default",
-    "define",
-    "defined",
-    "do",
-    "double",
-    "elif",
-    "else",
-    "endif",
-    "enum",
-    "error",
-    "event_t",
-    "extern",
-    "fabs",
-    "false",
-    "float",
-    "for",
-    "get_global_id",
-    "get_global_size",
-    "get_local_id",
-    "get_local_size",
-    "get_num_groups",
-    "global",
-    "goto",
-    "half",
-    "if",
-    "ifdef",
-    "ifndef",
-    "image1d_array_t",
-    "image1d_buffer_t",
-    "image1d_t",
-    "image2d_array_t",
-    "image2d_t",
-    "image3d_t",
-    "imaginary",
-    "include",
-    "inline",
-    "int",
-    "into",
-    "kernel",
-    "line",
-    "local",
-    "long",
-    "noreturn",
-    "pragma",
-    "private",
-    "quad",
-    "read_only",
-    "read_write",
-    "register",
-    "restrict",
-    "return",
-    "sampler_t",
-    "short",
-    "shuffle",
-    "signed",
-    "size_t",
-    "sizeof",
-    "sqrt",
-    "static",
-    "struct",
-    "switch",
-    "true",
-    "typedef",
-    "u32",
-    "uchar",
-    "uint",
-    "ulong",
-    "undef",
-    "union",
-    "unsigned",
-    "void",
-    "volatile",
-    "while",
-    "wide",
-    "write_only",
-  ]
+TOKEN_LISTS = json.loads(
+  bazelutil.DataString("phd/deeplearning/clgen/corpuses/token_lists.json")
 )
 
+OPENCL_ATOMS = TOKEN_LISTS["opencl"]["tokens"]
+
 # AsciiCharacterAtomizer
 
 

diff --git a/deeplearning/clgen/corpuses/corpuses_test.py b/deeplearning/clgen/corpuses/corpuses_test.py
@@ -18,14 +18,13 @@
 import pathlib
 import tempfile
 
-from labm8.py import app
-from labm8.py import test
-
 from deeplearning.clgen import errors
 from deeplearning.clgen.corpuses import corpuses
 from deeplearning.clgen.corpuses import encoded
 from deeplearning.clgen.corpuses import preprocessed
 from deeplearning.clgen.proto import corpus_pb2
+from labm8.py import app
+from labm8.py import test
 
 FLAGS = app.FLAGS