[Parser] Parser 2.0 part 2 (apache#6162)

* Add code from livestream with JK

* Fix errors parsing ResNet

* Parse metadata section efficiently and do most of plumbing to resolve metadata section references.

* WIP

* Change meta reference to an operator

* Meta references now work

* MetaReference expansion now works

* Start working on source map and move diagnostic context

* Convert tokenizer and parser to use new machinery

* Kill to_json

* Fix comment in type_infer.cc

* Remove old parser

* Rename parser tests and remove old ones

* Record span end information

* Convert to using spans everywhere

* Add span fields back to all Relay constructors

* Start passing spans

* Pass spans around visitors

* Format

* Fix

* Fix

* disable reference lint from string helpers

* Fix tokenizer

* Fix issue with empty metadata section

* Document new span fields and small tweaks

* Formatting

* Add span doc fields

* Add format tweak

* Improve errors and fix the semantic version tags in Prelude

* Update gradient.rly

* Clean up broken spans

* Clean up parser tests and turn on previously skipped tests

* Update errors to handle skipped cases

* Tweak

* Tweak

* Format

* Fix some minor issues with ADT tests

* Format

* Fix path

* WIP

* WIP

* Fix ir_text_printer

* format

* Formatted

* More formatting

* Repair test cases

* Fix CI

* Retrigger CI

.gitignore

@@ -230,7 +230,3 @@ conda/pkg
 # nix files
 .envrc
 *.nix
-
-# antlr files
-*.tokens
-*.interp

CMakeLists.txt

@@ -8,7 +8,6 @@ include(cmake/util/FindOpenCL.cmake)
 include(cmake/util/FindVulkan.cmake)
 include(cmake/util/FindLLVM.cmake)
 include(cmake/util/FindROCM.cmake)
-include(cmake/util/FindANTLR.cmake)
 
 if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/config.cmake)
   include(${CMAKE_CURRENT_BINARY_DIR}/config.cmake)
@@ -69,7 +68,6 @@ tvm_option(USE_NNPACK "Build with nnpack support" OFF)
 tvm_option(USE_RANDOM "Build with random support" OFF)
 tvm_option(USE_TENSORRT "Build with TensorRT, must have CUDA and CUDNN enabled" OFF)
 tvm_option(USE_MICRO_STANDALONE_RUNTIME "Build with micro.standalone_runtime support" OFF)
-tvm_option(USE_ANTLR "Build with ANTLR for Relay parsing" OFF)
 tvm_option(USE_CPP_RPC "Build CPP RPC" OFF)
 tvm_option(USE_TFLITE "Build with tflite support" OFF)
 tvm_option(USE_TENSORFLOW_PATH "TensorFlow root path when use TFLite" none)
@@ -324,7 +322,6 @@ include(cmake/modules/Metal.cmake)
 include(cmake/modules/ROCM.cmake)
 include(cmake/modules/LLVM.cmake)
 include(cmake/modules/Micro.cmake)
-include(cmake/modules/ANTLR.cmake)
 include(cmake/modules/contrib/BLAS.cmake)
 include(cmake/modules/contrib/CODEGENC.cmake)
 include(cmake/modules/contrib/DNNL.cmake)

docker/Dockerfile.ci_cpu

@@ -60,9 +60,6 @@ ENV PATH $PATH:$CARGO_HOME/bin:/usr/lib/go-1.10/bin
 COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
 RUN bash /install/ubuntu_install_java.sh
 
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
-
 # Chisel deps for TSIM
 COPY install/ubuntu_install_chisel.sh /install/ubuntu_install_chisel.sh
 RUN bash /install/ubuntu_install_chisel.sh

docker/Dockerfile.ci_gpu

@@ -87,9 +87,6 @@ RUN bash /install/ubuntu_install_vulkan.sh
 COPY install/ubuntu_install_redis.sh /install/ubuntu_install_redis.sh
 RUN bash /install/ubuntu_install_redis.sh
 
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
-
 # NNPACK deps
 COPY install/ubuntu_install_nnpack.sh /install/ubuntu_install_nnpack.sh
 RUN bash /install/ubuntu_install_nnpack.sh

docker/Dockerfile.ci_wasm

@@ -33,9 +33,6 @@ RUN bash /install/ubuntu1804_install_llvm.sh
 COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
 RUN bash /install/ubuntu_install_java.sh
 
-COPY install/ubuntu_install_antlr.sh /install/ubuntu_install_antlr.sh
-RUN bash /install/ubuntu_install_antlr.sh
-
 COPY install/ubuntu_install_nodejs.sh /install/ubuntu_install_nodejs.sh
 RUN bash /install/ubuntu_install_nodejs.sh
 

docker/install/ubuntu_install_python_package.sh

@@ -21,4 +21,4 @@ set -u
 set -o pipefail
 
 # install libraries for python package on ubuntu
-pip3 install pylint==1.9.4 six numpy pytest cython decorator scipy tornado typed_ast pytest mypy orderedset antlr4-python3-runtime attrs requests Pillow packaging
+pip3 install pylint==1.9.4 six numpy pytest cython decorator scipy tornado typed_ast pytest mypy orderedset attrs requests Pillow packaging

docs/README.txt

@@ -42,12 +42,12 @@ You can run the following script to reproduce the CI sphinx pre-check stage.
 This script skips the tutorial executions and is useful for quickly check the content.
 
 ```bash
-./tests/scrpts/task_sphinx_precheck.sh
+./tests/scripts/task_sphinx_precheck.sh
 ```
 
 The following script runs the full build which includes tutorial executions.
 You will need a gpu CI environment.
 
 ```bash
-./tests/scrpts/task_python_docs.sh
+./tests/scripts/task_python_docs.sh
 ```

docs/install/from_source.rst

@@ -213,13 +213,6 @@ like ``virtualenv``.
 
        pip3 install --user tornado psutil xgboost
 
-   * If you want to build tvm to compile a model, you must use Python 3 and run the following
-
-   .. code:: bash
-
-       sudo apt install antlr4
-       pip3 install --user mypy orderedset antlr4-python3-runtime
-
 
 Install Contrib Libraries
 -------------------------

include/tvm/ir/attrs.h

@@ -353,7 +353,8 @@ struct AttrInitEntry {
   ~AttrInitEntry() DMLC_THROW_EXCEPTION {
     if (value_missing_) {
       std::ostringstream os;
-      os << type_key_ << ": Cannot find required field \'" << key_ << "\' during initialization";
+      os << type_key_ << ": Cannot find required field \'" << key_ << "\' during initialization."
+         << "If the key is defined check that its type matches the declared type.";
       throw AttrError(os.str());
     }
   }

include/tvm/ir/span.h

@@ -80,21 +80,29 @@ class Span;
 class SpanNode : public Object {
  public:
   /*! \brief The source name. */
-  SourceName source;
+  SourceName source_name;
   /*! \brief The line number. */
   int line;
   /*! \brief The column offset. */
   int column;
+  /*! \brief The end line number. */
+  int end_line;
+  /*! \brief The end column number. */
+  int end_column;
 
   // override attr visitor
   void VisitAttrs(AttrVisitor* v) {
-    v->Visit("source", &source);
+    v->Visit("source_name", &source_name);
     v->Visit("line", &line);
     v->Visit("column", &column);
+    v->Visit("end_line", &end_line);
+    v->Visit("end_column", &end_column);
   }
 
   bool SEqualReduce(const SpanNode* other, SEqualReducer equal) const {
-    return equal(source, other->source) && equal(line, other->line) && equal(column, other->column);
+    return equal(source_name, other->source_name) && equal(line, other->line) &&
+           equal(column, other->column) && equal(end_line, other->end_line) &&
+           equal(end_column, other->end_column);
   }
 
   static constexpr const char* _type_key = "Span";
@@ -103,7 +111,10 @@ class SpanNode : public Object {
 
 class Span : public ObjectRef {
  public:
-  TVM_DLL Span(SourceName source, int lineno, int col_offset);
+  TVM_DLL Span(SourceName source_name, int line, int end_line, int column, int end_column);
+
+  /*! \brief Merge two spans into one which captures the combined regions. */
+  TVM_DLL Span Merge(const Span& other);
 
   TVM_DEFINE_OBJECT_REF_METHODS(Span, ObjectRef, SpanNode);
 };

include/tvm/parser/parser.h

@@ -32,7 +32,7 @@
 namespace tvm {
 namespace parser {
 
-IRModule Parse(std::string file_name, std::string file_content);
+IRModule ParseModule(std::string file_name, std::string file_content);
 
 }  // namespace parser
 }  // namespace tvm