Upgrade antlr to 4.11

.gitignore

@@ -17,3 +17,4 @@ pip-wheel-metadata
 .ipynb_checkpoints
 /.dmypy.json
 TODO.txt
+/venv

build_helpers/bin/antlr4

@@ -4,7 +4,7 @@ import subprocess
 import sys
 
 my_dir = os.path.realpath(os.path.dirname(__file__))
-antlr = "antlr-4.9.3-complete.jar"
+antlr = "antlr-4.11.1-complete.jar"
 args = ["java", "-jar", my_dir + "/" + antlr]
 args.extend(sys.argv[1:])
 subprocess.check_call(args)

build_helpers/bin/grun

@@ -4,7 +4,7 @@ import subprocess
 import sys
 
 my_dir = os.path.realpath(os.path.dirname(__file__))
-antlr = "antlr-4.9.3-complete.jar"
+antlr = "antlr-4.11.1-complete.jar"
 args = ["java", "-cp", my_dir + "/" + antlr, "org.antlr.v4.gui.TestRig"]
 args.extend(sys.argv[1:])
 subprocess.check_call(args)

build_helpers/build_helpers.py

@@ -6,7 +6,9 @@
 import re
 import shutil
 import subprocess
+from functools import partial
 from os.path import abspath, basename, dirname, exists, isdir, join
+from pathlib import Path
 from typing import List, Optional
 
 from setuptools import Command
@@ -185,7 +187,7 @@ def run(self) -> None:
             command = [
                 "java",
                 "-jar",
-                join(root_dir, "bin/antlr-4.9.3-complete.jar"),
+                join(root_dir, "bin/antlr-4.11.1-complete.jar"),
                 "-Dlanguage=Python3",
                 "-o",
                 join(project_root, "hydra/grammar/gen/"),
@@ -198,8 +200,37 @@ def run(self) -> None:
 
             subprocess.check_call(command)
 
+            log.info("Replacing imports of antlr4 in generated parsers")
+            self._fix_imports()
+
     def initialize_options(self) -> None:
         pass
 
     def finalize_options(self) -> None:
         pass
+
+    def _fix_imports(self) -> None:
+        """Fix imports from the generated parsers to use the vendored antlr4 instead"""
+        build_dir = Path(__file__).parent.absolute()
+        project_root = build_dir.parent
+        lib = "antlr4"
+        pkgname = 'omegaconf.vendor'
+
+        replacements = [
+            partial(  # import antlr4 -> import omegaconf.vendor.antlr4
+                re.compile(r'(^\s*)import {}\n'.format(lib), flags=re.M).sub,
+                r'\1from {} import {}\n'.format(pkgname, lib)
+            ),
+            partial(  # from antlr4 -> from fomegaconf.vendor.antlr4
+                re.compile(r'(^\s*)from {}(\.|\s+)'.format(lib), flags=re.M).sub,
+                r'\1from {}.{}\2'.format(pkgname, lib)
+            ),
+        ]
+
+        path = project_root / "hydra" / "grammar" / "gen"
+        for item in path.iterdir():
+            if item.is_file() and item.name.endswith(".py"):
+                text = item.read_text('utf8')
+                for replacement in replacements:
+                    text = replacement(text)
+                item.write_text(text, 'utf8')

hydra/core/override_parser/overrides_parser.py

@@ -2,7 +2,10 @@
 import sys
 from typing import Any, List, Optional
 
-from antlr4.error.Errors import LexerNoViableAltException, RecognitionException
+from omegaconf.vendor.antlr4.error.Errors import (
+    LexerNoViableAltException,
+    RecognitionException,
+)
 
 from hydra._internal.grammar import grammar_functions
 from hydra._internal.grammar.functions import Functions

hydra/core/override_parser/overrides_visitor.py

@@ -3,9 +3,9 @@
 import warnings
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from antlr4 import ParserRuleContext, TerminalNode, Token
-from antlr4.error.ErrorListener import ErrorListener
-from antlr4.tree.Tree import TerminalNodeImpl
+from omegaconf.vendor.antlr4 import ParserRuleContext, TerminalNode, Token
+from omegaconf.vendor.antlr4.error.ErrorListener import ErrorListener
+from omegaconf.vendor.antlr4.tree.Tree import TerminalNodeImpl
 
 from hydra._internal.grammar.functions import FunctionCall, Functions
 from hydra._internal.grammar.utils import _ESC_QUOTED_STR

requirements/requirements.txt

@@ -1,4 +1,3 @@
-omegaconf>=2.2,<2.4
-antlr4-python3-runtime==4.9.*
+omegaconf==2.4.0.dev1
 importlib-resources;python_version<'3.9'
 packaging