Parse Starlark files as raw bytes for Bzlmod

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/CompiledModuleFile.java

@@ -63,7 +63,8 @@ public static CompiledModuleFile parseAndCompile(
       ExtendedEventHandler eventHandler)
       throws ExternalDepsException {
     StarlarkFile starlarkFile =
-        StarlarkFile.parse(ParserInput.fromUTF8(moduleFile.getContent(), moduleFile.getLocation()));
+        StarlarkFile.parse(
+            ParserInput.fromLatin1(moduleFile.getContent(), moduleFile.getLocation()));
     if (!starlarkFile.ok()) {
       Event.replayEventsOn(eventHandler, starlarkFile.errors());
       throw ExternalDepsException.withMessage(

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/VendorFileFunction.java

@@ -156,7 +156,7 @@ private static StarlarkFile readAndParseVendorFile(Path path, Environment env)
           new IOException("error reading VENDOR.bazel file", e), Transience.TRANSIENT);
     }
     StarlarkFile starlarkFile =
-        StarlarkFile.parse(ParserInput.fromUTF8(contents, path.getPathString()));
+        StarlarkFile.parse(ParserInput.fromLatin1(contents, path.getPathString()));
     if (!starlarkFile.ok()) {
       Event.replayEventsOn(env.getListener(), starlarkFile.errors());
       throw new VendorFileFunctionException(

src/main/java/com/google/devtools/build/lib/skyframe/RepoFileFunction.java

@@ -108,7 +108,7 @@ private static StarlarkFile readAndParseRepoFile(Path path, Environment env)
           new IOException("error reading REPO.bazel file at " + path, e), Transience.TRANSIENT);
     }
     StarlarkFile starlarkFile =
-        StarlarkFile.parse(ParserInput.fromUTF8(contents, path.getPathString()));
+        StarlarkFile.parse(ParserInput.fromLatin1(contents, path.getPathString()));
     if (!starlarkFile.ok()) {
       Event.replayEventsOn(env.getListener(), starlarkFile.errors());
       throw new RepoFileFunctionException(

src/test/py/bazel/bzlmod/bazel_module_test.py

@@ -18,6 +18,7 @@
 import pathlib
 import shutil
 import subprocess
+import sys
 import tempfile
 from absl.testing import absltest
 from src.test.py.bazel import test_base
@@ -1109,6 +1110,28 @@ def testRegression22754(self):
     self.ScratchFile('testdata/WORKSPACE')
     self.RunBazel(['build', ':all'])
 
+  def testUnicodePaths(self):
+    if sys.getfilesystemencoding() != 'utf-8':
+      self.skipTest('Test requires UTF-8 by default (Python 3.7+)')
+
+    unicode_dir = 'äöüÄÖÜß'
+    self.ScratchFile(unicode_dir + '/MODULE.bazel', ['module(name = "module")'])
+    self.ScratchFile(unicode_dir + '/BUILD', [
+      'filegroup(name = "choose_me")',
+    ])
+    self.writeMainProjectFiles()
+    self.ScratchFile(
+      'MODULE.bazel',
+      [
+        'bazel_dep(name = "module")',
+        'local_path_override(',
+        '  module_name = "module",',
+        '  path = "%s",' % unicode_dir,
+        ')',
+        ],
+    )
+    self.RunBazel(['build', '@module//:choose_me'])
+
 
 if __name__ == '__main__':
   absltest.main()

src/test/py/bazel/bzlmod/test_utils.py

@@ -30,8 +30,8 @@
 
 def download(url):
   """Download a file and return its content in bytes."""
-  response = urllib.request.urlopen(url)
-  return response.read()
+  with urllib.request.urlopen(url) as response:
+    return response.read()
 
 
 def read(path):

src/test/py/bazel/test_base.py

@@ -16,7 +16,6 @@
 
 """Bazel Python integration test framework."""
 
-import locale
 import os
 import shutil
 import socket
@@ -313,7 +312,7 @@ def ScratchFile(self, path, lines=None, executable=False):
     if os.path.exists(abspath) and not os.path.isfile(abspath):
       raise IOError('"%s" (%s) exists and is not a file' % (path, abspath))
     self.ScratchDir(os.path.dirname(path))
-    with open(abspath, 'w') as f:
+    with open(abspath, 'w', encoding='utf-8') as f:
       if lines:
         for l in lines:
           f.write(l)
@@ -445,7 +444,7 @@ def StopRemoteWorker(self):
 
     self._worker_stdout.seek(0)
     stdout_lines = [
-        l.decode(locale.getpreferredencoding()).strip()
+        l.decode('utf-8').strip()
         for l in self._worker_stdout.readlines()
     ]
     if stdout_lines:
@@ -455,7 +454,7 @@ def StopRemoteWorker(self):
 
     self._worker_stderr.seek(0)
     stderr_lines = [
-        l.decode(locale.getpreferredencoding()).strip()
+        l.decode('utf-8').strip()
         for l in self._worker_stderr.readlines()
     ]
     if stderr_lines:
@@ -509,18 +508,14 @@ def RunProgram(
 
         stdout.seek(0)
         stdout_lines = [
-            l.decode(locale.getpreferredencoding()).rstrip()
-            if rstrip
-            else l.decode(locale.getpreferredencoding()).strip()
-            for l in stdout.readlines()
+          l.decode('utf-8').rstrip() if rstrip else l.decode('utf-8').strip()
+          for l in stdout.readlines()
         ]
 
         stderr.seek(0)
         stderr_lines = [
-            l.decode(locale.getpreferredencoding()).rstrip()
-            if rstrip
-            else l.decode(locale.getpreferredencoding()).strip()
-            for l in stderr.readlines()
+          l.decode('utf-8').rstrip() if rstrip else l.decode('utf-8').strip()
+          for l in stderr.readlines()
         ]
 
         if not allow_failure: