Added support for file encoding

fake_filesystem_test.py

@@ -17,6 +17,7 @@
 """Unittest for fake_filesystem module."""
 
 import errno
+import locale
 import os
 import re
 import stat
@@ -2487,6 +2488,7 @@ def tearDown(self):
 
 
 class FakeFileOpenTest(FakeFileOpenTestBase):
+
   def testOpenNoParentDir(self):
     """Expect raise when open'ing a file in a missing directory."""
     file_path = 'foo/bar.txt'
@@ -2526,15 +2528,18 @@ def testCompatibilityOfWithStatement(self):
   def testUnicodeContents(self):
     self.file = fake_filesystem.FakeFileOpen(self.filesystem)
     file_path = 'foo'
-    text_fractions =  '⅓ ⅔ ⅕ ⅖'
+    # note that this will work only if the string can be represented
+    # by the locale preferred encoding - which under Windows is
+    # usually not UTF-8, but something like Latin1, depending on the locale
+    text_fractions = 'Ümläüts'
     with self.file(file_path, 'w') as f:
       f.write(text_fractions)
     with self.file(file_path) as f:
       contents = f.read()
     self.assertEqual(contents, text_fractions)
 
   @unittest.skipIf(sys.version_info >= (3, 0),
-                   'Python2-specific behavior')
+                   'Python2 specific string handling')
   def testByteContentsPy2(self):
     self.file = fake_filesystem.FakeFileOpen(self.filesystem)
     file_path = 'foo'
@@ -2546,17 +2551,32 @@ def testByteContentsPy2(self):
     self.assertEqual(contents, byte_fractions)
 
   @unittest.skipIf(sys.version_info < (3, 0),
-                   'Python3-specific behavior')
+                   'Python3 specific string handling')
   def testByteContentsPy3(self):
     self.file = fake_filesystem.FakeFileOpen(self.filesystem)
     file_path = 'foo'
     byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96'
     with self.file(file_path, 'wb') as f:
       f.write(byte_fractions)
-    with self.file(file_path) as f:
+    # the encoding has to be specified, otherwise the locale default is used which
+    # can be different on different systems
+    with self.file(file_path, encoding='utf-8') as f:
       contents = f.read()
     self.assertEqual(contents, byte_fractions.decode('utf-8'))
 
+  def testWriteStrReadBytes(self):
+    self.file = fake_filesystem.FakeFileOpen(self.filesystem)
+    file_path = 'foo'
+    str_contents = 'Äsgül'
+    with self.file(file_path, 'w') as f:
+      f.write(str_contents)
+    with self.file(file_path, 'rb') as f:
+      contents = f.read()
+    if sys.version_info < (3, 0):
+      self.assertEqual(str_contents, contents)
+    else:
+      self.assertEqual(str_contents, contents.decode(locale.getpreferredencoding(False)))
+
   def testByteContents(self):
     self.file = fake_filesystem.FakeFileOpen(self.filesystem)
     file_path = 'foo'
@@ -2590,7 +2610,7 @@ def testOpenValidArgs(self):
     if sys.version_info >= (3, 0):
       self.assertEqual(
           contents, self.open(file_path, mode='r', buffering=1,
-                              encoding='utf-8', errors='strict', newline='\n',
+                              errors='strict', newline='\n',
                               closefd=False, opener=False).readlines())
 
   @unittest.skipIf(sys.version_info < (3, 0), 'only tested on 3.0 or greater')
@@ -3045,6 +3065,128 @@ def testCanReadFromBlockDevice(self):
       self.assertEqual('', fh.read())
 
 
+@unittest.skipIf(sys.version_info < (3, 0),
+                 'Python3 specific string handling')
+class OpenFileWithEncodingTest(TestCase):
+  """Tests that are similar to some open file tests above but using an explicit text encoding.
+    Note: these tests can also be run under Python 2 after support for Python 3.2 will be skipped
+          (by using the u literal in the strings which is not supported in Pzthon 3.2)
+  """
+
+  def setUp(self):
+    self.filesystem = fake_filesystem.FakeFilesystem(path_separator='/')
+    self.open = fake_filesystem.FakeFileOpen(self.filesystem, use_io=True)
+    self.file_path = 'foo'
+    self.os = fake_filesystem.FakeOsModule(self.filesystem)
+
+  def testWriteStrReadBytes(self):
+    # note: this and the following test can be run under Python 2
+    # after support for Python 3.2 will be skipped (by using the u literal)
+    str_contents = 'علي بابا'
+    with self.open(self.file_path, 'w', encoding='arabic') as f:
+      f.write(str_contents)
+    with self.open(self.file_path, 'rb') as f:
+      contents = f.read()
+    self.assertEqual(str_contents, contents.decode('arabic'))
+
+  def testWriteAndReadStr(self):
+    str_contents = 'علي بابا'
+    with self.open(self.file_path, 'w', encoding='arabic') as f:
+      f.write(str_contents)
+    with self.open(self.file_path, 'r', encoding='arabic') as f:
+      contents = f.read()
+    self.assertEqual(str_contents, contents)
+
+  def testCreateFileWithAppend(self):
+    contents = [
+        'Allons enfants de la Patrie,'
+        'Le jour de gloire est arrivé!',
+        'Contre nous de la tyrannie,',
+        'L’étendard sanglant est levé.',
+        ]
+    fake_file = self.open(self.file_path, 'a', encoding='utf-8')
+    for line in contents:
+      fake_file.write(line + '\n')
+    fake_file.close()
+    result = [line.rstrip() for line in self.open(self.file_path, encoding='utf-8')]
+    self.assertEqual(contents, result)
+
+  def testAppendExistingFile(self):
+    contents = [
+        'Оригинальное содержание'
+        'Дополнительное содержание',
+        ]
+    self.filesystem.CreateFile(self.file_path, contents=contents[0], encoding='cyrillic')
+    fake_file = self.open(self.file_path, 'a', encoding='cyrillic')
+    for line in contents[1:]:
+      fake_file.write(line + '\n')
+    fake_file.close()
+    result = [line.rstrip() for line in self.open(self.file_path, encoding='cyrillic')]
+    self.assertEqual(contents, result)
+
+  def testOpenWithWplus(self):
+    self.filesystem.CreateFile(self.file_path, contents='старое содержание', encoding='cyrillic')
+    fake_file = self.open(self.file_path, 'r', encoding='cyrillic')
+    self.assertEqual('старое содержание', fake_file.read())
+    fake_file.close()
+
+    fake_file = self.open(self.file_path, 'w+', encoding='cyrillic')
+    fake_file.write('новое содержание')
+    fake_file.seek(0)
+    self.assertTrue('новое содержание', fake_file.read())
+    fake_file.close()
+
+  def testOpenWithAppendFlag(self):
+    contents = [
+        'Калинка,\n',
+        'калинка,\n',
+        'калинка моя,\n'
+        ]
+    additional_contents = [
+        'В саду ягода-малинка,\n',
+        'малинка моя.\n'
+        ]
+    self.filesystem.CreateFile(self.file_path, contents=''.join(contents), encoding='cyrillic')
+    fake_file = self.open(self.file_path, 'a', encoding='cyrillic')
+    self.assertRaises(IOError, fake_file.read)
+    self.assertEqual('', fake_file.read(0))
+    self.assertEqual('', fake_file.readline(0))
+    self.assertEqual(len(''.join(contents)), fake_file.tell())
+    fake_file.seek(0)
+    self.assertEqual(0, fake_file.tell())
+    fake_file.writelines(additional_contents)
+    fake_file.close()
+    result = self.open(self.file_path, encoding='cyrillic').readlines()
+    self.assertEqual(contents + additional_contents, result)
+
+  def testAppendWithAplus(self):
+    self.filesystem.CreateFile(self.file_path, contents='старое содержание', encoding='cyrillic')
+    fake_file = self.open(self.file_path, 'r', encoding='cyrillic')
+    fake_file.close()
+
+    fake_file = self.open(self.file_path, 'a+', encoding='cyrillic')
+    self.assertEqual(0, fake_file.tell())
+    fake_file.seek(6, 1)
+    fake_file.write('новое содержание')
+    self.assertEqual(33, fake_file.tell())
+    fake_file.seek(0)
+    self.assertEqual('старое содержаниеновое содержание', fake_file.read())
+    fake_file.close()
+
+  def testReadWithRplus(self):
+    self.filesystem.CreateFile(self.file_path, contents='старое содержание здесь', encoding='cyrillic')
+    fake_file = self.open(self.file_path, 'r', encoding='cyrillic')
+    fake_file.close()
+
+    fake_file = self.open(self.file_path, 'r+', encoding='cyrillic')
+    self.assertEqual('старое содержание здесь', fake_file.read())
+    fake_file.seek(0)
+    fake_file.write('новое  содержание')
+    fake_file.seek(0)
+    self.assertEqual('новое  содержание здесь', fake_file.read())
+    fake_file.close()
+
+
 class OpenWithFileDescriptorTest(FakeFileOpenTestBase):
 
   @unittest.skipIf(sys.version_info < (3, 0), 'only tested on 3.0 or greater')
@@ -3085,10 +3227,13 @@ def OpenFileAndSeek(self, mode):
     fake_file.seek(0, 2)
     return fake_file
 
-  def WriteAndReopenFile(self, fake_file, mode='rb'):
+  def WriteAndReopenFile(self, fake_file, mode='rb', encoding=None):
     fake_file.write(self.file_contents)
     fake_file.close()
-    return self.file(self.file_path, mode=mode)
+    args = {'mode': mode}
+    if encoding:
+      args['encoding'] = encoding
+    return self.file(self.file_path, **args)
 
   def testReadBinary(self):
     fake_file = self.OpenFakeFile('rb')
@@ -3102,11 +3247,11 @@ def testWriteBinary(self):
     # Attempt to reopen the file in text mode
     fake_file = self.OpenFakeFile('wb')
     if sys.version_info >= (3, 0):
-        self.assertRaises(UnicodeDecodeError, self.WriteAndReopenFile, fake_file, mode='r')
+      fake_file = self.WriteAndReopenFile(fake_file, mode='r', encoding='ascii')
+      self.assertRaises(UnicodeDecodeError, fake_file.read)
     else:
-        fake_file = self.WriteAndReopenFile(fake_file, mode='r')
-        self.assertEqual(self.file_contents, fake_file.read())
-
+      fake_file = self.WriteAndReopenFile(fake_file, mode='r')
+      self.assertEqual(self.file_contents, fake_file.read())
 
   def testWriteAndReadBinary(self):
     fake_file = self.OpenFileAndSeek('w+b')
@@ -3127,7 +3272,7 @@ def setUp(self):
     if sys.version_info >= (3, 0):
       self.read_contents = 'two\nlines'
     self.filesystem.CreateFile(self.file_path, contents=self.file_contents)
-    # It's resonable to assume the file exists at this point
+    # It's reasonable to assume the file exists at this point
 
   def OpenFakeFile(self, mode):
     return self.file(self.file_path, mode=mode)
@@ -3595,11 +3740,11 @@ def testFileSystemSizeAfterAsciiStringFileCreation(self):
     self.assertEqual((100, 11, 89), self.filesystem.GetDiskUsage())
 
   def testFileSystemSizeAfter2ByteUnicodeStringFileCreation(self):
-    self.filesystem.CreateFile('/foo/bar', contents='сложно')
+    self.filesystem.CreateFile('/foo/bar', contents='сложно', encoding='utf-8')
     self.assertEqual((100, 12, 88), self.filesystem.GetDiskUsage())
 
   def testFileSystemSizeAfter3ByteUnicodeStringFileCreation(self):
-    self.filesystem.CreateFile('/foo/bar', contents='複雑')
+    self.filesystem.CreateFile('/foo/bar', contents='複雑', encoding='utf-8')
     self.assertEqual((100, 6, 94), self.filesystem.GetDiskUsage())
 
   def testFileSystemSizeAfterFileDeletion(self):

fake_tempfile_test.py

@@ -124,10 +124,7 @@ def testNamedTemporaryFileNoDelete(self):
     obj.write(b'foo')
     obj.close()
     file_obj = self.filesystem.GetObject(obj.name)
-    if sys.version_info >= (3, 0):
-        contents = file_obj.contents.recover(False)
-    else:
-        contents = file_obj.contents
+    contents = file_obj.contents
     self.assertEqual('foo', contents)
     obj = self.tempfile.NamedTemporaryFile(mode='w', delete=False)
     obj.write('foo')