From a8aafc12c909a519c914c9da5622fd6d4b80c23b Mon Sep 17 00:00:00 2001 From: Richard Simpson Date: Tue, 20 Dec 2016 05:27:28 -0600 Subject: [PATCH 1/3] (GH-1047) Add additional hash stream options to Providers --- .../infrastructure/adapters/HashAlgorithm.cs | 6 ++++++ .../infrastructure/adapters/IHashAlgorithm.cs | 3 +++ .../cryptography/CryptoHashProvider.cs | 16 +++++++++++++++- .../cryptography/IHashProvider.cs | 18 +++++++++++++++++- 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/chocolatey/infrastructure/adapters/HashAlgorithm.cs b/src/chocolatey/infrastructure/adapters/HashAlgorithm.cs index 1c303a1cec..b30b3206aa 100644 --- a/src/chocolatey/infrastructure/adapters/HashAlgorithm.cs +++ b/src/chocolatey/infrastructure/adapters/HashAlgorithm.cs @@ -1,5 +1,6 @@ namespace chocolatey.infrastructure.adapters { + using System.IO; using cryptography; public sealed class HashAlgorithm : IHashAlgorithm @@ -15,5 +16,10 @@ public byte[] ComputeHash(byte[] buffer) { return _algorithm.ComputeHash(buffer); } + + public byte[] ComputeHash(Stream stream) + { + return _algorithm.ComputeHash(stream); + } } } \ No newline at end of file diff --git a/src/chocolatey/infrastructure/adapters/IHashAlgorithm.cs b/src/chocolatey/infrastructure/adapters/IHashAlgorithm.cs index 5d87e8797e..1c6bf452f3 100644 --- a/src/chocolatey/infrastructure/adapters/IHashAlgorithm.cs +++ b/src/chocolatey/infrastructure/adapters/IHashAlgorithm.cs @@ -1,10 +1,13 @@ namespace chocolatey.infrastructure.adapters { + using System.IO; // ReSharper disable InconsistentNaming public interface IHashAlgorithm { byte[] ComputeHash(byte[] buffer); + + byte[] ComputeHash(Stream stream); } // ReSharper restore InconsistentNaming diff --git a/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs b/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs index c57ff63772..f3fa9bfeae 100644 --- a/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs +++ b/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs @@ -1,4 +1,4 @@ -// Copyright © 2011 - Present RealDimensions Software, LLC +// Copyright � 2011 - Present RealDimensions Software, LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -107,6 +107,20 @@ public string hash_file(string filePath) } } + public string hash_byte_array(byte[] buffer) + { + var hash = _hashAlgorithm.ComputeHash(buffer); + + return BitConverter.ToString(hash).Replace("-", string.Empty); + } + + public string hash_stream(Stream inputStream) + { + var hash = _hashAlgorithm.ComputeHash(inputStream); + + return BitConverter.ToString(hash).Replace("-", string.Empty); + } + private static bool file_is_locked(Exception exception) { var errorCode = 0; diff --git a/src/chocolatey/infrastructure/cryptography/IHashProvider.cs b/src/chocolatey/infrastructure/cryptography/IHashProvider.cs index a56b8afffc..6bb746e05d 100644 --- a/src/chocolatey/infrastructure/cryptography/IHashProvider.cs +++ b/src/chocolatey/infrastructure/cryptography/IHashProvider.cs @@ -1,4 +1,4 @@ -// Copyright © 2011 - Present RealDimensions Software, LLC +// Copyright � 2011 - Present RealDimensions Software, LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,6 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System.IO; + namespace chocolatey.infrastructure.cryptography { /// @@ -32,5 +34,19 @@ public interface IHashProvider /// The file path. /// A computed hash of the file, based on the contents. string hash_file(string filePath); + + /// + /// Returns a hash of the specified stream. + /// + /// The stream. + /// A computed hash of the stream, based on the contents. + string hash_stream(Stream inputStream); + + /// + /// Returns a hash of the specified byte array. + /// + /// The byte array. + /// A computed hash of the array, based on the contents. + string hash_byte_array(byte[] buffer); } } \ No newline at end of file From 89d0a708b5a8381141fa8f8478d41b15ad8282b5 Mon Sep 17 00:00:00 2001 From: Richard Simpson Date: Tue, 20 Dec 2016 05:28:08 -0600 Subject: [PATCH 2/3] (GH-1047) Add "replace" filesystem operation Allows for the more correct replacement and backup of files on the filesystem. --- .../filesystem/DotNetFileSystem.cs | 19 +++++++++++++++++++ .../infrastructure/filesystem/IFileSystem.cs | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/src/chocolatey/infrastructure/filesystem/DotNetFileSystem.cs b/src/chocolatey/infrastructure/filesystem/DotNetFileSystem.cs index 7c3e30fb36..986b27750c 100644 --- a/src/chocolatey/infrastructure/filesystem/DotNetFileSystem.cs +++ b/src/chocolatey/infrastructure/filesystem/DotNetFileSystem.cs @@ -390,6 +390,25 @@ public bool copy_file_unsafe(string sourceFilePath, string destinationFilePath, return success != 0; } + + public void replace_file(string sourceFilePath, string destinationFilePath, string backupFilePath) + { + this.Log().Debug(ChocolateyLoggers.Verbose, () => "Attempting to replace \"{0}\"{1} with \"{2}\". Backup placed at \"{3}\".".format_with(destinationFilePath, Environment.NewLine, sourceFilePath, backupFilePath)); + + allow_retries( + () => + { + try + { + File.Replace(sourceFilePath, destinationFilePath, backupFilePath); + } + catch (IOException) + { + Alphaleonis.Win32.Filesystem.File.Replace(sourceFilePath, destinationFilePath, backupFilePath); + } + }); + } + // ReSharper disable InconsistentNaming // http://msdn.microsoft.com/en-us/library/windows/desktop/aa363851.aspx diff --git a/src/chocolatey/infrastructure/filesystem/IFileSystem.cs b/src/chocolatey/infrastructure/filesystem/IFileSystem.cs index 4d41005bcc..14e70389c7 100644 --- a/src/chocolatey/infrastructure/filesystem/IFileSystem.cs +++ b/src/chocolatey/infrastructure/filesystem/IFileSystem.cs @@ -204,6 +204,14 @@ public interface IFileSystem /// true if copy was successful, otherwise false bool copy_file_unsafe(string sourceFilePath, string destinationFilePath, bool overwriteExisting); + /// + /// Replace an existing file. + /// + /// Where is the file now? + /// Where would you like it to go? + /// Where should the existing file be placed? Null if nowhere. + void replace_file(string sourceFilePath, string destinationFilePath, string backupFilePath); + /// /// Deletes the specified file. /// From 1b02bdfb1c32e2d003be549774c48ca9abcc75fa Mon Sep 17 00:00:00 2001 From: Richard Simpson Date: Tue, 20 Dec 2016 05:28:31 -0600 Subject: [PATCH 3/3] (GH-1047) Robust/efficient xml serialization Previously, the xml file serialization could cause corruption when it did not properly replace the file or when multiple choco processes were running at the same time. Additionally, there was no backup of the original config, so the configuration would need to be reset back to the default configuration and then readjusted. Switch to evaluating differences in memory instead of an update file. If the upates are different, then write out the update file and use replace file to have it create a backup of the old file and replace the config with the updated version. When deserializing, if the file is corrupt, look to use the backup and put it back in place if it works. Handle cases where there is no original config file and the backup file doesn't exist. Another benefit of comparing in memory is that no errors are logged for non-admins when the config update file can't be written. --- .../cryptography/CryptoHashProvider.cs | 2 +- .../cryptography/IHashProvider.cs | 2 +- .../infrastructure/services/XmlService.cs | 93 ++++++++++++++----- 3 files changed, 74 insertions(+), 23 deletions(-) diff --git a/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs b/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs index f3fa9bfeae..fd8c65288d 100644 --- a/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs +++ b/src/chocolatey/infrastructure/cryptography/CryptoHashProvider.cs @@ -1,4 +1,4 @@ -// Copyright � 2011 - Present RealDimensions Software, LLC +// Copyright © 2011 - Present RealDimensions Software, LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/chocolatey/infrastructure/cryptography/IHashProvider.cs b/src/chocolatey/infrastructure/cryptography/IHashProvider.cs index 6bb746e05d..e820ebd216 100644 --- a/src/chocolatey/infrastructure/cryptography/IHashProvider.cs +++ b/src/chocolatey/infrastructure/cryptography/IHashProvider.cs @@ -1,4 +1,4 @@ -// Copyright � 2011 - Present RealDimensions Software, LLC +// Copyright © 2011 - Present RealDimensions Software, LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/chocolatey/infrastructure/services/XmlService.cs b/src/chocolatey/infrastructure/services/XmlService.cs index 00619f627b..ad8b346784 100644 --- a/src/chocolatey/infrastructure/services/XmlService.cs +++ b/src/chocolatey/infrastructure/services/XmlService.cs @@ -15,6 +15,7 @@ namespace chocolatey.infrastructure.services { + using System; using System.IO; using System.Text; using System.Xml; @@ -43,14 +44,48 @@ public XmlType deserialize(string xmlFilePath) () => { var xmlSerializer = new XmlSerializer(typeof(XmlType)); - var xmlReader = XmlReader.Create(new StringReader(_fileSystem.read_file(xmlFilePath))); - if (!xmlSerializer.CanDeserialize(xmlReader)) + using (var fileStream = _fileSystem.open_file_readonly(xmlFilePath)) + using (var fileReader = new StreamReader(fileStream)) + using (var xmlReader = XmlReader.Create(fileReader)) { - this.Log().Warn("Cannot deserialize response of type {0}", typeof(XmlType)); - return default(XmlType); - } + if (!xmlSerializer.CanDeserialize(xmlReader)) + { + this.Log().Warn("Cannot deserialize response of type {0}", typeof(XmlType)); + return default(XmlType); + } + + try + { + return (XmlType)xmlSerializer.Deserialize(xmlReader); + } + catch(InvalidOperationException ex) + { + // Check if its just a malformed document. + if (ex.Message.Contains("There is an error in XML document")) + { + // If so, check for a backup file and try an parse that. + if (_fileSystem.file_exists(xmlFilePath + ".backup")) + { + using (var backupStream = _fileSystem.open_file_readonly(xmlFilePath + ".backup")) + using (var backupReader = new StreamReader(backupStream)) + using (var backupXmlReader = XmlReader.Create(backupReader)) + { + var validConfig = (XmlType)xmlSerializer.Deserialize(backupXmlReader); - return (XmlType)xmlSerializer.Deserialize(xmlReader); + // If there's no errors and it's valid, go ahead and replace the bad file with the backup. + if(validConfig != null) + { + _fileSystem.copy_file(xmlFilePath + ".backup", xmlFilePath, overwriteExisting: true); + } + + return validConfig; + } + } + } + + throw; + } + } }, "Error deserializing response of type {0}".format_with(typeof(XmlType)), throwError: true); @@ -65,30 +100,46 @@ public void serialize(XmlType xmlType, string xmlFilePath, bool isSilen { _fileSystem.create_directory_if_not_exists(_fileSystem.get_directory_name(xmlFilePath)); - var xmlUpdateFilePath = xmlFilePath + ".update"; - FaultTolerance.try_catch_with_logging_exception( () => { var xmlSerializer = new XmlSerializer(typeof(XmlType)); - //var textWriter = new StreamWriter(xmlUpdateFilePath, append: false, encoding: new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)) - var textWriter = new StreamWriter(xmlUpdateFilePath, append: false, encoding: new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)) + + // Write the updated file to memory + using(var memoryStream = new MemoryStream()) + using(var streamWriter = new StreamWriter(memoryStream, encoding: new UTF8Encoding(encoderShouldEmitUTF8Identifier: true))) { - AutoFlush = true - }; + xmlSerializer.Serialize(streamWriter, xmlType); + streamWriter.Flush(); - xmlSerializer.Serialize(textWriter, xmlType); - textWriter.Flush(); + memoryStream.Position = 0; + + // Grab the hash of both files and compare them. + var originalFileHash = _hashProvider.hash_file(xmlFilePath); + if (!originalFileHash.is_equal_to(_hashProvider.hash_stream(memoryStream))) + { + // If there wasn't a file there in the first place, just write the new one out directly. + if(string.IsNullOrEmpty(originalFileHash)) + { + using(var updateFileStream = _fileSystem.create_file(xmlFilePath)) + { + memoryStream.Position = 0; + memoryStream.CopyTo(updateFileStream); - textWriter.Close(); - textWriter.Dispose(); + return; + } + } - if (!_hashProvider.hash_file(xmlFilePath).is_equal_to(_hashProvider.hash_file(xmlUpdateFilePath))) - { - _fileSystem.copy_file(xmlUpdateFilePath, xmlFilePath, overwriteExisting: true); + // Otherwise, create an update file, and resiliently move it into place. + var tempUpdateFile = xmlFilePath + ".update"; + using(var updateFileStream = _fileSystem.create_file(tempUpdateFile)) + { + memoryStream.Position = 0; + memoryStream.CopyTo(updateFileStream); + } + _fileSystem.replace_file(tempUpdateFile, xmlFilePath, xmlFilePath + ".backup"); + } } - - _fileSystem.delete_file(xmlUpdateFilePath); }, "Error serializing type {0}".format_with(typeof(XmlType)), throwError: true,