Recordings for storage (Azure#3194)

* Recordings for storage (#4)

CONTRIBUTING.md

@@ -176,14 +176,44 @@ The following CMake options are available for adding/removing project features.
 
 #### Testing the project
 
-If you want to run tests also, generate build files using below command and then build.
+##### Test Mode
+
+Before running unit tests, you have to decide what is the test mode you want to use. To set the test mode,
+use the environment variable `AZURE_TEST_MODE`. See the supported values next:
+
+- LIVE
+
+When setting `AZURE_TEST_MODE=LIVE`, test cases will try to connect to `AZURE` cloud services using the [test environment configuration](#test-environment-configuration). Make sure to set up the environment variables required to run test cases.
+
+This is the default test mode if `AZURE_TEST_MODE` is not even set.
+
+- PLAYBACK
+
+When setting `AZURE_TEST_MODE=PLAYBACK`, test cases will consume pre-recorded data instead of sending requests to an AZURE cloud service. The [test environment configuration](#test-environment-configuration) is still required, but the configuration values won't be relevant. Use this test mode to run tests cases without a network connection.
+
+- RECORD
+
+When setting `AZURE_TEST_MODE=RECORD`, test cases will run the as when running `LIVE`. All the AZURE service network responses are recorded in a json file within the /recordings folder from the /test/ut directory. Use this test mode to generate pre-recorded data to be used on `PLAYBACK` mode.
+
+##### Test Environment Configuration
+
+Some environment variables are expected to be defined for some test binaries. For example, for `azure-storage-blobs-test`, there should be a `STANDARD_STORAGE_CONNECTION_STRING` variable to let tests know how to connect to the Azure Storage account.
+
+Even for running on `PLAYBACK` mode, the env configuration is mandatory. This is because a test case does not know about the test modes. A test case will always look for the environment configuration to connect/authenticate to Azure.
+
+Take a look to [this file](https://github.com/Azure/azure-sdk-for-cpp/blob/main/sdk/core/ci.yml#L52) which defines the required configuarion for each SDK package. Those settings are used to run all unit test on `PLAYBACK` mode on CI, you can use the same settings from that file to run on `PLAYBACK` locally.
+
+
+##### Running tests
+
+If you want to run tests, generate build files using below command and then build.
 
 ```sh
 cmake -DBUILD_TESTING=ON ..
 cmake --build .
 ```
 
-Tests are executed via the `ctest` command included with CMake. From the build directory, run:
+Tests are executed via the `ctest` command included with CMake. After setting a [test mode](#test-mode), from the build directory, run:
 
 ```sh
 # use -V for verbose
@@ -192,6 +222,7 @@ ctest -V
 ctest -N
 # Use -R to use a regular exp for what to run
 ctest -R Http # runs only HTTP tests
+ctest -R storage # runs all the azure storage unit tests
 ```
 
 #### Generating Code Coverage reports

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -207,7 +207,7 @@ jobs:
       Location: ${{ parameters.Location }}
       SubscriptionConfiguration: ${{ parameters.SubscriptionConfiguration }}
 
-  - script: ctest -C Debug --tests-regex ${{ parameters.CtestRegex }} --no-compress-output -T Test
+  - script: ctest -V -C Debug --tests-regex ${{ parameters.CtestRegex }} --no-compress-output -T Test
     workingDirectory: build
     displayName: ctest
     # Runs only if test-resources are happly deployed.

sdk/core/azure-core-test/inc/azure/core/test/interceptor_manager.hpp

@@ -31,6 +31,10 @@
 #include "azure/core/test/record_network_call_policy.hpp"
 #include "azure/core/test/test_context_manager.hpp"
 
+// Used by recordPolicy and playback transport adapter.
+#if !defined(RECORDING_BODY_STREAM_SENTINEL)
+#define RECORDING_BODY_STREAM_SENTINEL "__bodyStream__"
+#endif
 namespace Azure { namespace Core { namespace Test {
 
   /**

sdk/core/azure-core-test/inc/azure/core/test/network_models.hpp

@@ -8,9 +8,13 @@
 
 #pragma once
 
+#include <azure/core/http/policies/policy.hpp>
+#include <azure/core/io/body_stream.hpp>
+
 #include <list>
 #include <map>
 #include <string>
+#include <vector>
 
 namespace Azure { namespace Core { namespace Test {
 
@@ -48,4 +52,86 @@ namespace Azure { namespace Core { namespace Test {
     std::list<std::string> Variables;
   };
 
+  /**
+   * @brief A body stream which holds the memory inside.
+   *
+   * @remark The playback http uses this body stream to be returned as part of the raw response so
+   * the transport policy can read from it.
+   *
+   */
+  class WithMemoryBodyStream : public Azure::Core::IO::BodyStream {
+  private:
+    std::vector<uint8_t> m_memory;
+    Azure::Core::IO::MemoryBodyStream m_streamer;
+
+    size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override
+    {
+      return m_streamer.Read(buffer, count, context);
+    }
+
+  public:
+    // Forbid constructor for rval so we don't end up storing dangling ptr
+    WithMemoryBodyStream(std::vector<uint8_t> const&&) = delete;
+
+    /**
+     * @brief Construct using vector of bytes.
+     *
+     * @param buffer Vector of bytes with the contents to provide the data from to the readers.
+     */
+    WithMemoryBodyStream(std::vector<uint8_t> const& buffer)
+        : m_memory(buffer), m_streamer(m_memory)
+    {
+    }
+
+    int64_t Length() const override { return m_streamer.Length(); }
+
+    void Rewind() override { m_streamer.Rewind(); }
+  };
+
+  /**
+   * @brief Wraps a stream and keep reading bytes from it by rewinding it until some length.
+   *
+   * @note Enables to create a stream with huge size by re-using a small buffer (1Mb).
+   *
+   */
+  class CircularBodyStream : public Azure::Core::IO::BodyStream {
+  private:
+    std::unique_ptr<std::vector<uint8_t>> m_buffer;
+    size_t m_length;
+    size_t m_totalRead = 0;
+    Azure::Core::IO::MemoryBodyStream m_memoryStream;
+
+    size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override
+    {
+      auto available = m_length - m_totalRead;
+      if (available == 0)
+      {
+        return 0;
+      }
+
+      auto toRead = std::min(count, available);
+      auto read = m_memoryStream.Read(buffer, toRead, context);
+
+      // Circurlar implementation. Rewind the stream every time we reach the end
+      if (read == 0) // No more bytes to read from.
+      {
+        m_memoryStream.Rewind();
+        read = m_memoryStream.Read(buffer, toRead, context);
+      }
+
+      m_totalRead += read;
+      return read;
+    }
+
+  public:
+    CircularBodyStream(size_t size, uint8_t fillWith)
+        : m_buffer(std::make_unique<std::vector<uint8_t>>(1024 * 1024, fillWith)), m_length(size),
+          m_memoryStream(*m_buffer)
+    {
+    }
+
+    int64_t Length() const override { return m_length; }
+    void Rewind() override { m_totalRead = 0; }
+  };
+
 }}} // namespace Azure::Core::Test

sdk/core/azure-core-test/inc/azure/core/test/playback_http_client.hpp

@@ -22,42 +22,6 @@ namespace Azure { namespace Core { namespace Test {
   // Partial class. Required to reference the Interceptor that is defined in the implementation.
   class InterceptorManager;
 
-  /**
-   * @brief A body stream which holds the memory inside.
-   *
-   * @remark The playback http uses this body stream to be returned as part of the raw response so
-   * the transport policy can read from it.
-   *
-   */
-  class WithMemoryBodyStream : public Azure::Core::IO::BodyStream {
-  private:
-    std::vector<uint8_t> m_memory;
-    Azure::Core::IO::MemoryBodyStream m_streamer;
-
-    size_t OnRead(uint8_t* buffer, size_t count, Azure::Core::Context const& context) override
-    {
-      return m_streamer.Read(buffer, count, context);
-    }
-
-  public:
-    // Forbid constructor for rval so we don't end up storing dangling ptr
-    WithMemoryBodyStream(std::vector<uint8_t> const&&) = delete;
-
-    /**
-     * @brief Construct using vector of bytes.
-     *
-     * @param buffer Vector of bytes with the contents to provide the data from to the readers.
-     */
-    WithMemoryBodyStream(std::vector<uint8_t> const& buffer)
-        : m_memory(buffer), m_streamer(m_memory)
-    {
-    }
-
-    int64_t Length() const override { return m_streamer.Length(); }
-
-    void Rewind() override { m_streamer.Rewind(); }
-  };
-
   /**
    * @brief Creates an HTTP Transport adapter that answer to requests using recorded data.
    *

sdk/core/azure-core-test/inc/azure/core/test/record_network_call_policy.hpp

@@ -30,6 +30,14 @@ namespace Azure { namespace Core { namespace Test {
   class RecordNetworkCallPolicy : public Azure::Core::Http::Policies::HttpPolicy {
   private:
     Azure::Core::Test::InterceptorManager* m_interceptorManager;
+    // Used to save the first byte from request payloads.
+    // Then, get a subsequent request ask for a bodyStream response, the symbol is used to generate
+    // a bodyStream from it.
+    // This feature is helpful to let a storage tests ( for example ) to upload a big payload (more
+    // than 10Kb) and download it later.
+    // The request for upload will contain the payload to upload.
+    // Then the request for download will use the symbol to generate a bodyStream.
+    std::unique_ptr<uint8_t> m_symbol;
 
   public:
     /**
@@ -46,7 +54,7 @@ namespace Azure { namespace Core { namespace Test {
      * data.
      */
     RecordNetworkCallPolicy(Azure::Core::Test::InterceptorManager* interceptorManager)
-        : m_interceptorManager(interceptorManager)
+        : m_interceptorManager(interceptorManager), m_symbol(std::make_unique<uint8_t>('x'))
     {
     }