diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc
index b133762463fea..4d123028ea86e 100644
--- a/cpp/src/arrow/filesystem/azurefs_test.cc
+++ b/cpp/src/arrow/filesystem/azurefs_test.cc
@@ -1394,9 +1394,9 @@ TEST_F(TestAzuriteFileSystem, DeleteFileSuccess) {
 
 TEST_F(TestAzuriteFileSystem, DeleteFileFailureNonexistent) {
   const auto container_name = PreexistingData::RandomContainerName(rng_);
-  auto container_client = CreateContainer(container_name);
-  const auto blob_path = ConcatAbstractPath(container_name, "someblob");
-  ASSERT_RAISES(IOError, fs()->DeleteFile(blob_path));
+  ASSERT_OK(fs()->CreateDir(container_name));
+  const auto nonexistent_file_name = ConcatAbstractPath(container_name, "nonexistent");
+  ASSERT_RAISES(IOError, fs()->DeleteFile(nonexistent_file_name));
 }
 
 TEST_F(TestAzuriteFileSystem, DeleteFileFailureContainer) {