diff --git a/gdal/frmts/vrt/vrtdataset.cpp b/gdal/frmts/vrt/vrtdataset.cpp
index 32731cf7e5ee..065993aed4ce 100644
--- a/gdal/frmts/vrt/vrtdataset.cpp
+++ b/gdal/frmts/vrt/vrtdataset.cpp
@@ -1848,27 +1848,28 @@ void VRTDataset::BuildVirtualOverviews()
 
     int nOverviews = 0;
     GDALRasterBand* poFirstBand = nullptr;
-    for( int iBand = 0; iBand < nBands; iBand++ )
+
+    const auto CheckBandForOverview =
+        [&nOverviews, &poFirstBand, this](GDALRasterBand* poBand)
     {
-        if( !reinterpret_cast<VRTRasterBand *>(
-               papoBands[iBand] )->IsSourcedRasterBand())
-            return;
+        if( !cpl::down_cast<VRTRasterBand *>(poBand)->IsSourcedRasterBand())
+            return false;
 
         VRTSourcedRasterBand* poVRTBand
-            = reinterpret_cast<VRTSourcedRasterBand *>( papoBands[iBand] );
+            = cpl::down_cast<VRTSourcedRasterBand *>(poBand);
         if( poVRTBand->nSources != 1 )
-            return;
+            return false;
         if( !poVRTBand->papoSources[0]->IsSimpleSource() )
-            return;
+            return false;
 
         VRTSimpleSource* poSource
-            = reinterpret_cast<VRTSimpleSource *>( poVRTBand->papoSources[0] );
+            = cpl::down_cast<VRTSimpleSource *>( poVRTBand->papoSources[0] );
         if( !EQUAL(poSource->GetType(), "SimpleSource") &&
             !EQUAL(poSource->GetType(), "ComplexSource") )
-            return;
+            return false;
         GDALRasterBand* poSrcBand = poSource->GetBand();
         if( poSrcBand == nullptr )
-            return;
+            return false;
 
         // To prevent recursion
         m_apoOverviewsBak.push_back(nullptr);
@@ -1876,48 +1877,29 @@ void VRTDataset::BuildVirtualOverviews()
         m_apoOverviewsBak.resize(0);
 
         if( nOvrCount == 0 )
-            return;
-        if( iBand == 0 )
+            return false;
+        if( poFirstBand == nullptr )
         {
             if( poSrcBand->GetXSize() == 0 || poSrcBand->GetYSize() == 0 )
-                return;
+                return false;
             poFirstBand = poSrcBand;
             nOverviews = nOvrCount;
         }
         else if( nOvrCount < nOverviews )
             nOverviews = nOvrCount;
-    }
+        return true;
+    };
 
-    if( m_poMaskBand )
+    for( int iBand = 0; iBand < nBands; iBand++ )
     {
-        if( !m_poMaskBand->IsSourcedRasterBand())
+        if( !CheckBandForOverview(papoBands[iBand]) )
             return;
+    }
 
-        VRTSourcedRasterBand* poVRTBand
-            = cpl::down_cast<VRTSourcedRasterBand *>( m_poMaskBand );
-        if( poVRTBand->nSources != 1 )
-            return;
-        if( !poVRTBand->papoSources[0]->IsSimpleSource() )
-            return;
-
-        VRTSimpleSource* poSource
-            = cpl::down_cast<VRTSimpleSource *>( poVRTBand->papoSources[0] );
-        if( !EQUAL(poSource->GetType(), "SimpleSource") &&
-            !EQUAL(poSource->GetType(), "ComplexSource") )
-            return;
-        GDALRasterBand* poSrcBand = poSource->GetBand();
-        if( poSrcBand == nullptr )
-            return;
-
-        // To prevent recursion
-        m_apoOverviewsBak.push_back(nullptr);
-        const int nOvrCount = poSrcBand->GetOverviewCount();
-        m_apoOverviewsBak.resize(0);
-
-        if( nOvrCount == 0 )
+    if( m_poMaskBand )
+    {
+        if( !CheckBandForOverview(m_poMaskBand) )
             return;
-        if( nOvrCount < nOverviews )
-            nOverviews = nOvrCount;
     }
 
     for( int j = 0; j < nOverviews; j++)
@@ -1936,19 +1918,19 @@ void VRTDataset::BuildVirtualOverviews()
         VRTDataset* poOvrVDS = new VRTDataset(nOvrXSize, nOvrYSize);
         m_apoOverviews.push_back(poOvrVDS);
 
-        for( int i = 0; i < nBands; i++ )
+        const auto CreateOverviewBand =
+            [&poOvrVDS, nOvrXSize, nOvrYSize, dfXRatio, dfYRatio]
+            (GDALRasterBand* poBand)
         {
             VRTSourcedRasterBand* poVRTBand
-                = reinterpret_cast<VRTSourcedRasterBand *>(
-                    GetRasterBand(i+1) );
+                = cpl::down_cast<VRTSourcedRasterBand *>(poBand);
             VRTSourcedRasterBand* poOvrVRTBand = new VRTSourcedRasterBand(
                 poOvrVDS,
-                poOvrVDS->GetRasterCount() + 1,
+                poBand->GetBand(),
                 poVRTBand->GetRasterDataType(),
                 nOvrXSize, nOvrYSize);
-            poOvrVDS->SetBand( poOvrVDS->GetRasterCount() + 1, poOvrVRTBand );
 
-            VRTSimpleSource* poSrcSource = reinterpret_cast<VRTSimpleSource *>(
+            VRTSimpleSource* poSrcSource = cpl::down_cast<VRTSimpleSource *>(
                 poVRTBand->papoSources[0] );
             VRTSimpleSource* poNewSource = nullptr;
             if( EQUAL(poSrcSource->GetType(), "SimpleSource") )
@@ -1959,7 +1941,7 @@ void VRTDataset::BuildVirtualOverviews()
             else if( EQUAL(poSrcSource->GetType(), "ComplexSource") )
             {
               poNewSource = new VRTComplexSource(
-                  reinterpret_cast<VRTComplexSource *>( poSrcSource ),
+                  cpl::down_cast<VRTComplexSource *>( poSrcSource ),
                   dfXRatio, dfYRatio );
             }
             else
@@ -1972,43 +1954,19 @@ void VRTDataset::BuildVirtualOverviews()
                     poNewSource->GetBand()->GetDataset()->Reference();
                 poOvrVRTBand->AddSource(poNewSource);
             }
+
+            return poOvrVRTBand;
+        };
+
+        for( int i = 0; i < nBands; i++ )
+        {
+            poOvrVDS->SetBand( poOvrVDS->GetRasterCount() + 1,
+                               CreateOverviewBand(GetRasterBand(i+1)) );
         }
 
         if( m_poMaskBand )
         {
-            VRTSourcedRasterBand* poVRTBand
-                = cpl::down_cast<VRTSourcedRasterBand *>(m_poMaskBand );
-            VRTSourcedRasterBand* poOvrVRTBand = new VRTSourcedRasterBand(
-                poOvrVDS,
-                0,
-                poVRTBand->GetRasterDataType(),
-                nOvrXSize, nOvrYSize);
-            poOvrVDS->SetMaskBand( poOvrVRTBand );
-
-            VRTSimpleSource* poSrcSource = cpl::down_cast<VRTSimpleSource *>(
-                poVRTBand->papoSources[0] );
-            VRTSimpleSource* poNewSource = nullptr;
-            if( EQUAL(poSrcSource->GetType(), "SimpleSource") )
-            {
-                poNewSource =
-                    new VRTSimpleSource(poSrcSource, dfXRatio, dfYRatio);
-            }
-            else if( EQUAL(poSrcSource->GetType(), "ComplexSource") )
-            {
-              poNewSource = new VRTComplexSource(
-                  cpl::down_cast<VRTComplexSource *>( poSrcSource ),
-                  dfXRatio, dfYRatio );
-            }
-            else
-            {
-                CPLAssert(false);
-            }
-            if( poNewSource )
-            {
-                if( poNewSource->GetBand()->GetDataset() )
-                    poNewSource->GetBand()->GetDataset()->Reference();
-                poOvrVRTBand->AddSource(poNewSource);
-            }
+            poOvrVDS->SetMaskBand( CreateOverviewBand(m_poMaskBand) );
         }
     }
 }