diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml
index 59aee791b36e..12b0026ccd26 100644
--- a/eng/pipelines/docindex.yml
+++ b/eng/pipelines/docindex.yml
@@ -14,20 +14,16 @@ jobs:
       DocRepoName: azure-docs-sdk-python
       DocValidationImageId: azuresdkimages.azurecr.io/pyrefautocr:latest
     steps:
-      # Docs CI uses Python 3.9.13
+      # py2docfx requires Python >= 3.11
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.9.13'
+        displayName: 'Use Python 3.11'
         inputs:
-          versionSpec: '3.9.13'
+          versionSpec: '3.11'
 
       # Docs CI upgrades pip, wheel, and setuptools
       - pwsh: python -m pip install --upgrade pip wheel setuptools
         displayName: Update python tools for package verification
 
-      # Pull and build the docker image.
-      - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
-        parameters:
-          ImageId: "$(DocValidationImageId)"
       # Sync docs repo onboarding files/folders
       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
         parameters:
@@ -41,6 +37,8 @@ jobs:
             - Name: $(DocRepoOwner)/$(DocRepoName)
               WorkingDirectory: $(DocRepoLocation)
 
+      - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
       - task: Powershell@2
         inputs:
           pwsh: true
@@ -53,7 +51,7 @@ jobs:
         inputs:
           pwsh: true
           filePath: eng/common/scripts/Update-DocsMsPackages.ps1
-          arguments: -DocRepoLocation $(DocRepoLocation) -ImageId '$(DocValidationImageId)'
+          arguments: -DocRepoLocation $(DocRepoLocation)
         displayName: Update Docs Onboarding for main branch
         condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true')))
 
@@ -92,7 +90,7 @@ jobs:
         parameters:
           BaseRepoBranch: $(DefaultBranch)
           BaseRepoOwner: $(DocRepoOwner)
-          CommitMsg: "Update docs CI configuration"
+          CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
           TargetRepoName: $(DocRepoName)
           TargetRepoOwner: $(DocRepoOwner)
           WorkingDirectory: $(DocRepoLocation)
diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml
index fece5adabba3..8bdcfd24b8fa 100644
--- a/eng/pipelines/templates/stages/archetype-python-release.yml
+++ b/eng/pipelines/templates/stages/archetype-python-release.yml
@@ -9,7 +9,6 @@ parameters:
   TargetDocRepoOwner: ''
   TargetDocRepoName: ''
   PackageSourceOverride: "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
-  DocValidationImageId: "azuresdkimages.azurecr.io/pyrefautocr:latest"
 
 stages:
   - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}:
@@ -232,6 +231,15 @@ stages:
                                 - sdk/**/*.md
                                 - .github/CODEOWNERS
                           - download: current
+
+                          # py2docfx requires Python >= 3.11
+                          - task: UsePythonVersion@0
+                            displayName: 'Use Python 3.11'
+                            inputs:
+                              versionSpec: '3.11'
+
+                          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
                           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
                             parameters:
                               PackageInfoLocations:
@@ -244,7 +252,6 @@ stages:
                                 - docs-ref-services/
                                 - metadata/
                               PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-                              DocValidationImageId: ${{parameters.DocValidationImageId}}
 
           - deployment: UpdatePackageVersion
             displayName: "Update Package Version"
@@ -351,6 +358,14 @@ stages:
               Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/
             displayName: Show visible artifacts
 
+          # py2docfx requires Python >= 3.11
+          - task: UsePythonVersion@0
+            displayName: 'Use Python 3.11'
+            inputs:
+              versionSpec: '3.11'
+
+          - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml
+
           - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
             parameters:
               PackageInfoLocations:
@@ -366,6 +381,5 @@ stages:
                 - docs-ref-services/
                 - metadata/
               PackageSourceOverride: ${{parameters.PackageSourceOverride}}
-              DocValidationImageId: ${{parameters.DocValidationImageId}}
 
           - template: /eng/common/pipelines/templates/steps/docsms-ensure-validation.yml
diff --git a/eng/pipelines/templates/steps/install-rex-validation-tool.yml b/eng/pipelines/templates/steps/install-rex-validation-tool.yml
new file mode 100644
index 000000000000..f447b4dfca48
--- /dev/null
+++ b/eng/pipelines/templates/steps/install-rex-validation-tool.yml
@@ -0,0 +1,8 @@
+steps:
+  - pwsh: |
+      Write-Host "python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt"
+      python -m pip install -r eng/scripts/docs/py2docfx_requirements.txt
+      # After install py2docfx,
+      Write-Host "Testing the install. Running python -m py2docfx -h to display the help"
+      python -m py2docfx -h
+    displayName: Install py2docfx for package validation
diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1
index 2042223a73c3..b65b4b43ba53 100644
--- a/eng/scripts/Language-Settings.ps1
+++ b/eng/scripts/Language-Settings.ps1
@@ -440,36 +440,6 @@ function Import-Dev-Cert-python
     "The variables SSL_CERT_DIR, SSL_CERT_FILE, and REQUESTS_CA_BUNDLE are now dynamically set in proxy_startup.py"
 }
 
-# Defined in common.ps1 as:
-# $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
-function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId)
-{
-  # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch
-  if (!$PackageInfos) {
-    $PackageInfos =  @($PackageInfo)
-  }
-
-  $allSucceeded = $true
-  foreach ($item in $PackageInfos) {
-    # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage
-    if ($item.Version -eq 'IGNORE') {
-      continue
-    }
-
-    $result = ValidatePackage `
-      -packageName $item.Name `
-      -packageVersion "==$($item.Version)" `
-      -PackageSourceOverride $PackageSourceOverride `
-      -DocValidationImageId $DocValidationImageId
-
-    if (!$result) {
-      $allSucceeded = $false
-    }
-  }
-
-  return $allSucceeded
-}
-
 function Get-python-EmitterName() {
   return "@azure-tools/typespec-python"
 }
diff --git a/eng/scripts/docs/Docs-Onboarding.ps1 b/eng/scripts/docs/Docs-Onboarding.ps1
index 23f14cf20edb..70f8a7a68c0a 100644
--- a/eng/scripts/docs/Docs-Onboarding.ps1
+++ b/eng/scripts/docs/Docs-Onboarding.ps1
@@ -1,7 +1,7 @@
 . "$PSScriptRoot/Docs-ToC.ps1"
 
 # $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
-function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) { 
+function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) {
     $onboardingFile = GetOnboardingFileForMoniker $docRepoLocation $moniker
 
     $onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable
@@ -30,7 +30,7 @@ function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation,
 
         if ($package.ContainsKey('DocsCiConfigProperties')) {
             $overrides = $package['DocsCiConfigProperties']
-            
+
             # Merge properties from package_info object (duplicate values will)
             # be overwritten
             if ($overrides.ContainsKey('package_info')) {
@@ -40,7 +40,7 @@ function Set-python-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation,
             }
 
             # Directly override other keys like exlcude_path
-            foreach ($key in $overrides.Keys) { 
+            foreach ($key in $overrides.Keys) {
                 if ($key -in @('package_info')) {
                     # Skip over keys that have already been processed
                     continue
@@ -67,153 +67,144 @@ function Get-python-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) {
         -moniker $moniker
 }
 
-
-
-function ValidatePackage {
-    Param(
-        [Parameter(Mandatory = $true)]
-        [string]$packageName,
-        [Parameter(Mandatory = $true)]
-        [string]$packageVersion,
-        [Parameter(Mandatory = $false)]
-        [string]$PackageSourceOverride,
-        [Parameter(Mandatory = $false)]
-        [string]$DocValidationImageId
-    )
-    $installValidationFolder = Join-Path ([System.IO.Path]::GetTempPath()) "validation"
-    if (!(Test-Path $installValidationFolder)) {
-        New-Item -ItemType Directory -Force -Path $installValidationFolder | Out-Null
-    }
-    # Add more validation by replicating as much of the docs CI process as
-    # possible
-    # https://github.com/Azure/azure-sdk-for-python/issues/20109
-    $result = $true
-    if (!$DocValidationImageId) {
-        Write-Host "Validating using pip command directly on $packageName."
-        $result = FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $installValidationFolder -PackageSourceOverride $PackageSourceOverride
-    }
-    else {
-        Write-Host "Validating using $DocValidationImageId on $packageName."
-        $result = DockerValidation -packageName "$packageName" -packageVersion "$packageVersion" `
-            -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId -workingDirectory $installValidationFolder
-    }
-
-    return $result
-}
-function DockerValidation {
-    Param(
-        [Parameter(Mandatory = $true)]
-        [string]$packageName,
-        [Parameter(Mandatory = $true)]
-        [string]$packageVersion,
-        [Parameter(Mandatory = $false)]
-        [string]$PackageSourceOverride,
-        [Parameter(Mandatory = $false)]
-        [string]$DocValidationImageId,
-        [Parameter(Mandatory = $false)]
-        [string]$workingDirectory
-    )
-    if ($PackageSourceOverride) {
-        Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId"
-        $commandLine = docker run -v "${workingDirectory}:/workdir/out" -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion `
-            -e EXTRA_INDEX_URL=$PackageSourceOverride -t $DocValidationImageId 2>&1
-    }
-    else {
-        Write-Host "docker run -v ${workingDirectory}:/workdir/out -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId"
-        $commandLine = docker run -v "${workingDirectory}:/workdir/out" `
-            -e TARGET_PACKAGE=$packageName -e TARGET_VERSION=$packageVersion -t $DocValidationImageId 2>&1
-    }
-    # The docker exit codes: https://docs.docker.com/engine/reference/run/#exit-status
-    # If the docker failed because of docker itself instead of the application,
-    # we should skip the validation and keep the packages.
-
-    if ($LASTEXITCODE -eq 125 -Or $LASTEXITCODE -eq 126 -Or $LASTEXITCODE -eq 127) {
-        $commandLine | ForEach-Object { Write-Debug $_ }
-        LogWarning "The `docker` command does not work with exit code $LASTEXITCODE. Fall back to npm install $packageName directly."
-        FallbackValidation -packageName "$packageName" -packageVersion "$packageVersion" -workingDirectory $workingDirectory -PackageSourceOverride $PackageSourceOverride
-    }
-    elseif ($LASTEXITCODE -ne 0) {
-        $commandLine | ForEach-Object { Write-Debug $_ }
-        LogWarning "Package $packageName ref docs validation failed."
-        return $false
-    }
-    return $true
-}
-
-function FallbackValidation {
-    Param(
-        [Parameter(Mandatory = $true)]
-        [string]$packageName,
-        [Parameter(Mandatory = $true)]
-        [string]$packageVersion,
-        [Parameter(Mandatory = $true)]
-        [string]$workingDirectory,
-        [Parameter(Mandatory = $false)]
-        [string]$PackageSourceOverride
-    )
-    $installTargetFolder = Join-Path $workingDirectory $packageName
-    New-Item -ItemType Directory -Force -Path $installTargetFolder | Out-Null
-    $packageExpression = "$packageName$packageVersion"
-    try {
-        $pipInstallOutput = ""
-        if ($PackageSourceOverride) {
-            Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder --extra-index-url=$PackageSourceOverride"
-            $pipInstallOutput = python -m pip `
-                install `
-                $packageExpression `
-                --no-cache-dir `
-                --target $installTargetFolder `
-                --extra-index-url=$PackageSourceOverride 2>&1
-        }
-        else {
-            Write-Host "python -m pip install $packageExpression --no-cache-dir --target $installTargetFolder"
-            $pipInstallOutput = python -m pip `
-                install `
-                $packageExpression `
-                --no-cache-dir `
-                --target $installTargetFolder 2>&1
-        }
-        if ($LASTEXITCODE -ne 0) {
-            LogWarning "python -m pip install failed for $packageExpression"
-            Write-Host $pipInstallOutput
-            return $false
-        }
-    }
-    catch {
-        LogWarning "python -m pip install failed for $packageExpression with exception"
-        LogWarning $_.Exception
-        LogWarning $_.Exception.StackTrace
-        return $false
-    }
-
-    return $true
+# The package info format required by py2docfx to verify a single library.
+# The format can be found at https://github.com/MicrosoftDocs/azure-docs-sdk-python/blob/main/ci-configs/packages-latest.json
+# which is a file that contains all of package_info objects. For verifying a single library
+# the same format is used but there's only a single package_info object in the packages list.
+# This is an example:
+# {
+#   "packages": [
+#       {
+#           "package_info": {
+#             "name": "azure-core",
+#             "install_type": "pypi",
+#             "prefer_source_distribution": "true",
+#             "version": "==1.29.6a20231207001",
+#             "extra_index_url": "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/"
+#           },
+#           "exclude_path": [
+#             "test*",
+#             "example*",
+#             "sample*",
+#             "doc*"
+#           ]
+#         }
+#       ]
+#   }
+# The above example contains the common settings for Python track 2 libraries.
+# There are a few things of note:
+# 1. install_type can be something other than pypi. source_code or dist_file are two examples of this
+#    but those are for track 1 or libraries released by other teams and not through our engineering system.
+# 2. extra_index_url only needs to exist on the object if PackageSourceOverride is set
+# 3. The reason this needs to be done using a json file instead of just a command line is because py2docfx
+#    doesn't handle the autodoc_default_options on the command line.
+function Get-SinglePackageJsonForDocsValidation($PackageInfo, $PackageSourceOverride)
+{
+
+  $packageArr = @()
+  $packageSpec = [ordered]@{
+      package_info = [ordered]@{
+          name = $PackageInfo.Name
+          install_type = 'pypi'
+          prefer_source_distribution = 'true'
+          version = "==$($PackageInfo.Version)"
+      }
+      exclude_path = @("test*","example*","sample*","doc*")
+  }
+  if ($PackageSourceOverride) {
+      $packageSpec['package_info']['extra_index_url'] = $PackageSourceOverride
+  }
+  # Data-plane packages (not mgmt packages, and not manually added '00`packages)
+  # should document inherited members
+  if ($PackageInfo.Name -notlike 'azure-mgmt-*' -and $PackageInfo.Name -notlike '*-00-*') {
+      $packageSpec['extension_config'] = @{ 'autodoc_default_options' = @{ 'inherited-members' = 1 } }
+  }
+  $packageArr += $packageSpec
+
+  # "packages" must be an array of packages even if there's only a single package in it.
+  # There are other top level elements, required_packages, target_repo etc. that aren't
+  # required for validation of a single package.
+  $docsConfigPackage = [ordered]@{
+      packages = $packageArr
+  }
+
+  # Return the JSon string
+  return $docsConfigPackage | ConvertTo-Json -Depth 10
 }
 
 # Defined in common.ps1 as:
 # $ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
-function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId) {
-    # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch
-    if (!$PackageInfos) {
-        $PackageInfos = @($PackageInfo)
+function Validate-Python-DocMsPackages($PackageInfo, $PackageInfos, $PackageSourceOverride, $DocValidationImageId)
+{
+  # While eng/common/scripts/Update-DocsMsMetadata.ps1 is still passing a single packageInfo, process as a batch
+  if (!$PackageInfos) {
+    $PackageInfos =  @($PackageInfo)
+  }
+
+  # Adding these for diagnostics purposes so we know which version of python is being
+  # executed and dumping all the pip packages for this install of python
+  Write-Host "Executing: which python"
+  $whichOutput = which python 2>&1
+  $whichOutput | ForEach-Object { Write-Host $_ }
+  Write-Host "`n"
+
+  Write-Host "Executing: python -m pip freeze --all"
+  $pipFreezeOutput = python -m pip freeze --all 2>&1
+  $pipFreezeOutput | ForEach-Object { Write-Host $_ }
+  Write-Host "`n"
+
+  $tempDirs = @()
+  $allSucceeded = $true
+  try {
+    foreach ($packageInfo in $PackageInfos) {
+      # Some packages won't have a version and this is the case when they're being onboarded manually
+      # and there's no version, only a repository and a SHA. In that case package we skip traditional
+      # package validation since the library doesn't exist yet outside of source and there's nothing
+      # the verification tools can do with this.
+      if ($packageInfo.Version -eq 'IGNORE') {
+        continue
+      }
+
+      # Create a temporary directory. The json file being passed to py2docfx will be in the root and
+      # the docs will be generated to a docsOutput subdirectory.
+      $outputRoot = New-Item `
+      -ItemType Directory `
+      -Path (Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()))
+
+      $tempDirs += $outputRoot
+
+      # Create the JSON file
+      $outputJsonFile = New-Item `
+      -ItemType File `
+      -Path (Join-Path $outputRoot ($packageInfo.Name + ".json"))
+
+      ## Write out the json file and echo the contents
+      $JsonString = Get-SinglePackageJsonForDocsValidation $packageInfo $PackageSourceOverride
+      $JsonString | Out-File $outputJsonFile
+      Write-Host "$JsonString"
+
+      # Create the docs output subdirectory. This is where the tool will generate its docs
+      $outputDocsDir = New-Item `
+      -ItemType Directory `
+      -Path (Join-Path $outputRoot "docsOutput")
+
+      # Force the python output to be unbuffered so we see more than just the warnings.
+      Write-Host "Executing: python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir"
+      $pyOutput = python -u -m py2docfx --param-file-path $outputJsonFile -o $outputDocsDir 2>&1
+      $pyOutput | ForEach-Object { Write-Host $_ }
+      Write-Host "`n"
+      if ($LASTEXITCODE -ne 0) {
+        LogWarning "py2docfx command failed, see output above."
+        $allSucceeded = $false
+      }
     }
-
-    $allSucceeded = $true
-    foreach ($item in $PackageInfos) {
-        # If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage
-        if ($item.Version -eq 'IGNORE') {
-            continue
-        }
-
-        $result = ValidatePackage `
-            -packageName $item.Name `
-            -packageVersion "==$($item.Version)" `
-            -PackageSourceOverride $PackageSourceOverride `
-            -DocValidationImageId $DocValidationImageId
-
-        if (!$result) {
-            $allSucceeded = $false
-        }
+  }
+  finally {
+    # Clean up any temp directories
+    foreach ($tempDir in $tempDirs)
+    {
+      Remove-Item -Force -Recurse $tempDir | Out-Null
     }
-
-    return $allSucceeded
+  }
+  return $allSucceeded
 }
diff --git a/eng/scripts/docs/py2docfx_requirements.txt b/eng/scripts/docs/py2docfx_requirements.txt
new file mode 100644
index 000000000000..22113936dea0
--- /dev/null
+++ b/eng/scripts/docs/py2docfx_requirements.txt
@@ -0,0 +1 @@
+py2docfx==0.1.11.dev1902551