From 9cb3e60ae38a4834594ee8cd4a85a79e2e324b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Wed, 19 Sep 2018 16:02:15 -0700 Subject: [PATCH] [Search] Generate secondary region's auxiliary data (#555) Tweaks the `Search.GenerateAuxiliaryData` job to generate auxiliary data for each region, including secondary regions (like US South Central and South East Asia). --- .../Search.GenerateAuxiliaryData.Asia.cmd | 12 +++++----- ...ch.GenerateAuxiliaryData.SouthEastAsia.cmd | 16 ++++++++++++++ .../Search.GenerateAuxiliaryData.USSC.cmd | 16 ++++++++++++++ .../Scripts/Search.GenerateAuxiliaryData.cmd | 10 ++++----- .../Search.GenerateAuxiliaryData.nuspec | 4 +++- .../Settings/dev-southeastasia.json | 22 +++++++++++++++++++ .../Settings/dev-ussc.json | 22 +++++++++++++++++++ .../Settings/int-ussc.json | 22 +++++++++++++++++++ .../Settings/prod-southeastasia.json | 22 +++++++++++++++++++ .../Settings/prod-ussc.json | 22 +++++++++++++++++++ 10 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.SouthEastAsia.cmd create mode 100644 src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.USSC.cmd create mode 100644 src/Search.GenerateAuxiliaryData/Settings/dev-southeastasia.json create mode 100644 src/Search.GenerateAuxiliaryData/Settings/dev-ussc.json create mode 100644 src/Search.GenerateAuxiliaryData/Settings/int-ussc.json create mode 100644 src/Search.GenerateAuxiliaryData/Settings/prod-southeastasia.json create mode 100644 src/Search.GenerateAuxiliaryData/Settings/prod-ussc.json diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd index 90b165dc9..1556f9fc9 100644 --- a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.Asia.cmd @@ -5,12 +5,12 @@ REM This script is the same as Search.GenerateAuxillaryData.cmd. However, this c cd bin :Top - echo "Starting job - #{Jobs.search.generateauxiliarydata.Title}" + echo "Starting job - #{Jobs.Asia.search.generateauxiliarydata.Title}" - title #{Jobs.search.generateauxiliarydata.Title} + title #{Jobs.Asia.search.generateauxiliarydata.Title} - start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" - - echo "Finished #{Jobs.search.generateauxiliarydata.Title}" + start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.Asia.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" - goto Top \ No newline at end of file + echo "Finished #{Jobs.Asia.search.generateauxiliarydata.Title}" + + goto Top \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.SouthEastAsia.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.SouthEastAsia.cmd new file mode 100644 index 000000000..9fca05d6c --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.SouthEastAsia.cmd @@ -0,0 +1,16 @@ +@echo OFF + +REM This script is the same as Search.GenerateAuxillaryData.cmd. However, this copy is required until "Jobs.ServiceNames" deployment config is consolidated. + +cd bin + +:Top + echo "Starting job - #{Jobs.SouthEastAsia.search.generateauxiliarydata.Title}" + + title #{Jobs.SouthEastAsia.search.generateauxiliarydata.Title} + + start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.SouthEastAsia.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" + + echo "Finished #{Jobs.SouthEastAsia.search.generateauxiliarydata.Title}" + + goto Top \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.USSC.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.USSC.cmd new file mode 100644 index 000000000..392c8c3d9 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.USSC.cmd @@ -0,0 +1,16 @@ +@echo OFF + +REM This script is the same as Search.GenerateAuxillaryData.cmd. However, this copy is required until "Jobs.ServiceNames" deployment config is consolidated. + +cd bin + +:Top + echo "Starting job - #{Jobs.USSC.search.generateauxiliarydata.Title}" + + title #{Jobs.USSC.search.generateauxiliarydata.Title} + + start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.USSC.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" + + echo "Finished #{Jobs.USSC.search.generateauxiliarydata.Title}" + + goto Top \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd index 0ce435c6e..ff6cf1e39 100644 --- a/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd +++ b/src/Search.GenerateAuxiliaryData/Scripts/Search.GenerateAuxiliaryData.cmd @@ -3,12 +3,12 @@ cd bin :Top - echo "Starting job - #{Jobs.search.generateauxiliarydata.Title}" + echo "Starting job - #{Jobs.search.generateauxiliarydata.Title}" - title #{Jobs.search.generateauxiliarydata.Title} + title #{Jobs.search.generateauxiliarydata.Title} start /w search.generateauxiliarydata.exe -Configuration "#{Jobs.search.generateauxiliarydata.Configuration}" -verbose true -Sleep #{Jobs.search.generateauxiliarydata.Sleep} -InstrumentationKey "#{Jobs.search.generateauxiliarydata.ApplicationInsightsInstrumentationKey}" - - echo "Finished #{Jobs.search.generateauxiliarydata.Title}" - goto Top \ No newline at end of file + echo "Finished #{Jobs.search.generateauxiliarydata.Title}" + + goto Top \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec index 7352e9c37..7a4356e12 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.nuspec @@ -1,7 +1,7 @@ - Search.GenerateAuxiliaryData.$branch$ + Search.GenerateAuxiliaryData $version$ Search.GenerateAuxiliaryData .NET Foundation @@ -13,6 +13,8 @@ + + diff --git a/src/Search.GenerateAuxiliaryData/Settings/dev-southeastasia.json b/src/Search.GenerateAuxiliaryData/Settings/dev-southeastasia.json new file mode 100644 index 000000000..ba0358d2f --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/dev-southeastasia.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdevsea;AccountKey=$$Dev-NuGetDevSeaStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData.Asia;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData.Asia;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbReader.ClientId};AadCertificate=$$dev-statisticsdb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/dev-ussc.json b/src/Search.GenerateAuxiliaryData/Settings/dev-ussc.json new file mode 100644 index 000000000..856ff05d9 --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/dev-ussc.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdevussc;AccountKey=$$Dev-NuGetDevUsscStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.GalleryDbReader.ClientId};AadCertificate=$$dev-gallerydb-reader$$" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;Persist Security Info=False;Connect Timeout=30;Encrypt=True;TrustServerCertificate=False;Application Name=Search.GenerateAuxData;AadTenant=#{Deployment.Azure.ActiveDirectory.Tenant};AadClientId=#{Deployment.Azure.ActiveDirectory.StatisticsDbReader.ClientId};AadCertificate=$$dev-statisticsdb-reader$$" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/int-ussc.json b/src/Search.GenerateAuxiliaryData/Settings/int-ussc.json new file mode 100644 index 000000000..ce0ad68ff --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/int-ussc.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetintussc;AccountKey=$$Int-NuGetIntUsscStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBReadOnly-UserName$$;Password=$$Int-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/prod-southeastasia.json b/src/Search.GenerateAuxiliaryData/Settings/prod-southeastasia.json new file mode 100644 index 000000000..386c6905f --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/prod-southeastasia.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetprodsea;AccountKey=$$Prod-NuGetProdSeaStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBReadOnly-UserName$$;Password=$$Prod-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Search.GenerateAuxiliaryData/Settings/prod-ussc.json b/src/Search.GenerateAuxiliaryData/Settings/prod-ussc.json new file mode 100644 index 000000000..fade39b6c --- /dev/null +++ b/src/Search.GenerateAuxiliaryData/Settings/prod-ussc.json @@ -0,0 +1,22 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetprodussc;AccountKey=$$Prod-NuGetProdUsscStorage-Key$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBReadOnly-UserName$$;Password=$$Prod-StatisticsDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file