Add incremental build (#1700)

* Add semaphores for incremental build * Updates based on PR review comments * One more PR comment update
dotnet · Aug 14, 2020 · 32a3fcd · 32a3fcd
1 parent 6997f78
commit 32a3fcd
Show file tree

Hide file tree

Showing 17 changed files with 231 additions and 63 deletions.
diff --git a/.gitignore b/.gitignore
@@ -279,3 +279,4 @@ testing-smoke/
 # UpdateDependenciesAndSubmitPullRequest: if no upgrades are performed, the
 # sanity check fails when "git status" shows this file.
 msbuild.ProjectImports.zip
+Build_*.ProjectImports.zip
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -202,10 +202,13 @@
     <RootRepo>known-good</RootRepo>
     <RootRepoTests>known-good-tests</RootRepoTests>
   </PropertyGroup>
-
-  <ItemGroup>
-    <SourceBuiltPackage Include="$(SourceBuiltPackagesPath)/*.nupkg" />
-  </ItemGroup>
+
+  <!--
+    Semaphore path for incremental builds
+  -->
+  <PropertyGroup>
+    <CompletedSemaphorePath>$(BaseIntermediatePath)semaphores/</CompletedSemaphorePath>
+  </PropertyGroup>
 
   <!-- CLI internal version is statically set by us to a version that will never show up in the wild.
        This ensures we will never restore a public version instead of our source-built version.  We

diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -13,10 +13,14 @@
   <Target Name="AcquireDarc"
           Condition="
             '$(OfflineBuild)' != 'true' and
-            '$(SkipAcquireDarc)' != 'true'">
+            '$(SkipAcquireDarc)' != 'true'"
+          Inputs="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
+          Outputs="$(CompletedSemaphorePath)AcquireDarc.complete">
     <MSBuild
       Projects="$(MSBuildThisFileDirectory)tools-local\acquire-darc\acquire-darc.proj"
       Targets="AcquireDarcCore" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)AcquireDarc.complete" Overwrite="true" />
   </Target>
 
   <Target Name="GetCommonDarcCloneCommand" DependsOnTargets="AcquireDarc">

diff --git a/build-source-tarball.sh b/build-source-tarball.sh
@@ -143,6 +143,6 @@ mkdir -p "$FULL_TARBALL_ROOT"
 MAIN_BUILD_ARGS+=( "/p:TarballRoot=$FULL_TARBALL_ROOT" )
 MAIN_BUILD_ARGS+=( "/p:PackSourceBuildTarball=true" )
 
-$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} "$@"
+$SCRIPT_ROOT/build.sh ${MAIN_BUILD_ARGS[@]} /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTarball_$(date +"%m%d%H%M%S").binlog "$@"
 
 echo "Done. Tarball created: $TARBALL_ROOT"
diff --git a/build.proj b/build.proj
@@ -67,7 +67,9 @@
   -->
   <Target Name="CopyAndDisassembleReferenceOnlyPackages"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(ArchiveDownloadedPackages)' == 'true' and '$(OS)' != 'Windows_NT'">
+          Condition="'$(OfflineBuild)' != 'true' and '$(ArchiveDownloadedPackages)' == 'true' and '$(OS)' != 'Windows_NT'"
+          Inputs="$(MSBuildProjectFullPath)" 
+          Outputs="$(CompletedSemaphorePath)CopyAndDisassembleReferenceOnlyPackages.complete" >
 
     <CopyReferenceOnlyPackages
         PackageCacheDir="$(PackagesDir)"
@@ -124,6 +126,8 @@
     <ReplaceTextInFile InputFile="%(RefAssyNuspecFiles.Identity)"
         OldText="http://go.microsoft.com/fwlink/?LinkId=529443"
         NewText="https://microsoft.mit-license.org/" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)CopyAndDisassembleReferenceOnlyPackages.complete" Overwrite="true" />
   </Target>
 
   <Target Name="ReplaceLicenseFiles"
@@ -148,7 +152,9 @@
 
   <Target Name="DownloadSourceBuildReferencePackages"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'"
+          Inputs="$(MSBuildProjectFullPath)" 
+          Outputs="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" >
     <PropertyGroup Condition="'$(DownloadSourceBuildReferencePackagesTimeoutSeconds)' == ''">
       <DownloadSourceBuildReferencePackagesTimeoutSeconds>600</DownloadSourceBuildReferencePackagesTimeoutSeconds>
     </PropertyGroup>
@@ -157,11 +163,15 @@
           SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
           DestinationFolder="$(ExternalTarballsDir)"
           TimeoutSeconds="$(DownloadSourceBuildReferencePackagesTimeoutSeconds)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuildReferencePackages.complete" Overwrite="true" />
   </Target>
 
   <Target Name="DownloadSourceBuiltArtifacts"
           AfterTargets="Build"
-          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
+          Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'"
+          Inputs="$(MSBuildProjectFullPath)" 
+          Outputs="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" >
     <PropertyGroup Condition="'$(DownloadSourceBuiltArtifactsTimeoutSeconds)' == ''">
       <DownloadSourceBuiltArtifactsTimeoutSeconds>1500</DownloadSourceBuiltArtifactsTimeoutSeconds>
     </PropertyGroup>
@@ -170,6 +180,8 @@
           SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
           DestinationFolder="$(ExternalTarballsDir)"
           TimeoutSeconds="$(DownloadSourceBuiltArtifactsTimeoutSeconds)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)DownloadSourceBuiltArtifacts.complete" Overwrite="true" />
   </Target>
 
   <!-- After building, generate a prebuilt usage report. -->
@@ -191,23 +203,31 @@
 
   <Target Name="ReportPoisonUsage"
           AfterTargets="Build"
-          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'"
+          Inputs="$(MSBuildProjectFullPath)" 
+          Outputs="$(CompletedSemaphorePath)ReportPoisonUsage.complete" >
     <ItemGroup>
       <FinalCliTarball Include="$(SourceBuiltTarBallPath)**/*$(TarBallExtension)" />
     </ItemGroup>
     <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
                     PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
   </Target>
 
-  <Target Name="GeneratePrebuiltBurndownData">
+  <Target Name="GeneratePrebuiltBurndownData"
+          Inputs="$(MSBuildProjectFullPath)" 
+          Outputs="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" >
     <WriteUsageBurndownData RootDirectory="$(ProjectDir)"
                             PrebuiltBaselineFile="$(OnlineBaselineDataFile)"
                             OutputFilePath="$(OnlinePrebuiltBurndownDataFile)" />
     <WriteUsageBurndownData RootDirectory="$(ProjectDir)"
                             PrebuiltBaselineFile="$(OfflineBaselineDataFile)"
                             OutputFilePath="$(OfflinePrebuiltBurndownDataFile)" />
+
+    <WriteLinesToFile File="$(CompletedSemaphorePath)GeneratePrebuiltBurndownData.complete" Overwrite="true" />
   </Target>
 
   <Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">

diff --git a/build.sh b/build.sh
@@ -134,5 +134,5 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 if [ "$alternateTarget" == "true" ]; then
   "$CLIPATH/dotnet" $SDKPATH/MSBuild.dll "$scriptroot/build.proj" /bl:source-build-test.binlog /flp:v=diag /clp:v=m "$@"
 else
-  "$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false -bl /flp:v=diag "$@"
+  "$scriptroot/eng/common/build.sh" --restore --build -c Release --warnaserror false /bl:$scriptroot/artifacts/log/Debug/Build_$(date +"%m%d%H%M%S").binlog /flp:v=diag "$@"
 fi
diff --git a/repos/Directory.Build.props b/repos/Directory.Build.props
@@ -32,6 +32,11 @@
     <RepoConsoleLogFile>$(LoggingDir)$(RepositoryName).log</RepoConsoleLogFile>
     <RedirectRepoOutputToLog Condition="'$(MinimalConsoleLogOutput)' == 'true'">&gt;&gt; $(RepoConsoleLogFile) 2&gt;&amp;1</RedirectRepoOutputToLog>
     <IsStable Condition="'$(UseStableVersions)' == 'true'">true</IsStable>
+
+    <!--
+      Repo specific semaphore path for incremental build
+    -->
+    <RepoCompletedSemaphorePath>$(CompletedSemaphorePath)$(RepositoryName)/</RepoCompletedSemaphorePath>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(OS)' == 'Windows_NT'">