[Backport 7.x] Introduce .ci/make.sh (#4838)

.ci/DockerFile

@@ -3,8 +3,12 @@ FROM mcr.microsoft.com/dotnet/core/sdk:${DOTNET_VERSION} AS elasticsearch-net-bu
 
 WORKDIR /sln
 
+
 COPY ./*.sln ./nuget.config ./*.Build.props ./*.Build.targets ./
 
+COPY ./dotnet-tools.json ./
+RUN dotnet tool restore
+
 # todo standardize on Build.props as Directory.Build.props needs that form
 COPY ./src/*.Build.props  ./src/
 COPY ./tests/*.Build.props  ./tests/
@@ -21,13 +25,12 @@ RUN for file in $(find . -name "*.?sproj"); do mkdir -p $(dirname $file)/$(basen
 COPY build/scripts/scripts.fsproj ./build/scripts/
 COPY .ci/Jenkins.csproj ./.ci/
 
-RUN ls -al 
-RUN ls -al tests
-RUN ls -al src
-RUN ls -al src/Elasticsearch.Net
-
-RUN dotnet restore 
-
 # Install app dependencies
+RUN dotnet restore 
 
+# copy relevant files (see .dockerignore)
 COPY . .
+
+# making sure enough git info is available inside the container
+RUN git rev-parse HEAD .
+

.ci/Jenkins.csproj

@@ -4,4 +4,10 @@
         <TargetFramework>netcoreapp2.2</TargetFramework>
     </PropertyGroup>
 
+    <ItemGroup>
+      <Content Include="..\.dockerignore">
+        <Link>.dockerignore</Link>
+      </Content>
+    </ItemGroup>
+
 </Project>

.ci/make.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# parameters are available to this script
+
+
+# common build entry script for all elasticsearch clients
+
+# ./.ci/make.sh bump VERSION
+# ./.ci/make.sh release VERSION
+
+script_path=$(dirname "$(realpath -s "$0")")
+repo=$(realpath "$script_path/../")
+
+# shellcheck disable=SC1090
+cmd=$1
+VERSION=$2
+STACK_VERSION=$VERSION
+source "$script_path/functions/imports.sh"
+set -euo pipefail
+
+output_folder=".ci/output"
+OUTPUT_DIR="$repo/${output_folder}"
+
+DOTNET_VERSION=${DOTNET_VERSION-3.0.100}
+
+echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m OUTPUT_DIR ${OUTPUT_DIR}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m DOTNET_VERSION ${DOTNET_VERSION}\033[0m"
+
+echo -e "\033[1m>>>>> Build [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+docker build --file .ci/DockerFile --tag elastic/elasticsearch-net .
+
+echo -e "\033[1m>>>>> Run [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+mkdir -p "$OUTPUT_DIR"
+
+docker run \
+  --network=${network_name} \
+  --env "DOTNET_VERSION" \
+  --name test-runner \
+  --volume "${OUTPUT_DIR}:/sln/${output_folder}" \
+  --rm \
+  elastic/elasticsearch-net \
+  ./build.sh release "$VERSION" "$output_folder" "skiptests"

.dockerignore

@@ -1,8 +1,4 @@
 scripts/output
-.git
-
-
-.git
 
 **/Obj/
 **/obj/
@@ -35,8 +31,6 @@ PublishProfiles/
 *.vspx
 /.symbols
 nuget.exe
-*net45.csproj
-*k10.csproj
 App_Data/
 bower_components
 node_modules
@@ -48,3 +42,5 @@ node_modules
 launchSettings.json
 bin
 obj
+
+.git/objects/*

.gitignore

@@ -57,6 +57,8 @@ build/tools/*
 !build/*.targets
 !build/scripts
 
+.ci/output
+
 
 /dep/Newtonsoft.Json.4.0.2
 !docs/build

build/scripts/Commandline.fs

@@ -7,6 +7,7 @@ namespace Scripts
 open System
 open System.Runtime.InteropServices
 open Fake.Core
+open Fake.IO
 
 //this is ugly but a direct port of what used to be duplicated in our DOS and bash scripts
 module Commandline =
@@ -80,7 +81,7 @@ Execution hints can be provided anywhere on the command line
 
     type MultiTarget = All | One
 
-    type VersionArguments = { Version: string; }
+    type VersionArguments = { Version: string; OutputLocation: string option }
     type TestArguments = { TrxExport: bool; CodeCoverage: bool; TestFilter: string option; }
     type IntegrationArguments = { TrxExport: bool; TestFilter: string option; ClusterFilter: string option; ElasticsearchVersions: string list; }
 
@@ -197,7 +198,10 @@ Execution hints can be provided anywhere on the command line
         | ["profile"] -> parsed
         | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail }
 
-        | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version }; }
+        | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = None }; }
+        | ["release"; version; path] ->
+            if (not <| System.IO.Directory.Exists path) then failwithf "'%s' is not an existing directory" (Path.getFullName path)
+            { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = Some path }; }
         | ["canary"] ->
             {
                 parsed with CommandArguments = Test {

build/scripts/ReposTooling.fs

@@ -57,4 +57,9 @@ module ReposTooling =
     let Rewriter args =
         restoreOnce.Force()
         Tooling.DotNet.ExecIn "." (List.append [assemblyRewriter] (List.ofSeq args)) |> ignore
+
+    let private packageValidator = "nupkg-validator"
+    let PackageValidator args =
+        restoreOnce.Force()
+        Tooling.DotNet.ExecIn "." (List.append [packageValidator] (List.ofSeq args)) |> ignore
 

build/scripts/Targets.fs

@@ -10,6 +10,7 @@ open System.IO
 open Build
 open Commandline
 open Bullseye
+open Octokit
 open ProcNet
 open Fake.Core
 
@@ -90,7 +91,12 @@ module Main =
 
         //RELEASE
         command "release" releaseChain <| fun _ ->
-            printfn "Finished Release Build %O" artifactsVersion
+            let outputPath = match parsed.CommandArguments with | SetVersion c -> c.OutputLocation | _ -> None
+            match outputPath with
+            | None -> printfn "Finished Release Build %O, artifacts available at: %s" artifactsVersion Paths.BuildOutput
+            | Some path ->
+                Fake.IO.Shell.cp_r Paths.BuildOutput path
+                printfn "Finished Release Build %O, output copied to: %s" artifactsVersion path
 
         conditional "test-nuget-package" (not parsed.SkipTests && Environment.isWindows)  <| fun _ -> 
             // run release unit tests puts packages in the system cache prevent this from happening locally

build/scripts/Versioning.fs

@@ -12,6 +12,7 @@ open Commandline
 open Fake.Core
 open Fake.IO
 open Fake.IO.Globbing.Operators
+open Fake.Tools.Git
 open Newtonsoft.Json
 
 module Versioning = 
@@ -112,60 +113,7 @@ module Versioning =
         | NoChange n -> n
         | Update (newVersion, _) -> newVersion
 
-    let private sn () =
-            match notWindows with 
-            | true -> "sn"
-            | false ->
-                let programFiles = Environment.environVar "PROGRAMFILES(X86)"
-                if not (Directory.Exists programFiles) then failwith "Can not locate 64 bit program files"
-                let windowsSdks =  ["v10.0A"; "v8.1A"; "v8.1"; "v8.0"; "v7.0A";]
-                let dotNetVersion = ["4.7.2"; "4.7.1"; "4.7"; "4.6.2"; "4.6.1"; "4.0"]
-                let combinations = List.allPairs windowsSdks dotNetVersion
-                let winFolder w = Path.Combine(programFiles, "Microsoft SDKs", "Windows", w, "bin")
-                let sdkFolder w d = 
-                    let folder = sprintf "NETFX %s Tools" d
-                    Path.Combine(winFolder w, folder)
-                let snExe w d = Path.Combine(sdkFolder w d, "sn.exe")
-                let sn = combinations |> List.map (fun (w, d) -> snExe w d) |> List.tryFind File.exists
-                match sn with
-                | Some sn -> sn
-                | None -> failwithf "Could not locate sn.exe"
-
     let private officialToken = "96c599bbe3e70f5d"
-    let private keyFile = Paths.Keys "keypair.snk"
-
-    let private validate dll name =
-        let sn = sn ()
-        let out = Tooling.readQuiet sn ["-v"; dll;]
-
-        // Mono StrongName - version 5.18.1.0
-        // returns `is strongnamed` 
-        let valid = (out.ExitCode, out.Output |> Seq.tryFindIndex(fun s -> s.Line.Contains("is valid") || s.Line.Contains("is strongnamed")))
-        match valid with
-        | (0, Some i) when i >= 0 -> printfn "%s is strongnamed" name 
-        | (_, _) -> failwithf "%s is NOT strongnamed" dll
-
-        let out = Tooling.readQuiet sn ["-T"; dll;]
-
-        let tokenMessage = (out.Output |> Seq.tryFind(fun s -> s.Line.Contains("Public key token", StringComparison.OrdinalIgnoreCase)));
-
-        // Mono StrongName - version 5.18.1.0
-        // returns `Key Token:` 
-        let token =
-            match tokenMessage with
-            | Some s -> Some <| (s.Line.Replace("Public Key Token:", "").Replace("Public key token is", "")).Trim()
-            | None -> None
-
-        let valid = (out.ExitCode, token)
-        match valid with
-        | (0, Some t) when t = officialToken  -> printfn "%s was signed with official key token %s" name t
-        | (_, Some t) -> printfn "%s was not signed with the official token: %s but %s" name officialToken t
-        | (_, None) -> printfn "%s was not signed at all" name 
-
-    let private validateDllStrongName dll name =
-        match File.Exists dll with
-        | true -> validate dll name 
-        | _ -> failwithf "Attempted to verify signature of %s but it was not found!" dll
 
     let BuiltArtifacts (version: AnchoredVersion) = 
         let packages =
@@ -179,14 +127,25 @@ module Versioning =
         packages
 
     let ValidateArtifacts version =
-        let fileVersion = version.AssemblyFile
         let tmp = "build/output/_packages/tmp"
 
         let packages = BuiltArtifacts version
         printf "%O" packages
 
         packages
+        // do not validate versioned packages for now
+        |> Seq.filter(fun f -> not <| f.NugetId.EndsWith(sprintf ".v%i" version.Assembly.Major))
         |> Seq.iter(fun p ->
+            let v = sprintf "%O+%s" version.Full (Information.getCurrentSHA1("."))
+            // loading dlls is locked down on APPVEYOR so we can not assert release mode
+            let ciArgs =
+                let appVeyor = Environment.hasEnvironVar "APPVEYOR"
+                let azDevops = Environment.hasEnvironVar "TF_BUILD"
+                if  appVeyor || azDevops then ["-r"; "true"] else []
+            ReposTooling.PackageValidator
+                <| [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] @ ciArgs
+                |> ignore 
+
             Zip.unzip tmp p.Package
             let nugetId = p.NugetId
 
@@ -200,22 +159,9 @@ module Versioning =
 
                 let command = [ sprintf "previous-nuget|%s|%s|%s" nugetId (version.Full.ToString()) tfm;
                                 sprintf "directory|%s" fullPath
-                                "-a"; "-f"; "github-comment";]
+                                "-a"; "-f"; "github-comment"; "--output"; Paths.BuildOutput]
 
                 ReposTooling.Differ command
             )
-
-            !! (sprintf "%s/**/*.dll" tmp)
-            |> Seq.iter(fun f -> 
-                let fv = FileVersionInfo.GetVersionInfo(f)
-                let name = AssemblyName.GetAssemblyName(f)
-                let a = name.Version
-                printfn "Assembly: %A File: %s Product: %s => %s" a fv.FileVersion fv.ProductVersion f
-                if (a.Minor > 0 || a.Revision > 0 || a.Build > 0) then failwith (sprintf "%s assembly version is not sticky to its major component" f)
-                if (parse (fv.ProductVersion) <> version.Full) then
-                    failwith (sprintf "Expected product info %s to match new version %O " fv.ProductVersion fileVersion)
-
-                validateDllStrongName f f
-            )
             Directory.delete tmp
         )

build/scripts/scripts.fsproj

@@ -44,6 +44,7 @@
     <PackageReference Include="Fake.Core.SemVer" Version="5.15.0" />
     <PackageReference Include="Fake.IO.FileSystem" Version="5.15.0" />
     <PackageReference Include="Fake.IO.Zip" Version="5.15.0" />
+    <PackageReference Include="Fake.Tools.Git" Version="5.15.0" />
 
     <PackageReference Include="ILRepack" Version="2.1.0-beta1" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />

dotnet-tools.json

@@ -13,6 +13,12 @@
       "commands": [
         "assembly-differ"
       ]
+    },
+    "nupkg-validator": {
+      "version": "0.3.1",
+      "commands": [
+        "nupkg-validator"
+      ]
     }
   }
 }