diff --git a/.ci/DockerFile b/.ci/DockerFile index db977bd523f..74238516204 100644 --- a/.ci/DockerFile +++ b/.ci/DockerFile @@ -3,8 +3,12 @@ FROM mcr.microsoft.com/dotnet/core/sdk:${DOTNET_VERSION} AS elasticsearch-net-bu WORKDIR /sln + COPY ./*.sln ./nuget.config ./*.Build.props ./*.Build.targets ./ +COPY ./dotnet-tools.json ./ +RUN dotnet tool restore + # todo standardize on Build.props as Directory.Build.props needs that form COPY ./src/*.Build.props ./src/ COPY ./tests/*.Build.props ./tests/ @@ -21,13 +25,12 @@ RUN for file in $(find . -name "*.?sproj"); do mkdir -p $(dirname $file)/$(basen COPY build/scripts/scripts.fsproj ./build/scripts/ COPY .ci/Jenkins.csproj ./.ci/ -RUN ls -al -RUN ls -al tests -RUN ls -al src -RUN ls -al src/Elasticsearch.Net - -RUN dotnet restore - # Install app dependencies +RUN dotnet restore +# copy relevant files (see .dockerignore) COPY . . + +# making sure enough git info is available inside the container +RUN git rev-parse HEAD . + diff --git a/.ci/Jenkins.csproj b/.ci/Jenkins.csproj index 17ca88b06a7..a6b212d6faa 100644 --- a/.ci/Jenkins.csproj +++ b/.ci/Jenkins.csproj @@ -4,4 +4,10 @@ netcoreapp2.2 + + + .dockerignore + + + diff --git a/.ci/make.sh b/.ci/make.sh new file mode 100755 index 00000000000..aaa0395b3a4 --- /dev/null +++ b/.ci/make.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# parameters are available to this script + + +# common build entry script for all elasticsearch clients + +# ./.ci/make.sh bump VERSION +# ./.ci/make.sh release VERSION + +script_path=$(dirname "$(realpath -s "$0")") +repo=$(realpath "$script_path/../") + +# shellcheck disable=SC1090 +cmd=$1 +VERSION=$2 +STACK_VERSION=$VERSION +source "$script_path/functions/imports.sh" +set -euo pipefail + +output_folder=".ci/output" +OUTPUT_DIR="$repo/${output_folder}" + +DOTNET_VERSION=${DOTNET_VERSION-3.0.100} + +echo -e "\033[34;1mINFO:\033[0m VERSION ${STACK_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m OUTPUT_DIR ${OUTPUT_DIR}\033[0m" +echo -e "\033[34;1mINFO:\033[0m DOTNET_VERSION ${DOTNET_VERSION}\033[0m" + +echo -e "\033[1m>>>>> Build [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build --file .ci/DockerFile --tag elastic/elasticsearch-net . + +echo -e "\033[1m>>>>> Run [elastic/elasticsearch-net container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +mkdir -p "$OUTPUT_DIR" + +docker run \ + --network=${network_name} \ + --env "DOTNET_VERSION" \ + --name test-runner \ + --volume "${OUTPUT_DIR}:/sln/${output_folder}" \ + --rm \ + elastic/elasticsearch-net \ + ./build.sh release "$VERSION" "$output_folder" "skiptests" diff --git a/.dockerignore b/.dockerignore index d11d281070c..68c27f47b1c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,4 @@ scripts/output -.git - - -.git **/Obj/ **/obj/ @@ -35,8 +31,6 @@ PublishProfiles/ *.vspx /.symbols nuget.exe -*net45.csproj -*k10.csproj App_Data/ bower_components node_modules @@ -48,3 +42,5 @@ node_modules launchSettings.json bin obj + +.git/objects/* diff --git a/.gitignore b/.gitignore index 531a83d3858..172e1dcc869 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,8 @@ build/tools/* !build/*.targets !build/scripts +.ci/output + /dep/Newtonsoft.Json.4.0.2 !docs/build diff --git a/build/scripts/Commandline.fs b/build/scripts/Commandline.fs index 7cf6513d0ec..3ca58b9a90f 100644 --- a/build/scripts/Commandline.fs +++ b/build/scripts/Commandline.fs @@ -7,6 +7,7 @@ namespace Scripts open System open System.Runtime.InteropServices open Fake.Core +open Fake.IO //this is ugly but a direct port of what used to be duplicated in our DOS and bash scripts module Commandline = @@ -80,7 +81,7 @@ Execution hints can be provided anywhere on the command line type MultiTarget = All | One - type VersionArguments = { Version: string; } + type VersionArguments = { Version: string; OutputLocation: string option } type TestArguments = { TrxExport: bool; CodeCoverage: bool; TestFilter: string option; } type IntegrationArguments = { TrxExport: bool; TestFilter: string option; ClusterFilter: string option; ElasticsearchVersions: string list; } @@ -197,7 +198,10 @@ Execution hints can be provided anywhere on the command line | ["profile"] -> parsed | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail } - | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version }; } + | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = None }; } + | ["release"; version; path] -> + if (not <| System.IO.Directory.Exists path) then failwithf "'%s' is not an existing directory" (Path.getFullName path) + { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = Some path }; } | ["canary"] -> { parsed with CommandArguments = Test { diff --git a/build/scripts/ReposTooling.fs b/build/scripts/ReposTooling.fs index 92e1d08e149..2efb4016155 100644 --- a/build/scripts/ReposTooling.fs +++ b/build/scripts/ReposTooling.fs @@ -57,4 +57,9 @@ module ReposTooling = let Rewriter args = restoreOnce.Force() Tooling.DotNet.ExecIn "." (List.append [assemblyRewriter] (List.ofSeq args)) |> ignore + + let private packageValidator = "nupkg-validator" + let PackageValidator args = + restoreOnce.Force() + Tooling.DotNet.ExecIn "." (List.append [packageValidator] (List.ofSeq args)) |> ignore diff --git a/build/scripts/Targets.fs b/build/scripts/Targets.fs index 29e9df9b5d0..40cab418d7d 100644 --- a/build/scripts/Targets.fs +++ b/build/scripts/Targets.fs @@ -10,6 +10,7 @@ open System.IO open Build open Commandline open Bullseye +open Octokit open ProcNet open Fake.Core @@ -90,7 +91,12 @@ module Main = //RELEASE command "release" releaseChain <| fun _ -> - printfn "Finished Release Build %O" artifactsVersion + let outputPath = match parsed.CommandArguments with | SetVersion c -> c.OutputLocation | _ -> None + match outputPath with + | None -> printfn "Finished Release Build %O, artifacts available at: %s" artifactsVersion Paths.BuildOutput + | Some path -> + Fake.IO.Shell.cp_r Paths.BuildOutput path + printfn "Finished Release Build %O, output copied to: %s" artifactsVersion path conditional "test-nuget-package" (not parsed.SkipTests && Environment.isWindows) <| fun _ -> // run release unit tests puts packages in the system cache prevent this from happening locally diff --git a/build/scripts/Versioning.fs b/build/scripts/Versioning.fs index 619713c59f7..597bdfd5df4 100644 --- a/build/scripts/Versioning.fs +++ b/build/scripts/Versioning.fs @@ -12,6 +12,7 @@ open Commandline open Fake.Core open Fake.IO open Fake.IO.Globbing.Operators +open Fake.Tools.Git open Newtonsoft.Json module Versioning = @@ -112,60 +113,7 @@ module Versioning = | NoChange n -> n | Update (newVersion, _) -> newVersion - let private sn () = - match notWindows with - | true -> "sn" - | false -> - let programFiles = Environment.environVar "PROGRAMFILES(X86)" - if not (Directory.Exists programFiles) then failwith "Can not locate 64 bit program files" - let windowsSdks = ["v10.0A"; "v8.1A"; "v8.1"; "v8.0"; "v7.0A";] - let dotNetVersion = ["4.7.2"; "4.7.1"; "4.7"; "4.6.2"; "4.6.1"; "4.0"] - let combinations = List.allPairs windowsSdks dotNetVersion - let winFolder w = Path.Combine(programFiles, "Microsoft SDKs", "Windows", w, "bin") - let sdkFolder w d = - let folder = sprintf "NETFX %s Tools" d - Path.Combine(winFolder w, folder) - let snExe w d = Path.Combine(sdkFolder w d, "sn.exe") - let sn = combinations |> List.map (fun (w, d) -> snExe w d) |> List.tryFind File.exists - match sn with - | Some sn -> sn - | None -> failwithf "Could not locate sn.exe" - let private officialToken = "96c599bbe3e70f5d" - let private keyFile = Paths.Keys "keypair.snk" - - let private validate dll name = - let sn = sn () - let out = Tooling.readQuiet sn ["-v"; dll;] - - // Mono StrongName - version 5.18.1.0 - // returns `is strongnamed` - let valid = (out.ExitCode, out.Output |> Seq.tryFindIndex(fun s -> s.Line.Contains("is valid") || s.Line.Contains("is strongnamed"))) - match valid with - | (0, Some i) when i >= 0 -> printfn "%s is strongnamed" name - | (_, _) -> failwithf "%s is NOT strongnamed" dll - - let out = Tooling.readQuiet sn ["-T"; dll;] - - let tokenMessage = (out.Output |> Seq.tryFind(fun s -> s.Line.Contains("Public key token", StringComparison.OrdinalIgnoreCase))); - - // Mono StrongName - version 5.18.1.0 - // returns `Key Token:` - let token = - match tokenMessage with - | Some s -> Some <| (s.Line.Replace("Public Key Token:", "").Replace("Public key token is", "")).Trim() - | None -> None - - let valid = (out.ExitCode, token) - match valid with - | (0, Some t) when t = officialToken -> printfn "%s was signed with official key token %s" name t - | (_, Some t) -> printfn "%s was not signed with the official token: %s but %s" name officialToken t - | (_, None) -> printfn "%s was not signed at all" name - - let private validateDllStrongName dll name = - match File.Exists dll with - | true -> validate dll name - | _ -> failwithf "Attempted to verify signature of %s but it was not found!" dll let BuiltArtifacts (version: AnchoredVersion) = let packages = @@ -179,14 +127,25 @@ module Versioning = packages let ValidateArtifacts version = - let fileVersion = version.AssemblyFile let tmp = "build/output/_packages/tmp" let packages = BuiltArtifacts version printf "%O" packages packages + // do not validate versioned packages for now + |> Seq.filter(fun f -> not <| f.NugetId.EndsWith(sprintf ".v%i" version.Assembly.Major)) |> Seq.iter(fun p -> + let v = sprintf "%O+%s" version.Full (Information.getCurrentSHA1(".")) + // loading dlls is locked down on APPVEYOR so we can not assert release mode + let ciArgs = + let appVeyor = Environment.hasEnvironVar "APPVEYOR" + let azDevops = Environment.hasEnvironVar "TF_BUILD" + if appVeyor || azDevops then ["-r"; "true"] else [] + ReposTooling.PackageValidator + <| [p.Package; "-v"; v; "-a"; p.AssemblyName; "-k"; officialToken] @ ciArgs + |> ignore + Zip.unzip tmp p.Package let nugetId = p.NugetId @@ -200,22 +159,9 @@ module Versioning = let command = [ sprintf "previous-nuget|%s|%s|%s" nugetId (version.Full.ToString()) tfm; sprintf "directory|%s" fullPath - "-a"; "-f"; "github-comment";] + "-a"; "-f"; "github-comment"; "--output"; Paths.BuildOutput] ReposTooling.Differ command ) - - !! (sprintf "%s/**/*.dll" tmp) - |> Seq.iter(fun f -> - let fv = FileVersionInfo.GetVersionInfo(f) - let name = AssemblyName.GetAssemblyName(f) - let a = name.Version - printfn "Assembly: %A File: %s Product: %s => %s" a fv.FileVersion fv.ProductVersion f - if (a.Minor > 0 || a.Revision > 0 || a.Build > 0) then failwith (sprintf "%s assembly version is not sticky to its major component" f) - if (parse (fv.ProductVersion) <> version.Full) then - failwith (sprintf "Expected product info %s to match new version %O " fv.ProductVersion fileVersion) - - validateDllStrongName f f - ) Directory.delete tmp ) \ No newline at end of file diff --git a/build/scripts/scripts.fsproj b/build/scripts/scripts.fsproj index 633e63c61f5..8deddc9f6f6 100644 --- a/build/scripts/scripts.fsproj +++ b/build/scripts/scripts.fsproj @@ -44,6 +44,7 @@ + diff --git a/dotnet-tools.json b/dotnet-tools.json index 675f50c9fe3..4d75caa706b 100644 --- a/dotnet-tools.json +++ b/dotnet-tools.json @@ -13,6 +13,12 @@ "commands": [ "assembly-differ" ] + }, + "nupkg-validator": { + "version": "0.3.1", + "commands": [ + "nupkg-validator" + ] } } } \ No newline at end of file