From 6812715dce15fdfda055736e35f75c3248408c50 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 12:32:29 +0100 Subject: [PATCH 01/18] Fix `carton dev` crashing with SO sanitizer --- Package.swift | 10 +-- Sources/CartonCLI/Commands/Bundle.swift | 14 ++--- Sources/CartonCLI/Commands/Dev.swift | 12 ++-- Sources/CartonCLI/Commands/Test.swift | 2 +- Sources/CartonKit/Server/Application.swift | 2 +- Sources/CartonKit/Server/Server.swift | 11 ++-- Sources/CartonKit/Server/StaticArchive.swift | 10 +-- Sources/SwiftToolchain/BuildDescription.swift | 22 +++++++ Sources/SwiftToolchain/Builder.swift | 29 +++++++-- Sources/SwiftToolchain/Toolchain.swift | 62 ++++++------------- .../SwiftToolchain/ToolchainManagement.swift | 2 +- Sources/carton-release/HashArchive.swift | 7 +++ Tests/Fixtures/TestApp/Package.resolved | 4 +- Tests/Fixtures/TestApp/Package.swift | 2 +- .../TestApp/Sources/TestApp/main.swift | 5 +- 15 files changed, 104 insertions(+), 90 deletions(-) create mode 100644 Sources/SwiftToolchain/BuildDescription.swift diff --git a/Package.swift b/Package.swift index 852c69bf..9812b5a6 100644 --- a/Package.swift +++ b/Package.swift @@ -52,15 +52,6 @@ let package = Package( name: "Carton", dependencies: [ "CartonCLI", - // commented out for now. Will remove once confirmed working -// .product(name: "ArgumentParser", package: "swift-argument-parser"), -// .product(name: "AsyncHTTPClient", package: "async-http-client"), -// .product(name: "Crypto", package: "swift-crypto"), -// .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), -// .product(name: "Vapor", package: "vapor"), -// "CartonHelpers", -// openCombineProduct, -// "SwiftToolchain", ] ), .target( @@ -95,6 +86,7 @@ let package = Package( .product(name: "AsyncHTTPClient", package: "async-http-client"), openCombineProduct, "Splash", + "WasmTransformer", ] ), // This target is used only for release automation tasks and diff --git a/Sources/CartonCLI/Commands/Bundle.swift b/Sources/CartonCLI/Commands/Bundle.swift index fceff8dd..1cd4ee41 100644 --- a/Sources/CartonCLI/Commands/Bundle.swift +++ b/Sources/CartonCLI/Commands/Bundle.swift @@ -58,21 +58,21 @@ struct Bundle: ParsableCommand { let toolchain = try Toolchain(localFileSystem, terminal) let flavor = buildFlavor() - let (_, mainWasmPath, inferredProduct) = try toolchain.buildCurrentProject( + let build = try toolchain.buildCurrentProject( product: product, flavor: flavor ) try terminal.logLookup( "Right after building the main binary size is ", - localFileSystem.humanReadableFileSize(mainWasmPath), + localFileSystem.humanReadableFileSize(build.mainWasmPath), newline: true ) - try strip(mainWasmPath) + try strip(build.mainWasmPath) try terminal.logLookup( "After stripping debug info the main binary size is ", - localFileSystem.humanReadableFileSize(mainWasmPath), + localFileSystem.humanReadableFileSize(build.mainWasmPath), newline: true ) @@ -81,7 +81,7 @@ struct Bundle: ParsableCommand { try localFileSystem.createDirectory(bundleDirectory) let optimizedPath = AbsolutePath(bundleDirectory, "main.wasm") try ProcessRunner( - ["wasm-opt", "-Os", mainWasmPath.pathString, "-o", optimizedPath.pathString], + ["wasm-opt", "-Os", build.mainWasmPath.pathString, "-o", optimizedPath.pathString], terminal ).waitUntilFinished() try terminal.logLookup( @@ -93,10 +93,10 @@ struct Bundle: ParsableCommand { try copyToBundle( terminal: terminal, optimizedPath: optimizedPath, - buildDirectory: mainWasmPath.parentDirectory, + buildDirectory: build.mainWasmPath.parentDirectory, bundleDirectory: bundleDirectory, toolchain: toolchain, - product: inferredProduct + product: build.product ) terminal.write("Bundle generation finished successfully\n", inColor: .green, bold: true) diff --git a/Sources/CartonCLI/Commands/Dev.swift b/Sources/CartonCLI/Commands/Dev.swift index 0f512f85..e701ff16 100644 --- a/Sources/CartonCLI/Commands/Dev.swift +++ b/Sources/CartonCLI/Commands/Dev.swift @@ -97,7 +97,7 @@ struct Dev: ParsableCommand { } let flavor = buildFlavor() - let (arguments, mainWasmPath, inferredProduct) = try toolchain.buildCurrentProject( + let build = try toolchain.buildCurrentProject( product: product, flavor: flavor ) @@ -114,16 +114,16 @@ struct Dev: ParsableCommand { let sources = try paths.flatMap { try localFileSystem.traverseRecursively($0) } try Server( - with: .init( + .init( builder: Builder( - arguments: arguments, - mainWasmPath: mainWasmPath, + arguments: build.arguments, + mainWasmPath: build.mainWasmPath, pathsToWatch: sources, flavor, localFileSystem, terminal ), - mainWasmPath: mainWasmPath, + mainWasmPath: build.mainWasmPath, verbose: verbose, skipAutoOpen: skipAutoOpen, port: port, @@ -131,7 +131,7 @@ struct Dev: ParsableCommand { customIndexContent: HTML.readCustomIndexPage(at: customIndexPage, on: localFileSystem), // swiftlint:disable:next force_try manifest: try! toolchain.manifest.get(), - product: inferredProduct, + product: build.product, entrypoint: Self.entrypoint ), terminal diff --git a/Sources/CartonCLI/Commands/Test.swift b/Sources/CartonCLI/Commands/Test.swift index 5825d787..7f598e7f 100644 --- a/Sources/CartonCLI/Commands/Test.swift +++ b/Sources/CartonCLI/Commands/Test.swift @@ -100,7 +100,7 @@ struct Test: ParsableCommand { try runner.waitUntilFinished() } else { try Server( - with: .init( + .init( builder: nil, mainWasmPath: testBundlePath, verbose: true, diff --git a/Sources/CartonKit/Server/Application.swift b/Sources/CartonKit/Server/Application.swift index e83e99c9..f3c2213f 100644 --- a/Sources/CartonKit/Server/Application.swift +++ b/Sources/CartonKit/Server/Application.swift @@ -31,7 +31,7 @@ extension Application { let onWebSocketClose: (WebSocket) -> () } - func configure(with configuration: Configuration) { + func configure(_ configuration: Configuration) { http.server.configuration.port = configuration.port http.server.configuration.hostname = configuration.host diff --git a/Sources/CartonKit/Server/Server.swift b/Sources/CartonKit/Server/Server.swift index a7ba7172..48949451 100644 --- a/Sources/CartonKit/Server/Server.swift +++ b/Sources/CartonKit/Server/Server.swift @@ -114,7 +114,7 @@ public final class Server { } public init( - with configuration: Configuration, + _ configuration: Configuration, _ terminal: InteractiveWriter ) throws { if let builder = configuration.builder { @@ -133,7 +133,7 @@ public final class Server { try LoggingSystem.bootstrap(from: &env) app = Application(env) app.configure( - with: .init( + .init( port: configuration.port, host: configuration.host, mainWasmPath: configuration.mainWasmPath, @@ -143,7 +143,7 @@ public final class Server { entrypoint: configuration.entrypoint, onWebSocketOpen: { [weak self] ws, environment in if let handler = self?.createWSHandler( - with: configuration, + configuration, in: environment, terminal: terminal ) { @@ -177,7 +177,7 @@ public final class Server { .store(in: &subscriptions) } - /// Blocking function that starts the HTTP server + /// Blocking function that starts the HTTP server. public func run() throws { defer { app.shutdown() } try app.run() @@ -205,8 +205,9 @@ public final class Server { } extension Server { + /// Returns a handler that responds to WebSocket messages coming from the browser. func createWSHandler( - with configuration: Configuration, + _ configuration: Configuration, in environment: DestinationEnvironment, terminal: InteractiveWriter ) -> (WebSocket, String) -> () { diff --git a/Sources/CartonKit/Server/StaticArchive.swift b/Sources/CartonKit/Server/StaticArchive.swift index 7398622d..4bd98fd7 100644 --- a/Sources/CartonKit/Server/StaticArchive.swift +++ b/Sources/CartonKit/Server/StaticArchive.swift @@ -2,20 +2,20 @@ import TSCBasic public let devEntrypointSHA256 = ByteString([ 0xB1, 0x75, 0x6E, 0xC8, 0x39, 0x45, 0xE2, 0x06, 0x00, 0xDD, 0x2E, 0x4D, 0x67, 0xD4, 0xD6, 0xCE, - 0x35, 0x62, 0xA0, 0x3D, 0x9F, 0x63, 0x6E, 0x63, 0xAD, 0x79, 0xF2, 0x2D, 0x9D, 0x3C, 0x5C, 0x0A, + 0x35, 0x62, 0xA0, 0x3D, 0x9F, 0x63, 0x6E, 0x63, 0xAD, 0x79, 0xF2, 0x2D, 0x9D, 0x3C, 0x5C, 0x0A ]) public let bundleEntrypointSHA256 = ByteString([ 0x2B, 0xD2, 0xB5, 0x0C, 0x08, 0xFB, 0x5A, 0x8D, 0x55, 0xC4, 0x4B, 0x3A, 0x51, 0x63, 0x80, 0x1F, - 0xB9, 0x15, 0x50, 0xD2, 0xB6, 0x12, 0xC3, 0xDE, 0x3A, 0x7D, 0xEB, 0xB0, 0x1F, 0x40, 0x06, 0x3F, + 0xB9, 0x15, 0x50, 0xD2, 0xB6, 0x12, 0xC3, 0xDE, 0x3A, 0x7D, 0xEB, 0xB0, 0x1F, 0x40, 0x06, 0x3F ]) public let testEntrypointSHA256 = ByteString([ 0xCE, 0x38, 0xCE, 0x55, 0x05, 0x45, 0x93, 0x75, 0xE6, 0xCD, 0xCE, 0x1C, 0xA6, 0x67, 0x14, 0x16, - 0xBB, 0xAE, 0xF3, 0xFC, 0xFB, 0x86, 0xCB, 0x98, 0x1A, 0x66, 0x5C, 0xC7, 0x6C, 0x3B, 0x81, 0xF4, + 0xBB, 0xAE, 0xF3, 0xFC, 0xFB, 0x86, 0xCB, 0x98, 0x1A, 0x66, 0x5C, 0xC7, 0x6C, 0x3B, 0x81, 0xF4 ]) public let staticArchiveHash = ByteString([ - 0x4E, 0xC8, 0x39, 0xCA, 0xEE, 0x4F, 0x09, 0x8A, 0xA4, 0x95, 0x17, 0xD1, 0x13, 0xE1, 0xD9, 0x79, - 0xB9, 0x05, 0xE2, 0xF7, 0x9A, 0x75, 0xA2, 0x8D, 0x20, 0xDC, 0x6F, 0xEB, 0xDC, 0x00, 0xCA, 0x80, + 0x7A, 0x27, 0x1A, 0x64, 0x7A, 0xEB, 0xE1, 0x94, 0x24, 0xE6, 0x4A, 0x98, 0x14, 0x5F, 0xC3, 0xA7, + 0xB5, 0x6B, 0x81, 0x51, 0x75, 0xA0, 0x8E, 0xF4, 0x8D, 0x48, 0xC9, 0x9A, 0x60, 0x25, 0x24, 0x2F, ]) diff --git a/Sources/SwiftToolchain/BuildDescription.swift b/Sources/SwiftToolchain/BuildDescription.swift new file mode 100644 index 00000000..2e47b4e7 --- /dev/null +++ b/Sources/SwiftToolchain/BuildDescription.swift @@ -0,0 +1,22 @@ +// Copyright 2021 Carton contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import PackageModel +import TSCBasic + +public struct BuildDescription { + public let arguments: [String] + public let mainWasmPath: AbsolutePath + public let product: ProductDescription +} diff --git a/Sources/SwiftToolchain/Builder.swift b/Sources/SwiftToolchain/Builder.swift index 4d4a9135..83b1c846 100644 --- a/Sources/SwiftToolchain/Builder.swift +++ b/Sources/SwiftToolchain/Builder.swift @@ -49,12 +49,12 @@ public final class Builder { self.fileSystem = fileSystem } - public func run() -> AnyPublisher { + private func processPublisher(builderArguments: [String]) -> AnyPublisher { let buildStarted = Date() let process = ProcessRunner( - arguments, + builderArguments, loadingMessage: "Compiling...", - parser: DiagnosticsParser(), + parser: nil, terminal ) currentProcess = process @@ -69,8 +69,7 @@ public final class Builder { String(format: "%.2f seconds", abs(buildStarted.timeIntervalSinceNow)) ) - var transformers: [(inout InputByteStream, inout InMemoryOutputWriter) throws -> ()] = [ - ] + var transformers: [(inout InputByteStream, inout InMemoryOutputWriter) throws -> ()] = [] if self.flavor.environment != .other { transformers.append(I64ImportTransformer().transform) } @@ -85,7 +84,7 @@ public final class Builder { guard !transformers.isEmpty else { return } // FIXME: errors from these `try` expressions should be recoverable, not sure how to - // do that in `handleEvents`, and `flatMap` doesnt' fit here as we need to track + // do that in `handleEvents`, and `flatMap` doesn't fit here as we need to track // publisher completion. // swiftlint:disable force_try let binary = try! self.fileSystem.readFileContents(self.mainWasmPath) @@ -110,6 +109,24 @@ public final class Builder { .eraseToAnyPublisher() } + public func run() -> AnyPublisher { + switch flavor.sanitize { + case .none: + return processPublisher(builderArguments: arguments) + case .stackOverflow: + let sanitizerFile = + fileSystem.homeDirectory.appending(components: ".carton", "static", "so_sanitizer.wasm") + + var modifiedArguments = arguments + modifiedArguments.append(contentsOf: [ + "-Xlinker", sanitizerFile.pathString, + // stack-overflow-sanitizer depends on "--stack-first" + "-Xlinker", "--stack-first", + ]) + return processPublisher(builderArguments: modifiedArguments) + } + } + public func runAndWaitUntilFinished() throws { try tsc_await { completion in subscription = run() diff --git a/Sources/SwiftToolchain/Toolchain.swift b/Sources/SwiftToolchain/Toolchain.swift index 3515b2ba..55bcf3d4 100644 --- a/Sources/SwiftToolchain/Toolchain.swift +++ b/Sources/SwiftToolchain/Toolchain.swift @@ -206,32 +206,10 @@ public final class Toolchain { } } - private func buildPrelude(_ flavor: BuildFlavor, builderArguments: [String], - _ next: (_ builderArguments: [String]) throws -> ()) throws - { - switch flavor.sanitize { - case .none: - try next(builderArguments) - case .stackOverflow: - try withTemporaryFile(prefix: "stack-overflow-sanitizer") { supportingObjectFile in - supportingObjectFile.fileHandle.write( - Data(StackOverflowSanitizer.supportObjectFile) - ) - var builderArguments = builderArguments - builderArguments.append(contentsOf: [ - "-Xlinker", supportingObjectFile.path.pathString, - // stack-overflow-sanitizer depends on "--stack-first" - "-Xlinker", "--stack-first", - ]) - try next(builderArguments) - } - } - } - public func buildCurrentProject( product: String?, flavor: BuildFlavor - ) throws -> (builderArguments: [String], mainWasmPath: AbsolutePath, ProductDescription) { + ) throws -> BuildDescription { guard let product = try inferDevProduct(hint: product) else { throw ToolchainError.noExecutableProduct } @@ -264,16 +242,14 @@ public final class Toolchain { "--product", product.name, "--enable-test-discovery", "--triple", "wasm32-unknown-wasi", ] - try buildPrelude(flavor, builderArguments: builderArguments) { builderArguments in - try Builder( - arguments: builderArguments, - mainWasmPath: mainWasmPath, - flavor, - fileSystem, - terminal - ) - .runAndWaitUntilFinished() - } + try Builder( + arguments: builderArguments, + mainWasmPath: mainWasmPath, + flavor, + fileSystem, + terminal + ) + .runAndWaitUntilFinished() guard fileSystem.exists(mainWasmPath) else { terminal.write( @@ -283,7 +259,7 @@ public final class Toolchain { throw ToolchainError.failedToBuild(product: product.name) } - return (builderArguments, mainWasmPath, product) + return .init(arguments: builderArguments, mainWasmPath: mainWasmPath, product: product) } /// Returns an absolute path to the resulting test bundle @@ -307,16 +283,14 @@ public final class Toolchain { "-Xswiftc", "-color-diagnostics", ] - try buildPrelude(flavor, builderArguments: builderArguments) { builderArguments in - try Builder( - arguments: builderArguments, - mainWasmPath: testBundlePath, - flavor, - fileSystem, - terminal - ) - .runAndWaitUntilFinished() - } + try Builder( + arguments: builderArguments, + mainWasmPath: testBundlePath, + flavor, + fileSystem, + terminal + ) + .runAndWaitUntilFinished() guard fileSystem.exists(testBundlePath) else { terminal.write( diff --git a/Sources/SwiftToolchain/ToolchainManagement.swift b/Sources/SwiftToolchain/ToolchainManagement.swift index 542f163e..08c989af 100644 --- a/Sources/SwiftToolchain/ToolchainManagement.swift +++ b/Sources/SwiftToolchain/ToolchainManagement.swift @@ -259,7 +259,7 @@ public class ToolchainSystem { } public func fetchAllSwiftVersions() throws -> [String] { - try resolvers.flatMap { (try? $0.fetchVersions()) ?? [] } + resolvers.flatMap { (try? $0.fetchVersions()) ?? [] } .filter { fileSystem.isDirectory($0.path) } .map(\.version) .sorted() diff --git a/Sources/carton-release/HashArchive.swift b/Sources/carton-release/HashArchive.swift index 351eaa3a..bd6d38e2 100644 --- a/Sources/carton-release/HashArchive.swift +++ b/Sources/carton-release/HashArchive.swift @@ -15,6 +15,7 @@ import ArgumentParser import CartonHelpers import TSCBasic +import WasmTransformer struct HashArchive: ParsableCommand { /** Converts a hexadecimal hash string to Swift code that represents a static @@ -56,6 +57,12 @@ struct HashArchive: ParsableCommand { .hexadecimalRepresentation.uppercased()) } + try localFileSystem.writeFileContents( + staticPath.appending(component: "so_sanitizer.wasm"), + bytes: .init(StackOverflowSanitizer.supportObjectFile) + ) + print("file written to \(staticPath.appending(component: "so_sanitizer.wasm"))") + let archiveSources = try localFileSystem.traverseRecursively(staticPath) // `traverseRecursively` also returns the `staticPath` directory itself, dropping it here .dropFirst() diff --git a/Tests/Fixtures/TestApp/Package.resolved b/Tests/Fixtures/TestApp/Package.resolved index ad2bfcf7..6c392945 100644 --- a/Tests/Fixtures/TestApp/Package.resolved +++ b/Tests/Fixtures/TestApp/Package.resolved @@ -6,8 +6,8 @@ "repositoryURL": "https://github.com/swiftwasm/JavaScriptKit", "state": { "branch": null, - "revision": "b7a02434c6e973c08c3fd5069105ef44dd82b891", - "version": "0.9.0" + "revision": "b19e7c8b10a2750ed47753e31ed13613171f3294", + "version": "0.10.1" } } ] diff --git a/Tests/Fixtures/TestApp/Package.swift b/Tests/Fixtures/TestApp/Package.swift index 64ded5b8..5461bdc3 100644 --- a/Tests/Fixtures/TestApp/Package.swift +++ b/Tests/Fixtures/TestApp/Package.swift @@ -9,7 +9,7 @@ let package = Package( .executable(name: "TestApp", targets: ["TestApp"]), ], dependencies: [ - .package(url: "https://github.com/swiftwasm/JavaScriptKit", from: "0.9.0"), + .package(url: "https://github.com/swiftwasm/JavaScriptKit", from: "0.10.1"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test diff --git a/Tests/Fixtures/TestApp/Sources/TestApp/main.swift b/Tests/Fixtures/TestApp/Sources/TestApp/main.swift index f9f36605..0e2b9017 100644 --- a/Tests/Fixtures/TestApp/Sources/TestApp/main.swift +++ b/Tests/Fixtures/TestApp/Sources/TestApp/main.swift @@ -35,12 +35,13 @@ func crash() { } var buttonNode = document.getElementsByTagName("button")[0] -let handler = JSClosure { _ -> () in +let handler = JSClosure { _ in print(text) crash() + return .undefined } -buttonNode.onclick = .function(handler) +buttonNode.onclick = .object(handler) var div = document.createElement("div") div.innerHTML = .string(#""" From d005a24f9541934df1e5670531b1bf6ea21f63b8 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 13:33:31 +0100 Subject: [PATCH 02/18] Avoid building in `release` mode when testing --- .github/workflows/swift.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 871754ac..7c4415c8 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -24,9 +24,9 @@ jobs: # swift_version: 5.3 # xcode: /Applications/Xcode_12.2.app/Contents/Developer - os: ubuntu-18.04 - swift_version: 5.3 + swift_version: 5.4 - os: ubuntu-20.04 - swift_version: 5.3 + swift_version: 5.4 name: Build on ${{ matrix.os }} with Swift ${{ matrix.swift_version }} runs-on: ${{ matrix.os }} @@ -58,14 +58,14 @@ jobs: mkdir -p $HOME/.carton cp -r static $HOME/.carton/static - name: Build Project - run: swift build -c release --build-tests --enable-test-discovery + run: swift build - name: Run Tests run: | set -ex if [ -e /home/runner/.wasmer/wasmer.sh ]; then source /home/runner/.wasmer/wasmer.sh fi - swift test -c release --enable-test-discovery + swift test env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4d3096a7f06d9ab8bba8f4c381b25d41b49af599 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 13:39:01 +0100 Subject: [PATCH 03/18] Remove deprecated `LinuxMain.swift` --- Tests/LinuxMain.swift | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Tests/LinuxMain.swift diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift deleted file mode 100644 index a1e3520f..00000000 --- a/Tests/LinuxMain.swift +++ /dev/null @@ -1 +0,0 @@ -fatalError("Use `swift test --enable-test-discovery` to run tests") From d59aa7cec4ded968c0a2368543f8bf22e1d49cf9 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 13:52:18 +0100 Subject: [PATCH 04/18] Fix Swift 5.2 package manifest --- Package@swift-5.2.swift | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Package@swift-5.2.swift b/Package@swift-5.2.swift index 45a1decf..f3a105d6 100644 --- a/Package@swift-5.2.swift +++ b/Package@swift-5.2.swift @@ -33,7 +33,7 @@ let package = Package( .package(url: "https://github.com/vapor/vapor.git", from: "4.29.3"), .package(url: "https://github.com/apple/swift-crypto.git", from: "1.1.0"), .package(url: "https://github.com/JohnSundell/Splash.git", from: "0.14.0"), - .package(url: "https://github.com/swiftwasm/WasmTransformer", .upToNextMinor(from: "0.0.1")), + .package(url: "https://github.com/swiftwasm/WasmTransformer", .upToNextMinor(from: "0.0.3")), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module @@ -43,15 +43,6 @@ let package = Package( name: "Carton", dependencies: [ "CartonCLI", - // commented out for now. Will remove once confirmed working -// .product(name: "ArgumentParser", package: "swift-argument-parser"), -// .product(name: "AsyncHTTPClient", package: "async-http-client"), -// .product(name: "Crypto", package: "swift-crypto"), -// .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), -// .product(name: "Vapor", package: "vapor"), -// "CartonHelpers", -// openCombineProduct, -// "SwiftToolchain", ] ), .target( @@ -88,6 +79,7 @@ let package = Package( .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), "OpenCombine", "Splash", + "WasmTransformer", ] ), // This target is used only for release automation tasks and From 5b0247854e39867328c62d00272d6b305bc98988 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 13:54:30 +0100 Subject: [PATCH 05/18] Fix PR triggers --- .github/workflows/swift.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 7c4415c8..5e20fc9e 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - branches: [main] + branches: "*" jobs: swift-test: From b9b626adbaef3a41f5a4021bc1c95de61f8e34c3 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 14:24:58 +0100 Subject: [PATCH 06/18] Update `static.zip` release URL --- .vscode/tasks.json | 5 +++++ Sources/CartonKit/Model/Entrypoint.swift | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ecb3258d..1347322e 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -108,6 +108,11 @@ "type": "shell", "command": "swift build && cd Tests/Fixtures/TestApp && ../../../.build/debug/carton bundle --custom-index-page index.html" }, + { + "label": "test", + "type": "shell", + "command": "swift test" + }, { "type": "npm", "script": "build", diff --git a/Sources/CartonKit/Model/Entrypoint.swift b/Sources/CartonKit/Model/Entrypoint.swift index 9ba6b249..30e5eaae 100644 --- a/Sources/CartonKit/Model/Entrypoint.swift +++ b/Sources/CartonKit/Model/Entrypoint.swift @@ -22,7 +22,7 @@ import TSCUtility instead of the forthcoming release, because the corresponding new release tag doesn't exist yet. */ private let staticArchiveURL = - "https://github.com/swiftwasm/carton/releases/download/0.8.2/static.zip" + "https://github.com/swiftwasm/carton/releases/download/0.9.1/static.zip" private let verifyHash = Equality { """ From d9a729772bca5f93979ec8b8f7781cc95fdefdb0 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 14:30:23 +0100 Subject: [PATCH 07/18] Reduce the PR diff --- Sources/SwiftToolchain/Builder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftToolchain/Builder.swift b/Sources/SwiftToolchain/Builder.swift index 83b1c846..8aed919c 100644 --- a/Sources/SwiftToolchain/Builder.swift +++ b/Sources/SwiftToolchain/Builder.swift @@ -54,7 +54,7 @@ public final class Builder { let process = ProcessRunner( builderArguments, loadingMessage: "Compiling...", - parser: nil, + parser: DiagnosticsParser(), terminal ) currentProcess = process From 69637416d6e5f1b25f27450df1f7df1bbd73f724 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 15:06:54 +0100 Subject: [PATCH 08/18] Try to increase the timeout to allow for slower CI hosts --- Tests/CartonCommandTests/DevCommandTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index 1d5651d6..e14af322 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -91,7 +91,7 @@ final class DevCommandTests: XCTestCase { let timeOut: Int64 = 60 // client delay... let the server start up - let delay: UInt32 = 30 + let delay: UInt32 = 60 // only try 5 times. let polls = 5 From e904b3ebfa386de8ee547eba572905db12c2d179 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 21:02:20 +0100 Subject: [PATCH 09/18] Allow output debugging in test cases --- Tests/CartonCommandTests/CommandTestHelper.swift | 6 ++++-- Tests/CartonCommandTests/DevCommandTests.swift | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/CartonCommandTests/CommandTestHelper.swift b/Tests/CartonCommandTests/CommandTestHelper.swift index fc9dbbf1..89b33b97 100644 --- a/Tests/CartonCommandTests/CommandTestHelper.swift +++ b/Tests/CartonCommandTests/CommandTestHelper.swift @@ -197,11 +197,13 @@ public extension XCTest { /// Execute shell command and return the process the command is running in /// /// - parameter command: The command to execute. + /// - paramater shouldPrintOutput: whether the command output should be printed to stdout/stderr. /// - parameter cwd: The current working directory for executing the command. /// - parameter file: The file the assertion is coming from. /// - parameter line: The line the assertion is coming from. func executeCommand( command: String, + shouldPrintOutput: Bool = false, cwd: URL? = nil, // To allow for testing of file based output file: StaticString = #file, line: UInt = #line ) -> Process? { @@ -229,9 +231,9 @@ public extension XCTest { } let output = Pipe() - process.standardOutput = output + process.standardOutput = shouldPrintOutput ? FileHandle.standardOutput : output let error = Pipe() - process.standardError = error + process.standardError = shouldPrintOutput ? FileHandle.standardError : error if #available(macOS 10.13, *) { guard (try? process.run()) != nil else { diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index e14af322..ac2ee7eb 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -45,6 +45,7 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose", + shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") @@ -73,6 +74,7 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose --port 8081 --host 0.0.0.0", + shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") From 77dcb2afb6c0d800f05dc49f2ec437f8a6ddfdbf Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 21:03:32 +0100 Subject: [PATCH 10/18] Disable `DiagnosticsParser` for now --- Sources/SwiftToolchain/Builder.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftToolchain/Builder.swift b/Sources/SwiftToolchain/Builder.swift index 8aed919c..83b1c846 100644 --- a/Sources/SwiftToolchain/Builder.swift +++ b/Sources/SwiftToolchain/Builder.swift @@ -54,7 +54,7 @@ public final class Builder { let process = ProcessRunner( builderArguments, loadingMessage: "Compiling...", - parser: DiagnosticsParser(), + parser: nil, terminal ) currentProcess = process From 323525e3b4fc2bcc4008ce0b14a8265f2448ba62 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 22:25:26 +0100 Subject: [PATCH 11/18] Revert debug output printing change, fix CI --- .github/workflows/swift.yml | 12 ++++++------ Tests/CartonCommandTests/DevCommandTests.swift | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 5e20fc9e..7fad3b30 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -50,15 +50,15 @@ jobs: curl https://get.wasmer.io -sSfL | sh if: startsWith(matrix.os, 'ubuntu') - - name: Build and install JavaScript resources + - name: Build Project + run: swift build + + - name: Build and install JavaScript and sanitizer resources run: | - npm install - npm run build - rm -rf $HOME/.carton/static + swift run carton-release hash-archive mkdir -p $HOME/.carton cp -r static $HOME/.carton/static - - name: Build Project - run: swift build + - name: Run Tests run: | set -ex diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index ac2ee7eb..e14af322 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -45,7 +45,6 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose", - shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") @@ -74,7 +73,6 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose --port 8081 --host 0.0.0.0", - shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") From 79e6523240aa17dc0949d258d8b69932002d03e7 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 22:25:56 +0100 Subject: [PATCH 12/18] Revert the test delay change --- Tests/CartonCommandTests/DevCommandTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index e14af322..1d5651d6 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -91,7 +91,7 @@ final class DevCommandTests: XCTestCase { let timeOut: Int64 = 60 // client delay... let the server start up - let delay: UInt32 = 60 + let delay: UInt32 = 30 // only try 5 times. let polls = 5 From f6e25a430c6742e906471b303758f85445ab7e02 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Sun, 23 May 2021 22:40:26 +0100 Subject: [PATCH 13/18] Add back `npm install` to `swift.yml` --- .github/workflows/swift.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index cd84e5a9..ddcbd895 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -55,6 +55,7 @@ jobs: - name: Build and install JavaScript and sanitizer resources run: | + npm install swift run carton-release hash-archive mkdir -p $HOME/.carton cp -r static $HOME/.carton/static From e344ae4d8a39ecaa2429cbc87d7bb1c9ce1d4c2d Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 24 May 2021 10:35:02 +0100 Subject: [PATCH 14/18] Print command output in tests for debugging --- Tests/CartonCommandTests/DevCommandTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index 1d5651d6..1f022074 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -45,6 +45,7 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose", + shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") @@ -73,6 +74,7 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose --port 8081 --host 0.0.0.0", + shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") From 0b5dc3fc28c0a44b08513c6409b57cdb3d34a9de Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 24 May 2021 10:51:08 +0100 Subject: [PATCH 15/18] Debug executed commands in `swift.yml` --- .github/workflows/swift.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index ddcbd895..1c3b88f2 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -55,6 +55,7 @@ jobs: - name: Build and install JavaScript and sanitizer resources run: | + set -ex npm install swift run carton-release hash-archive mkdir -p $HOME/.carton From 63e25f80d70d721475d8435822bc9d091349107f Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 24 May 2021 11:04:40 +0100 Subject: [PATCH 16/18] Print `static` directory contents in `swift.yml` --- .github/workflows/swift.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 1c3b88f2..fca7edb9 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -60,6 +60,7 @@ jobs: swift run carton-release hash-archive mkdir -p $HOME/.carton cp -r static $HOME/.carton/static + ls -R $HOME/.carton/static - name: Run Tests run: | From fc749fcdf67718885f079617c277b89beeb83544 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 24 May 2021 11:10:07 +0100 Subject: [PATCH 17/18] Fix `static` copy step in `swift.yml` CI --- .github/workflows/swift.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index fca7edb9..7b671ea7 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -59,7 +59,7 @@ jobs: npm install swift run carton-release hash-archive mkdir -p $HOME/.carton - cp -r static $HOME/.carton/static + cp -r static $HOME/.carton ls -R $HOME/.carton/static - name: Run Tests From 6772a5de97be3980a6b48acc50c84608b7ebcbc4 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Mon, 24 May 2021 11:24:10 +0100 Subject: [PATCH 18/18] Clean up debug logging --- .github/workflows/swift.yml | 1 - Tests/CartonCommandTests/DevCommandTests.swift | 2 -- 2 files changed, 3 deletions(-) diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 7b671ea7..07e0f2e1 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -60,7 +60,6 @@ jobs: swift run carton-release hash-archive mkdir -p $HOME/.carton cp -r static $HOME/.carton - ls -R $HOME/.carton/static - name: Run Tests run: | diff --git a/Tests/CartonCommandTests/DevCommandTests.swift b/Tests/CartonCommandTests/DevCommandTests.swift index 1f022074..1d5651d6 100644 --- a/Tests/CartonCommandTests/DevCommandTests.swift +++ b/Tests/CartonCommandTests/DevCommandTests.swift @@ -45,7 +45,6 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose", - shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process") @@ -74,7 +73,6 @@ final class DevCommandTests: XCTestCase { guard let process = executeCommand( command: "carton dev --verbose --port 8081 --host 0.0.0.0", - shouldPrintOutput: true, cwd: packageDirectory.url ) else { XCTFail("Could not create process")