Update Rust

default.nix

@@ -32,9 +32,15 @@ let
     llvmPackages_12.bintools
     rustc-nightly
     cargo-nightly
-    xargo
     wasmtime
     rust-bindgen
+    python3
+    tree
+  ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
+    libiconv
+    # The following are usually also necessary:
+    # pkgs.darwin.apple_sdk.frameworks.Security
+    # pkgs.darwin.apple_sdk.frameworks.CoreFoundation
   ];
 
   llvmEnv = ''
@@ -152,46 +158,104 @@ in
 rec {
   rts =
     let
-      # We run this on motoko-rts-tests, to get the union of all
-      # dependencies
-      rustDeps = nixpkgs.rustPlatform-nightly.fetchCargoTarball {
+      # Build Rust package cargo-vendor-tools
+      cargoVendorTools = nixpkgs.rustPlatform.buildRustPackage rec {
+        name = "cargo-vendor-tools";
+        src = ./rts/cargo-vendor-tools;
+        cargoSha256 = "0zi3fiq9sy6c9dv7fd2xc9lan85d16gfax47n6g6f5q5c1zb5r47";
+      };
+
+      # Path to vendor-rust-std-deps, provided by cargo-vendor-tools
+      vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps";
+
+      # Vendors Rust std deps in the output directory
+      rustStdDepsVendor = rust: sha256: nixpkgs.stdenvNoCC.mkDerivation {
+        name = "rustc-std-deps";
+
+        nativeBuildInputs = with nixpkgs; [
+          curl
+        ];
+
+        buildCommand = ''
+          mkdir $out
+          cd $out
+          ${vendorRustStdDeps} ${rust} .
+        '';
+
+        outputHash = sha256;
+        outputHashAlgo = "sha256";
+        outputHashMode = "recursive";
+      };
+
+      # SHA256 of Rust std deps
+      rustStdDepsHash = "0wxx8prh66i19vd5078iky6x5bzs6ppz7c1vbcyx9h4fg0f7pfj6";
+
+      # Vendor directory for Rust std deps
+      rustStdDeps = rustStdDepsVendor nixpkgs.rustc-nightly rustStdDepsHash;
+
+      # Vendor tarball of the RTS
+      rtsDeps = nixpkgs.rustPlatform.fetchCargoTarball {
         name = "motoko-rts-deps";
         src = subpath ./rts;
         sourceRoot = "rts/motoko-rts-tests";
-        sha256 = "0jyp3j8n5bj5cy1fd26d7h55zmc4v14qc2w8adxqwmsv5riqz41g";
+        sha256 = "129gfmn96vm7di903pxirg7zybl83q6nkwiqr3rsy7l1q8667kxx";
         copyLockfile = true;
       };
+
+      # Unpacked RTS deps
+      rtsDepsUnpacked = nixpkgs.stdenvNoCC.mkDerivation {
+        name = rtsDeps.name + "-unpacked";
+        buildCommand = ''
+          tar xf ${rtsDeps}
+          mv *.tar.gz $out
+        '';
+      };
+
+      # Merges two vendor directories (just copies dir contents into $out)
+      mergeDeps = rtsDeps: rustStdDeps:
+        nixpkgs.stdenvNoCC.mkDerivation {
+          name = "merged-rust-deps";
+
+          buildCommand = ''
+            mkdir -p $out
+            cp -r ${rtsDeps}/* $out/
+            cp -r ${rustStdDeps}/* $out/
+          '';
+        };
+
+      # All dependencies needed to build the RTS, including Rust std deps, to
+      # allow `cargo -Zbuild-std`. (rust-lang/wg-cargo-std-aware#23)
+      allDeps = mergeDeps rtsDepsUnpacked rustStdDeps;
     in
 
     stdenv.mkDerivation {
       name = "moc-rts";
 
       src = subpath ./rts;
-      nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo ];
+
+      nativeBuildInputs = [ nixpkgs.makeWrapper nixpkgs.removeReferencesTo nixpkgs.cacert ];
 
       buildInputs = rtsBuildInputs;
 
       preBuild = ''
-        export XARGO_HOME=$PWD/xargo-home
         export CARGO_HOME=$PWD/cargo-home
 
-        # this replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix
+        # This replicates logic from nixpkgs’ pkgs/build-support/rust/default.nix
         mkdir -p $CARGO_HOME
-        echo "Using vendored sources from ${rustDeps}"
-        unpackFile ${rustDeps}
+        echo "Using vendored sources from ${rtsDeps}"
+        unpackFile ${allDeps}
         cat > $CARGO_HOME/config <<__END__
           [source."crates-io"]
           "replace-with" = "vendored-sources"
 
           [source."vendored-sources"]
-          "directory" = "$(stripHash ${rustDeps})"
+          "directory" = "$(stripHash ${allDeps})"
         __END__
 
         ${llvmEnv}
         export TOMMATHSRC=${nixpkgs.sources.libtommath}
         export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
         export MUSL_WASI_SYSROOT=${musl-wasi-sysroot}
-
       '';
 
       doCheck = true;
@@ -206,11 +270,16 @@ rec {
         cp mo-rts-debug.wasm $out/rts
       '';
 
-      # This needs to be self-contained. Remove mention of
-      # nix path in debug message.
+      # This needs to be self-contained. Remove mention of nix path in debug
+      # message.
       preFixup = ''
-        remove-references-to -t ${nixpkgs.rustc-nightly} -t ${rustDeps} $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm
+        remove-references-to \
+          -t ${nixpkgs.rustc-nightly} \
+          -t ${rtsDeps} \
+          -t ${rustStdDeps} \
+          $out/rts/mo-rts.wasm $out/rts/mo-rts-debug.wasm
       '';
+
       allowedRequisites = [];
     };
 
@@ -440,7 +509,6 @@ rec {
 
   filecheck = nixpkgs.linkFarm "FileCheck"
     [ { name = "bin/FileCheck"; path = "${nixpkgs.llvm}/bin/FileCheck";} ];
-  inherit (nixpkgs) xargo;
 
   # gitMinimal is used by nix/gitSource.nix; building it here warms the nix cache
   inherit (nixpkgs) gitMinimal;

nix/default.nix

@@ -46,7 +46,7 @@ let
         # Rust nightly
         (self: super: let
           moz_overlay = import self.sources.nixpkgs-mozilla self super;
-          rust-channel = moz_overlay.rustChannelOf { date = "2020-07-22"; channel = "nightly"; };
+          rust-channel = moz_overlay.rustChannelOf { date = "2021-10-25"; channel = "nightly"; };
         in rec {
           rustc-nightly = rust-channel.rust.override {
             targets = [
@@ -62,7 +62,6 @@ let
             rustc = rustc-nightly;
             cargo = cargo-nightly;
           };
-          xargo = self.callPackage ./xargo.nix {};
         })
 
         # wasm-profiler

rts/Makefile

@@ -223,11 +223,11 @@ $(TOMMATH_BINDINGS_RS): | _build
 
 
 $(RTS_RUST_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && xargo build --release --target=wasm32-unknown-emscripten
+	cd motoko-rts && cargo build --release --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
 	cp motoko-rts/target/wasm32-unknown-emscripten/release/libmotoko_rts.a $@
 
 $(RTS_RUST_DEBUG_WASM_A): $(TOMMATH_BINDINGS_RS) $(RTS_RUST_FILES) $(RTS_CARGO_FILES) | _build/wasm
-	cd motoko-rts && xargo build --target=wasm32-unknown-emscripten
+	cd motoko-rts && cargo build --target=wasm32-unknown-emscripten -Zbuild-std=core,alloc
 	cp motoko-rts/target/wasm32-unknown-emscripten/debug/libmotoko_rts.a $@
 
 #
@@ -286,5 +286,4 @@ clean:
 	  mo-rts-debug.wasm \
 	  motoko-rts/target \
 	  motoko-rts-tests/target \
-	  motoko-rts/xargo-home \
 	  motoko-rts/cargo-home

rts/README.md

@@ -58,31 +58,36 @@ See `motoko-rts/src/bigint.rs` for the technical details.
 Rust build
 ----------
 
-The Rust parts are built from `motoko-rts`, using `xargo` and `cargo`.
+To build Motoko RTS in nix we need pre-fetch Rust dependencies. This works in
+`nix-build` by:
 
-To build this in nix, we need pre-fetch some dependencies (currently
-`compiler_builtins` and `libc`). This works in `nix-build` by:
-
- * Adding`compiler_builtins` as a dependency in `Cargo.toml` (even though not
-   needed), so that it shows up with a hash in `Cargo.lock`
-
- * Building a directory with vendored sources in `default.nix` (see `rustDeps`)
+ * Building a directory with vendored sources in `default.nix`
 
  * Configuring `cargo` to use that vendored directory (see `preBuild`)
 
 If you change dependencies (e.g. bump versions, add more crates),
 
- 1. Add them to `Cargo.toml`
- 2. Make sure that `Cargo.lock` is up to date
- 3. In `default.nix`, invalidate the `sha256` of `rustDeps` (e.g. change one
+ 1. Make sure that `motoko-rts-tests/Cargo.lock` is up to date. This can be
+    done by running `cargo build --target=wasm32-wasi` in `motoko-rts-tests/`
+    directory.
+ 2. In `default.nix`, invalidate the `sha256` of `rtsDeps` (e.g. change one
     character)
- 4. Run `nix-build -A rts`. You should get an error message about the actual
+ 3. Run `nix-build -A rts`. You should get an error message about the actual
     checksum.
- 5. Set that as `sha256` of `rustDeps` in `default.nix`
+ 4. Set that as `sha256` of `rtsDeps` in `default.nix`
 
 Warning: nix will happily use a stale version of the dependencies if you do not
 do step 3.
 
+**Updating rustc**:
+
+1. Update Rust version in `nix/default.nix`, in the line with
+   `moz_overlay.rustChannelOf { ... }`.
+2. Invalidate `rustStdDepsHash` in `default.nix`.
+3. Run `nix-build -A rts`. You should get an error message about the expected
+   value of `rustStdDepsHash`.
+4. Update `rustStdDepsHash` with the expected value in the error message.
+
 Running RTS tests
 -----------------