Auto merge of rust-lang#85993 - bjorn3:serde_json, r=wesleywiser

Remove all json handling from rustc_serialize

Json is now handled using serde_json. Where appropriate I have replaced json usage with binary serialization (rmeta files) or manual string formatting (emcc linker arg generation).

This allowed for removing and simplifying a lot of code, which hopefully results in faster serialization/deserialization and faster compiles of rustc itself.

Where sensible we now use serde. Metadata and incr cache serialization keeps using a heavily modified (compared to crates.io) rustc-serialize version that in the future could probably be extended with zero-copy deserialization or other perf tricks that serde can't support due to supporting more than one serialization format.

Note that I had to remove `-Zast-json` and `-Zast-json-noexpand` as the relevant AST types don't implement `serde::Serialize`.

Fixes rust-lang#40177

See also rust-lang/compiler-team#418

Cargo.lock

@@ -3686,6 +3686,7 @@ dependencies = [
  "rustc_span",
  "rustc_symbol_mangling",
  "rustc_target",
+ "serde_json",
  "smallvec",
  "snap",
  "tempfile",
@@ -3774,6 +3775,7 @@ dependencies = [
  "rustc_span",
  "rustc_target",
  "rustc_typeck",
+ "serde_json",
  "tracing",
  "winapi",
 ]
@@ -3809,6 +3811,8 @@ dependencies = [
  "rustc_macros",
  "rustc_serialize",
  "rustc_span",
+ "serde",
+ "serde_json",
  "termcolor",
  "termize",
  "tracing",
@@ -4024,6 +4028,7 @@ dependencies = [
  "rustc_serialize",
  "rustc_span",
  "rustc_target",
+ "serde",
 ]
 
 [[package]]
@@ -4445,6 +4450,7 @@ dependencies = [
  "rustc_macros",
  "rustc_serialize",
  "rustc_span",
+ "serde_json",
  "tracing",
 ]
 

compiler/rustc_ast/src/ast.rs

@@ -41,9 +41,6 @@ use std::convert::TryFrom;
 use std::fmt;
 use std::mem;
 
-#[cfg(test)]
-mod tests;
-
 /// A "Label" is an identifier of some point in sources,
 /// e.g. in the following code:
 ///
@@ -2476,8 +2473,8 @@ rustc_index::newtype_index! {
 }
 
 impl<S: Encoder> rustc_serialize::Encodable<S> for AttrId {
-    fn encode(&self, s: &mut S) -> Result<(), S::Error> {
-        s.emit_unit()
+    fn encode(&self, _s: &mut S) -> Result<(), S::Error> {
+        Ok(())
     }
 }
 

compiler/rustc_codegen_ssa/Cargo.toml

@@ -16,6 +16,7 @@ jobserver = "0.1.22"
 tempfile = "3.2"
 thorin-dwp = "0.2"
 pathdiff = "0.2.0"
+serde_json = "1.0.59"
 snap = "1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 regex = "1.4"

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -14,7 +14,6 @@ use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind};
 use rustc_middle::ty::TyCtxt;
-use rustc_serialize::{json, Encoder};
 use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
@@ -1152,21 +1151,12 @@ impl<'a> Linker for EmLinker<'a> {
         self.cmd.arg("-s");
 
         let mut arg = OsString::from("EXPORTED_FUNCTIONS=");
-        let mut encoded = String::new();
-
-        {
-            let mut encoder = json::Encoder::new(&mut encoded);
-            let res = encoder.emit_seq(symbols.len(), |encoder| {
-                for (i, sym) in symbols.iter().enumerate() {
-                    encoder.emit_seq_elt(i, |encoder| encoder.emit_str(&("_".to_owned() + sym)))?;
-                }
-                Ok(())
-            });
-            if let Err(e) = res {
-                self.sess.fatal(&format!("failed to encode exported symbols: {}", e));
-            }
-        }
+        let encoded = serde_json::to_string(
+            &symbols.iter().map(|sym| "_".to_owned() + sym).collect::<Vec<_>>(),
+        )
+        .unwrap();
         debug!("{}", encoded);
+
         arg.push(encoded);
 
         self.cmd.arg(arg);

compiler/rustc_driver/Cargo.toml

@@ -9,6 +9,7 @@ crate-type = ["dylib"]
 [dependencies]
 libc = "0.2"
 tracing = { version = "0.1.28" }
+serde_json = "1.0.59"
 rustc_log = { path = "../rustc_log" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }

compiler/rustc_driver/src/lib.rs

@@ -30,7 +30,6 @@ use rustc_log::stdout_isatty;
 use rustc_metadata::locator;
 use rustc_save_analysis as save;
 use rustc_save_analysis::DumpHandler;
-use rustc_serialize::json::ToJson;
 use rustc_session::config::{nightly_options, CG_OPTIONS, DB_OPTIONS};
 use rustc_session::config::{ErrorOutputType, Input, OutputType, PrintRequest, TrimmedDefPaths};
 use rustc_session::cstore::MetadataLoader;
@@ -40,6 +39,7 @@ use rustc_session::{config, DiagnosticOutput, Session};
 use rustc_session::{early_error, early_error_no_abort, early_warn};
 use rustc_span::source_map::{FileLoader, FileName};
 use rustc_span::symbol::sym;
+use rustc_target::json::ToJson;
 
 use std::borrow::Cow;
 use std::cmp::max;
@@ -343,10 +343,7 @@ fn run_compiler(
                 return early_exit();
             }
 
-            if sess.opts.debugging_opts.parse_only
-                || sess.opts.debugging_opts.show_span.is_some()
-                || sess.opts.debugging_opts.ast_json_noexpand
-            {
+            if sess.opts.debugging_opts.parse_only || sess.opts.debugging_opts.show_span.is_some() {
                 return early_exit();
             }
 
@@ -375,7 +372,7 @@ fn run_compiler(
 
             queries.global_ctxt()?;
 
-            if sess.opts.debugging_opts.no_analysis || sess.opts.debugging_opts.ast_json {
+            if sess.opts.debugging_opts.no_analysis {
                 return early_exit();
             }
 
@@ -665,7 +662,9 @@ fn print_crate_info(
             }
             Sysroot => println!("{}", sess.sysroot.display()),
             TargetLibdir => println!("{}", sess.target_tlib_path.dir.display()),
-            TargetSpec => println!("{}", sess.target.to_json().pretty()),
+            TargetSpec => {
+                println!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap());
+            }
             FileNames | CrateName => {
                 let input = input.unwrap_or_else(|| {
                     early_error(ErrorOutputType::default(), "no input file provided")

compiler/rustc_errors/Cargo.toml

@@ -19,6 +19,8 @@ atty = "0.2"
 termcolor = "1.0"
 annotate-snippets = "0.8.0"
 termize = "0.1.1"
+serde = { version = "1.0.125", features = ["derive"] }
+serde_json = "1.0.59"
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["handleapi", "synchapi", "winbase"] }

compiler/rustc_errors/src/json.rs

@@ -28,7 +28,7 @@ use std::path::Path;
 use std::sync::{Arc, Mutex};
 use std::vec;
 
-use rustc_serialize::json::{as_json, as_pretty_json};
+use serde::Serialize;
 
 #[cfg(test)]
 mod tests;
@@ -126,9 +126,9 @@ impl Emitter for JsonEmitter {
     fn emit_diagnostic(&mut self, diag: &crate::Diagnostic) {
         let data = Diagnostic::from_errors_diagnostic(diag, self);
         let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", as_pretty_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
         } else {
-            writeln!(&mut self.dst, "{}", as_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
         }
         .and_then(|_| self.dst.flush());
         if let Err(e) = result {
@@ -139,9 +139,9 @@ impl Emitter for JsonEmitter {
     fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) {
         let data = ArtifactNotification { artifact: path, emit: artifact_type };
         let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", as_pretty_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
         } else {
-            writeln!(&mut self.dst, "{}", as_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
         }
         .and_then(|_| self.dst.flush());
         if let Err(e) = result {
@@ -161,9 +161,9 @@ impl Emitter for JsonEmitter {
             .collect();
         let report = FutureIncompatReport { future_incompat_report: data };
         let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", as_pretty_json(&report))
+            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&report).unwrap())
         } else {
-            writeln!(&mut self.dst, "{}", as_json(&report))
+            writeln!(&mut self.dst, "{}", serde_json::to_string(&report).unwrap())
         }
         .and_then(|_| self.dst.flush());
         if let Err(e) = result {
@@ -175,9 +175,9 @@ impl Emitter for JsonEmitter {
         let lint_level = lint_level.as_str();
         let data = UnusedExterns { lint_level, unused_extern_names: unused_externs };
         let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", as_pretty_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
         } else {
-            writeln!(&mut self.dst, "{}", as_json(&data))
+            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
         }
         .and_then(|_| self.dst.flush());
         if let Err(e) = result {
@@ -204,7 +204,7 @@ impl Emitter for JsonEmitter {
 
 // The following data types are provided just for serialisation.
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct Diagnostic {
     /// The primary error message.
     message: String,
@@ -218,7 +218,7 @@ struct Diagnostic {
     rendered: Option<String>,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct DiagnosticSpan {
     file_name: String,
     byte_start: u32,
@@ -245,7 +245,7 @@ struct DiagnosticSpan {
     expansion: Option<Box<DiagnosticSpanMacroExpansion>>,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct DiagnosticSpanLine {
     text: String,
 
@@ -255,7 +255,7 @@ struct DiagnosticSpanLine {
     highlight_end: usize,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct DiagnosticSpanMacroExpansion {
     /// span where macro was applied to generate this code; note that
     /// this may itself derive from a macro (if
@@ -269,28 +269,28 @@ struct DiagnosticSpanMacroExpansion {
     def_site_span: DiagnosticSpan,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct DiagnosticCode {
     /// The code itself.
     code: String,
     /// An explanation for the code.
     explanation: Option<&'static str>,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct ArtifactNotification<'a> {
     /// The path of the artifact.
     artifact: &'a Path,
     /// What kind of artifact we're emitting.
     emit: &'a str,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct FutureBreakageItem {
     diagnostic: Diagnostic,
 }
 
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct FutureIncompatReport {
     future_incompat_report: Vec<FutureBreakageItem>,
 }
@@ -299,7 +299,7 @@ struct FutureIncompatReport {
 // doctest component (as well as cargo).
 // We could unify this struct the one in rustdoc but they have different
 // ownership semantics, so doing so would create wasteful allocations.
-#[derive(Encodable)]
+#[derive(Serialize)]
 struct UnusedExterns<'a, 'b, 'c> {
     /// The severity level of the unused dependencies lint
     lint_level: &'a str,

compiler/rustc_errors/src/json/tests.rs

@@ -5,12 +5,18 @@ use rustc_span::source_map::{FilePathMapping, SourceMap};
 
 use crate::emitter::{ColorConfig, HumanReadableErrorType};
 use crate::Handler;
-use rustc_serialize::json;
 use rustc_span::{BytePos, Span};
 
 use std::str;
 
-#[derive(Debug, PartialEq, Eq)]
+use serde::Deserialize;
+
+#[derive(Deserialize, Debug, PartialEq, Eq)]
+struct TestData {
+    spans: Vec<SpanTestData>,
+}
+
+#[derive(Deserialize, Debug, PartialEq, Eq)]
 struct SpanTestData {
     pub byte_start: u32,
     pub byte_end: u32,
@@ -61,19 +67,11 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
 
         let bytes = output.lock().unwrap();
         let actual_output = str::from_utf8(&bytes).unwrap();
-        let actual_output = json::from_str(&actual_output).unwrap();
-        let spans = actual_output["spans"].as_array().unwrap();
+        let actual_output: TestData = serde_json::from_str(actual_output).unwrap();
+        let spans = actual_output.spans;
         assert_eq!(spans.len(), 1);
-        let obj = &spans[0];
-        let actual_output = SpanTestData {
-            byte_start: obj["byte_start"].as_u64().unwrap() as u32,
-            byte_end: obj["byte_end"].as_u64().unwrap() as u32,
-            line_start: obj["line_start"].as_u64().unwrap() as u32,
-            line_end: obj["line_end"].as_u64().unwrap() as u32,
-            column_start: obj["column_start"].as_u64().unwrap() as u32,
-            column_end: obj["column_end"].as_u64().unwrap() as u32,
-        };
-        assert_eq!(expected_output, actual_output);
+
+        assert_eq!(expected_output, spans[0])
     })
 }
 

compiler/rustc_interface/src/passes.rs

@@ -27,7 +27,6 @@ use rustc_passes::{self, hir_stats, layout_test};
 use rustc_plugin_impl as plugin;
 use rustc_query_impl::{OnDiskCache, Queries as TcxQueries};
 use rustc_resolve::{Resolver, ResolverArenas};
-use rustc_serialize::json;
 use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType};
 use rustc_session::cstore::{MetadataLoader, MetadataLoaderDyn};
 use rustc_session::output::{filename_for_input, filename_for_metadata};
@@ -59,10 +58,6 @@ pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
         }
     })?;
 
-    if sess.opts.debugging_opts.ast_json_noexpand {
-        println!("{}", json::as_json(&krate));
-    }
-
     if sess.opts.debugging_opts.input_stats {
         eprintln!("Lines of code:             {}", sess.source_map().count_lines());
         eprintln!("Pre-expansion node count:  {}", count_nodes(&krate));
@@ -423,10 +418,6 @@ pub fn configure_and_expand(
         hir_stats::print_ast_stats(&krate, "POST EXPANSION AST STATS");
     }
 
-    if sess.opts.debugging_opts.ast_json {
-        println!("{}", json::as_json(&krate));
-    }
-
     resolver.resolve_crate(&krate);
 
     // Needs to go *after* expansion to be able to check the results of macro expansion.