From 3d0230f46958a7821ac64493303e3c6df61153b4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 21 Jun 2024 17:53:10 +0100 Subject: [PATCH] [red-knot] Add more tests asserting that the VendoredFileSystem and the `VERSIONS` parser work with the vendored typeshed stubs (#11970) --- Cargo.lock | 1 + crates/red_knot_module_resolver/Cargo.toml | 2 + .../red_knot_module_resolver/src/typeshed.rs | 74 ++++++++++++++++--- .../src/typeshed/versions.rs | 57 ++++++++++++++ 4 files changed, 125 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba4dccc5aa4e0..55562d269138d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1997,6 +1997,7 @@ version = "0.0.0" dependencies = [ "anyhow", "insta", + "path-slash", "ruff_db", "ruff_python_stdlib", "rustc-hash", diff --git a/crates/red_knot_module_resolver/Cargo.toml b/crates/red_knot_module_resolver/Cargo.toml index 2d2f256ab7d53..3d98f98b7de13 100644 --- a/crates/red_knot_module_resolver/Cargo.toml +++ b/crates/red_knot_module_resolver/Cargo.toml @@ -27,7 +27,9 @@ zip = { workspace = true } [dev-dependencies] anyhow = { workspace = true } insta = { workspace = true } +path-slash = { workspace = true } tempfile = { workspace = true } +walkdir = { workspace = true } [lints] workspace = true diff --git a/crates/red_knot_module_resolver/src/typeshed.rs b/crates/red_knot_module_resolver/src/typeshed.rs index bf7369e328bb7..fcec52b5ce552 100644 --- a/crates/red_knot_module_resolver/src/typeshed.rs +++ b/crates/red_knot_module_resolver/src/typeshed.rs @@ -5,14 +5,20 @@ mod tests { use std::io::{self, Read}; use std::path::Path; - #[test] - fn typeshed_zip_created_at_build_time() -> anyhow::Result<()> { - // The file path here is hardcoded in this crate's `build.rs` script. - // Luckily this crate will fail to build if this file isn't available at build time. - const TYPESHED_ZIP_BYTES: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); + use path_slash::PathExt; + + use ruff_db::vendored::VendoredFileSystem; + use ruff_db::vfs::VendoredPath; - let mut typeshed_zip_archive = zip::ZipArchive::new(io::Cursor::new(TYPESHED_ZIP_BYTES))?; + // The file path here is hardcoded in this crate's `build.rs` script. + // Luckily this crate will fail to build if this file isn't available at build time. + const TYPESHED_ZIP_BYTES: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); + + #[test] + fn typeshed_zip_created_at_build_time() { + let mut typeshed_zip_archive = + zip::ZipArchive::new(io::Cursor::new(TYPESHED_ZIP_BYTES)).unwrap(); let path_to_functools = Path::new("stdlib").join("functools.pyi"); let mut functools_module_stub = typeshed_zip_archive @@ -21,9 +27,59 @@ mod tests { assert!(functools_module_stub.is_file()); let mut functools_module_stub_source = String::new(); - functools_module_stub.read_to_string(&mut functools_module_stub_source)?; + functools_module_stub + .read_to_string(&mut functools_module_stub_source) + .unwrap(); assert!(functools_module_stub_source.contains("def update_wrapper(")); - Ok(()) + } + + #[test] + fn typeshed_vfs_consistent_with_vendored_stubs() { + let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap(); + let vendored_typeshed_stubs = VendoredFileSystem::new(TYPESHED_ZIP_BYTES).unwrap(); + + let mut empty_iterator = true; + for entry in walkdir::WalkDir::new(&vendored_typeshed_dir).min_depth(1) { + empty_iterator = false; + let entry = entry.unwrap(); + let absolute_path = entry.path(); + let file_type = entry.file_type(); + + let relative_path = absolute_path + .strip_prefix(&vendored_typeshed_dir) + .unwrap_or_else(|_| { + panic!("Expected {absolute_path:?} to be a child of {vendored_typeshed_dir:?}") + }); + + let posix_style_path = relative_path + .to_slash() + .unwrap_or_else(|| panic!("Expected {relative_path:?} to be a valid UTF-8 path")); + + let vendored_path = VendoredPath::new(&*posix_style_path); + + assert!( + vendored_typeshed_stubs.exists(vendored_path), + "Expected {vendored_path:?} to exist in the `VendoredFileSystem`!" + ); + + let vendored_path_kind = vendored_typeshed_stubs + .metadata(vendored_path) + .unwrap_or_else(|| { + panic!("Expected metadata for {vendored_path:?} to be retrievable from the `VendoredFileSystem!") + }) + .kind(); + + assert_eq!( + vendored_path_kind.is_directory(), + file_type.is_dir(), + "{vendored_path:?} had type {vendored_path_kind:?}, inconsistent with fs path {relative_path:?}: {file_type:?}" + ); + } + + assert!( + !empty_iterator, + "Expected there to be at least one file or directory in the vendored typeshed stubs!" + ); } } diff --git a/crates/red_knot_module_resolver/src/typeshed/versions.rs b/crates/red_knot_module_resolver/src/typeshed/versions.rs index e247fbc9dc2f5..aea7b2cab494c 100644 --- a/crates/red_knot_module_resolver/src/typeshed/versions.rs +++ b/crates/red_knot_module_resolver/src/typeshed/versions.rs @@ -308,11 +308,14 @@ impl From for PyVersion { #[cfg(test)] mod tests { use std::num::{IntErrorKind, NonZeroU16}; + use std::path::Path; use super::*; use insta::assert_snapshot; + const TYPESHED_STDLIB_DIR: &str = "stdlib"; + #[allow(unsafe_code)] const ONE: NonZeroU16 = unsafe { NonZeroU16::new_unchecked(1) }; @@ -345,6 +348,60 @@ mod tests { assert!(!versions.module_exists_on_version(audioop, SupportedPyVersion::Py313)); } + #[test] + fn typeshed_versions_consistent_with_vendored_stubs() { + const VERSIONS_DATA: &str = include_str!("../../vendor/typeshed/stdlib/VERSIONS"); + let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap(); + let vendored_typeshed_versions = TypeshedVersions::from_str(VERSIONS_DATA).unwrap(); + + let mut empty_iterator = true; + + let stdlib_stubs_path = vendored_typeshed_dir.join(TYPESHED_STDLIB_DIR); + + for entry in std::fs::read_dir(&stdlib_stubs_path).unwrap() { + empty_iterator = false; + let entry = entry.unwrap(); + let absolute_path = entry.path(); + + let relative_path = absolute_path + .strip_prefix(&stdlib_stubs_path) + .unwrap_or_else(|_| panic!("Expected path to be a child of {stdlib_stubs_path:?} but found {absolute_path:?}")); + + let relative_path_str = relative_path.as_os_str().to_str().unwrap_or_else(|| { + panic!("Expected all typeshed paths to be valid UTF-8; got {relative_path:?}") + }); + if relative_path_str == "VERSIONS" { + continue; + } + + let top_level_module = if let Some(extension) = relative_path.extension() { + // It was a file; strip off the file extension to get the module name: + let extension = extension + .to_str() + .unwrap_or_else(||panic!("Expected all file extensions to be UTF-8; was not true for {relative_path:?}")); + + relative_path_str + .strip_suffix(extension) + .and_then(|string| string.strip_suffix('.')).unwrap_or_else(|| { + panic!("Expected path {relative_path_str:?} to end with computed extension {extension:?}") + }) + } else { + // It was a directory; no need to do anything to get the module name + relative_path_str + }; + + let top_level_module = ModuleName::new(top_level_module) + .unwrap_or_else(|| panic!("{top_level_module:?} was not a valid module name!")); + + assert!(vendored_typeshed_versions.contains_module(&top_level_module)); + } + + assert!( + !empty_iterator, + "Expected there to be at least one file or directory in the vendored typeshed stubs" + ); + } + #[test] fn can_parse_mock_versions_file() { const VERSIONS: &str = "\